diff --git a/.gitignore b/.gitignore index e9a4e11b85..777a52efc7 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,4 @@ rebuild-docker.sh */.run/** .run/** .run +.claude/ diff --git a/application/src/main/data/json/system/widget_types/bar_chart_with_labels.json b/application/src/main/data/json/system/widget_types/bar_chart_with_labels.json index 7e0bfe1e66..f23747fb52 100644 --- a/application/src/main/data/json/system/widget_types/bar_chart_with_labels.json +++ b/application/src/main/data/json/system/widget_types/bar_chart_with_labels.json @@ -11,7 +11,7 @@ "resources": [], "templateHtml": "\n", "templateCss": ".legend {\n font-size: 13px;\n line-height: 10px;\n}\n\n.legend table { \n border-spacing: 0px;\n border-collapse: separate;\n}\n\n.mouse-events .flot-overlay {\n cursor: crosshair; \n}\n\n", - "controllerScript": "self.onInit = function() {\n self.ctx.$scope.barChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.barChartWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n supportsUnitConversion: true,\n hasAdditionalLatestDataKeys: false,\n defaultDataKeysFunction: function() {\n return [{ name: 'humidity', label: 'Humidity', type: 'timeseries' }];\n }\n };\n}\n", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.barChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.barChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.barChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n supportsUnitConversion: true,\n hasAdditionalLatestDataKeys: false,\n defaultDataKeysFunction: function() {\n return [{ name: 'humidity', label: 'Humidity', type: 'timeseries' }];\n }\n };\n}\n", "settingsForm": [], "dataKeySettingsForm": [], "latestDataKeySettingsForm": [], diff --git a/application/src/main/data/json/system/widget_types/range_chart.json b/application/src/main/data/json/system/widget_types/range_chart.json index 5d8293debe..1e033ecb2b 100644 --- a/application/src/main/data/json/system/widget_types/range_chart.json +++ b/application/src/main/data/json/system/widget_types/range_chart.json @@ -11,7 +11,7 @@ "resources": [], "templateHtml": "\n", "templateCss": ".legend {\n font-size: 13px;\n line-height: 10px;\n}\n\n.legend table { \n border-spacing: 0px;\n border-collapse: separate;\n}\n\n.mouse-events .flot-overlay {\n cursor: crosshair; \n}\n\n", - "controllerScript": "self.onInit = function() {\n self.ctx.$scope.rangeChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.rangeChartWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n maxDataKeys: 1,\n singleEntity: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n supportsUnitConversion: true,\n hasAdditionalLatestDataKeys: false,\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries' }];\n }\n };\n}\n", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.rangeChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.rangeChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.rangeChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n maxDataKeys: 1,\n singleEntity: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n supportsUnitConversion: true,\n hasAdditionalLatestDataKeys: false,\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries' }];\n }\n };\n}\n", "settingsForm": [], "dataKeySettingsForm": [], "latestDataKeySettingsForm": [], diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java index bb025a5fcf..cde3129d53 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; import org.thingsboard.server.common.data.query.AvailableEntityKeys; +import org.thingsboard.server.common.data.query.AvailableEntityKeysV2; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; @@ -47,6 +48,8 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.query.EntityQueryService; import org.thingsboard.server.service.security.permission.Operation; +import java.util.Set; + import static org.thingsboard.server.controller.ControllerConstants.ALARM_DATA_QUERY_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_COUNT_QUERY_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_DATA_QUERY_DESCRIPTION; @@ -115,9 +118,11 @@ public class EntityQueryController extends BaseController { return entityQueryService.countAlarmsByQuery(getCurrentUser(), query); } + @Deprecated(forRemoval = true) @ApiOperation( - value = "Find Available Entity Keys by Query", + value = "Find Available Entity Keys by Query (deprecated)", notes = """ + **Deprecated.** Use the V2 endpoint (`POST /api/v2/entitiesQuery/find/keys`) instead.\n Returns unique time series and/or attribute key names from entities matching the query.\n Executes the Entity Data Query to find up to 100 entities, then fetches and aggregates all distinct key names.\n Primarily used for UI features like autocomplete suggestions.""" + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH @@ -128,9 +133,6 @@ public class EntityQueryController extends BaseController { @Parameter(description = "Entity data query to find entities. Page size is capped at 100.") @RequestBody EntityDataQuery query, - // fixme: combination of timeseries = false and attributes = false is allowed, but always results in empty response, therefore does not make any sense - // such combinations should NOT be allowed, but changing this will break clients - @Parameter(description = """ When true, includes unique time series key names in the response. When false, the 'timeseries' list will be empty.""") @@ -155,6 +157,54 @@ public class EntityQueryController extends BaseController { return wrapFuture(entityQueryService.getKeysByQuery(getCurrentUser(), getTenantId(), query, includeTimeseries, includeAttributes, scope)); } + @ApiOperation( + value = "Find Available Entity Keys By Query", + notes = """ + Discovers unique time series and/or attribute key names available on entities that match the given query. + Works in two steps: first, the request body (an Entity Data Query) is executed to find matching entities + (page size is capped at 100); then, all distinct key names are collected from those entities.\n + Optionally, each key can include a sample — the most recent value (by timestamp) for that key + across all matched entities.""" + + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH + ) + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") + @PostMapping("/v2/entitiesQuery/find/keys") + public DeferredResult findAvailableEntityKeysByQueryV2( + @Parameter(description = "Entity data query to find entities. Page size is capped at 100.") + @RequestBody EntityDataQuery query, + + @Parameter(description = """ + When true, includes unique time series keys in the response. + When false, the 'timeseries' field is omitted. At least one of 'includeTimeseries' or 'includeAttributes' must be true.""") + @RequestParam(defaultValue = "true") boolean includeTimeseries, + + @Parameter(description = """ + When true, includes unique attribute keys in the response. + When false, the 'attributes' field is omitted. At least one of 'includeTimeseries' or 'includeAttributes' must be true.""") + @RequestParam(defaultValue = "true") boolean includeAttributes, + + @Parameter(description = """ + Filters attribute keys by scope. Only applies when 'includeAttributes' is true. + When not specified, scopes are auto-determined: all three scopes (server, client, shared) for device entities, + server scope only for other entity types.""", + schema = @Schema(allowableValues = {"SERVER_SCOPE", "SHARED_SCOPE", "CLIENT_SCOPE"})) + @RequestParam(required = false) Set scopes, + + @Parameter(description = """ + When true, each key entry includes a 'sample' object with the most recent value and timestamp. + When false, only key names are returned (sample is omitted from JSON).""") + @RequestParam(defaultValue = "false") boolean includeSamples + ) throws ThingsboardException { + resolveQuery(query); + EntityDataPageLink pageLink = query.getPageLink(); + if (pageLink.getPageSize() > MAX_PAGE_SIZE) { + pageLink.setPageSize(MAX_PAGE_SIZE); + } + return wrapFuture(entityQueryService.findAvailableEntityKeysByQuery( + getCurrentUser(), query, + includeTimeseries, includeAttributes, scopes, includeSamples)); + } + @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") @PostMapping("/edqs/system/request") public void processSystemEdqsRequest(@RequestBody ToCoreEdqsRequest request) { diff --git a/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java b/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java index 4d76989a92..50f0f0e75a 100644 --- a/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java +++ b/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java @@ -15,13 +15,15 @@ */ package org.thingsboard.server.service.query; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.KvUtil; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; @@ -30,11 +32,17 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; import org.thingsboard.server.common.data.query.AvailableEntityKeys; +import org.thingsboard.server.common.data.query.AvailableEntityKeysV2; +import org.thingsboard.server.common.data.query.AvailableEntityKeysV2.KeyInfo; +import org.thingsboard.server.common.data.query.AvailableEntityKeysV2.KeySample; import org.thingsboard.server.common.data.query.ComplexFilterPredicate; import org.thingsboard.server.common.data.query.DynamicValue; import org.thingsboard.server.common.data.query.EntityCountQuery; @@ -59,11 +67,13 @@ import org.thingsboard.server.service.security.model.SecurityUser; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; @@ -253,7 +263,7 @@ public class DefaultEntityQueryService implements EntityQueryService { if (isAttributes) { Map> typesMap = ids.stream().collect(Collectors.groupingBy(EntityId::getEntityType)); List>> futures = new ArrayList<>(typesMap.size()); - typesMap.forEach((type, entityIds) -> futures.add(dbCallbackExecutor.submit(() -> attributesService.findAllKeysByEntityIds(tenantId, entityIds, scope)))); + typesMap.forEach((type, entityIds) -> futures.add(dbCallbackExecutor.submit(() -> attributesService.findAllKeysByEntityIdsAndScope(tenantId, entityIds, scope)))); attributesKeysFuture = Futures.transform(Futures.allAsList(futures), lists -> { if (CollectionUtils.isEmpty(lists)) { return Collections.emptyList(); @@ -274,4 +284,135 @@ public class DefaultEntityQueryService implements EntityQueryService { }, dbCallbackExecutor); } + @Override + public ListenableFuture findAvailableEntityKeysByQuery(SecurityUser securityUser, EntityDataQuery query, + boolean includeTimeseries, boolean includeAttributes, + Set scopes, boolean includeSamples) { + if (!includeTimeseries && !includeAttributes) { + return Futures.immediateFailedFuture( + new IllegalArgumentException("At least one of 'includeTimeseries' or 'includeAttributes' must be true")); + } + + return Futures.transformAsync(findEntityIdsByQueryAsync(securityUser, query), ids -> { + if (ids.isEmpty()) { + return immediateFuture(new AvailableEntityKeysV2( + Collections.emptySet(), + includeTimeseries ? Collections.emptyList() : null, + includeAttributes ? Collections.emptyMap() : null)); + } + + TenantId tenantId = securityUser.getTenantId(); + Set entityTypes = ids.stream().map(EntityId::getEntityType).collect(Collectors.toSet()); + + var tsFuture = includeTimeseries ? fetchTimeseriesKeys(tenantId, ids, includeSamples) : null; + + Set effectiveScopes = includeAttributes + ? resolveAttributeScopes(scopes, entityTypes) : Collections.emptySet(); + var attrFutures = effectiveScopes.stream() + .map(scope -> fetchAttributeKeys(tenantId, ids, scope, includeSamples)) + .toList(); + + return assembleResult(entityTypes, tsFuture, attrFutures); + }, dbCallbackExecutor); + } + + private ListenableFuture> findEntityIdsByQueryAsync(SecurityUser securityUser, EntityDataQuery query) { + return Futures.transform(entityService.findEntityDataByQueryAsync(securityUser.getTenantId(), securityUser.getCustomerId(), query), + page -> page.getData().stream() + .map(EntityData::getEntityId) + .toList(), + dbCallbackExecutor); + } + + private static Set resolveAttributeScopes(Set requestedScopes, Set entityTypes) { + boolean hasDevices = entityTypes.contains(EntityType.DEVICE); + Set scopes; + if (CollectionUtils.isNotEmpty(requestedScopes)) { + scopes = requestedScopes; + } else { // auto-determine scopes + scopes = hasDevices + ? Set.of(AttributeScope.SERVER_SCOPE, AttributeScope.CLIENT_SCOPE, AttributeScope.SHARED_SCOPE) + : Collections.singleton(AttributeScope.SERVER_SCOPE); + } + // Non-device entities only support SERVER_SCOPE + if (!hasDevices) { + return scopes.contains(AttributeScope.SERVER_SCOPE) + ? Collections.singleton(AttributeScope.SERVER_SCOPE) + : Collections.emptySet(); + } + return scopes; + } + + private ListenableFuture> fetchTimeseriesKeys(TenantId tenantId, List entityIds, boolean includeSamples) { + if (includeSamples) { + return Futures.transform( + timeseriesService.findLatestByEntityIdsAsync(tenantId, entityIds), + entries -> toKeyInfos(entries, true), + dbCallbackExecutor); + } + return Futures.transform( + timeseriesService.findAllKeysByEntityIdsAsync(tenantId, entityIds), + keys -> keys.stream().sorted().map(k -> new KeyInfo(k, null)).toList(), + dbCallbackExecutor); + } + + private ListenableFuture>> fetchAttributeKeys( + TenantId tenantId, List entityIds, AttributeScope scope, boolean includeSamples) { + if (includeSamples) { + return Futures.transform( + attributesService.findLatestByEntityIdsAndScopeAsync(tenantId, entityIds, scope), + entries -> Map.entry(scope, toKeyInfos(entries, true)), + dbCallbackExecutor); + } + return Futures.transform( + attributesService.findAllKeysByEntityIdsAndScopeAsync(tenantId, entityIds, scope), + keys -> Map.entry(scope, keys.stream().sorted().map(k -> new KeyInfo(k, null)).toList()), + dbCallbackExecutor); + } + + private ListenableFuture assembleResult( + Set entityTypes, + ListenableFuture> tsFuture, + List>>> attrFutures) { + var allAttrFuture = attrFutures.isEmpty() + ? immediateFuture(List.>>of()) + : Futures.allAsList(attrFutures); + + List> allFutures = new ArrayList<>(); + if (tsFuture != null) { + allFutures.add(tsFuture); + } + allFutures.add(allAttrFuture); + + var finalTsFuture = tsFuture; + return Futures.whenAllComplete(allFutures) + .call(() -> { + List tsKeys = finalTsFuture != null ? Futures.getDone(finalTsFuture) : null; + Map> attrMap = attrFutures.isEmpty() ? null : new TreeMap<>(); + if (attrMap != null) { + for (var entry : Futures.getDone(allAttrFuture)) { + attrMap.put(entry.getKey(), entry.getValue()); + } + } + return new AvailableEntityKeysV2(entityTypes, tsKeys, attrMap); + }, dbCallbackExecutor); + } + + private static List toKeyInfos(List entries, boolean includeSamples) { + return entries.stream() + .map(e -> new KeyInfo(e.getKey(), includeSamples ? toKeySample(e) : null)) + .sorted(Comparator.comparing(KeyInfo::key)) + .toList(); + } + + private static KeySample toKeySample(KvEntry entry) { + long ts = entry instanceof TsKvEntry tsKv ? tsKv.getTs() + : entry instanceof AttributeKvEntry attr ? attr.getLastUpdateTs() + : 0; + JsonNode value = entry.getDataType() == DataType.JSON + ? JacksonUtil.toJsonNode(entry.getJsonValue().get()) + : JacksonUtil.valueToTree(entry.getValue()); + return new KeySample(ts, value); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java b/application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java index a90cf684a1..354f8e9278 100644 --- a/application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java +++ b/application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java @@ -23,11 +23,14 @@ import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; import org.thingsboard.server.common.data.query.AvailableEntityKeys; +import org.thingsboard.server.common.data.query.AvailableEntityKeysV2; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.service.security.model.SecurityUser; +import java.util.Set; + public interface EntityQueryService { long countEntitiesByQuery(SecurityUser securityUser, EntityCountQuery query); @@ -41,4 +44,8 @@ public interface EntityQueryService { ListenableFuture getKeysByQuery(SecurityUser securityUser, TenantId tenantId, EntityDataQuery query, boolean isTimeseries, boolean isAttributes, AttributeScope scope); + ListenableFuture findAvailableEntityKeysByQuery(SecurityUser securityUser, EntityDataQuery query, + boolean includeTimeseries, boolean includeAttributes, + Set scopes, boolean includeSamples); + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index d0f7bfa81c..24fa3aafb3 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -147,15 +147,15 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa private void processMsgs(List> msgs, TbQueueConsumer> consumer, Object consumerKey, QueueConfig config) throws Exception { List> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); ConcurrentMap> pendingMap = orderedMsgList.stream().collect( - Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); + Collectors.toConcurrentMap(IdMsgPair::uuid, IdMsgPair::msg)); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); TbPackProcessingContext> ctx = new TbPackProcessingContext<>( processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); PendingMsgHolder pendingMsgHolder = new PendingMsgHolder<>(); Future packSubmitFuture = consumersExecutor.submit(() -> { orderedMsgList.forEach((element) -> { - UUID id = element.getUuid(); - TbProtoQueueMsg msg = element.getMsg(); + UUID id = element.uuid(); + TbProtoQueueMsg msg = element.msg(); log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); TbCallback callback = new TbPackCallback<>(id, ctx); try { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 6399e55e03..9bf2d027cb 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -260,15 +260,15 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> msgs, TbQueueConsumer> consumer, Object consumerKey, QueueConfig config) throws Exception { List> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); ConcurrentMap> pendingMap = orderedMsgList.stream().collect( - Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); + Collectors.toConcurrentMap(IdMsgPair::uuid, IdMsgPair::msg)); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); TbPackProcessingContext> ctx = new TbPackProcessingContext<>( processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); PendingMsgHolder pendingMsgHolder = new PendingMsgHolder<>(); Future packSubmitFuture = consumersExecutor.submit(() -> { orderedMsgList.forEach((element) -> { - UUID id = element.getUuid(); - TbProtoQueueMsg msg = element.getMsg(); + UUID id = element.uuid(); + TbProtoQueueMsg msg = element.msg(); log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); TbCallback callback = new TbPackCallback<>(id, ctx); try { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java index 53da1bb8a1..4a28f8a84a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java @@ -128,15 +128,15 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService> msgs, TbQueueConsumer> consumer, Object consumerKey, QueueConfig edgeQueueConfig) throws InterruptedException { List> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); ConcurrentMap> pendingMap = orderedMsgList.stream().collect( - Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); + Collectors.toConcurrentMap(IdMsgPair::uuid, IdMsgPair::msg)); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); TbPackProcessingContext> ctx = new TbPackProcessingContext<>( processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); PendingMsgHolder pendingMsgHolder = new PendingMsgHolder<>(); Future submitFuture = consumersExecutor.submit(() -> { orderedMsgList.forEach((element) -> { - UUID id = element.getUuid(); - TbProtoQueueMsg msg = element.getMsg(); + UUID id = element.uuid(); + TbProtoQueueMsg msg = element.msg(); TbCallback callback = new TbPackCallback<>(id, ctx); try { ToEdgeMsg toEdgeMsg = msg.getValue(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 63de93d6ad..72cac08846 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -70,6 +70,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractPartitionBasedCo private final TbRuleEngineConsumerContext ctx; private final QueueService queueService; private final TbRuleEngineDeviceRpcService tbDeviceRpcService; + private final TbMsgPackProcessingContextFactory packProcessingContextFactory; private final ConcurrentMap consumers = new ConcurrentHashMap<>(); @@ -85,11 +86,13 @@ public class DefaultTbRuleEngineConsumerService extends AbstractPartitionBasedCo PartitionService partitionService, ApplicationEventPublisher eventPublisher, JwtSettingsService jwtSettingsService, - CalculatedFieldCache calculatedFieldCache) { + CalculatedFieldCache calculatedFieldCache, + TbMsgPackProcessingContextFactory packProcessingContextFactory) { super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, tbResourceDataCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); this.ctx = ctx; this.tbDeviceRpcService = tbDeviceRpcService; this.queueService = queueService; + this.packProcessingContextFactory = packProcessingContextFactory; } @Override @@ -255,6 +258,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractPartitionBasedCo .consumerExecutor(consumersExecutor) .scheduler(scheduler) .taskExecutor(mgmtExecutor) + .packProcessingContextFactory(packProcessingContextFactory) .build(); consumers.put(queueKey, consumer); consumer.init(queue); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextFactory.java new file mode 100644 index 0000000000..08307a68ca --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextFactory.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import org.springframework.stereotype.Component; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; + +public interface TbMsgPackProcessingContextFactory { + + TbMsgPackProcessingContext create(String queueName, TbRuleEngineSubmitStrategy submitStrategy, boolean skipTimeouts); + + @Component + class DefaultTbMsgPackProcessingContextFactory implements TbMsgPackProcessingContextFactory { + + @Override + public TbMsgPackProcessingContext create(String queueName, TbRuleEngineSubmitStrategy submitStrategy, boolean skipTimeouts) { + return new TbMsgPackProcessingContext(queueName, submitStrategy, skipTimeouts); + } + + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index ec3b50b7ce..4b57101e69 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -134,13 +134,13 @@ public abstract class AbstractConsumerService> msgs, TbQueueConsumer> consumer) throws Exception { List> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); ConcurrentMap> pendingMap = orderedMsgList.stream().collect( - Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); + Collectors.toConcurrentMap(IdMsgPair::uuid, IdMsgPair::msg)); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); TbPackProcessingContext> ctx = new TbPackProcessingContext<>( processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); orderedMsgList.forEach(element -> { - UUID id = element.getUuid(); - TbProtoQueueMsg msg = element.getMsg(); + UUID id = element.uuid(); + TbProtoQueueMsg msg = element.msg(); log.trace("[{}] Creating notification callback for message: {}", id, msg.getValue()); TbCallback callback = new TbPackCallback<>(id, ctx); try { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java index 9f3344ca3e..291d9fc9b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java @@ -44,21 +44,21 @@ public abstract class AbstractTbRuleEngineSubmitStrategy implements TbRuleEngine @Override public ConcurrentMap> getPendingMap() { - return orderedMsgList.stream().collect(Collectors.toConcurrentMap(pair -> pair.uuid, pair -> pair.msg)); + return orderedMsgList.stream().collect(Collectors.toConcurrentMap(pair -> pair.uuid(), pair -> pair.msg())); } @Override public void update(ConcurrentMap> reprocessMap) { List> newOrderedMsgList = new ArrayList<>(reprocessMap.size()); for (IdMsgPair pair : orderedMsgList) { - if (reprocessMap.containsKey(pair.uuid)) { - if (StringUtils.isNotEmpty(pair.getMsg().getValue().getFailureMessage())) { - var toRuleEngineMsg = TransportProtos.ToRuleEngineMsg.newBuilder(pair.getMsg().getValue()) + if (reprocessMap.containsKey(pair.uuid())) { + if (StringUtils.isNotEmpty(pair.msg().getValue().getFailureMessage())) { + var toRuleEngineMsg = TransportProtos.ToRuleEngineMsg.newBuilder(pair.msg().getValue()) .clearFailureMessage() .clearRelationTypes() .build(); - var newMsg = new TbProtoQueueMsg<>(pair.getMsg().getKey(), toRuleEngineMsg, pair.getMsg().getHeaders()); - newOrderedMsgList.add(new IdMsgPair<>(pair.getUuid(), newMsg)); + var newMsg = new TbProtoQueueMsg<>(pair.msg().getKey(), toRuleEngineMsg, pair.msg().getHeaders()); + newOrderedMsgList.add(new IdMsgPair<>(pair.uuid(), newMsg)); } else { newOrderedMsgList.add(pair); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java index 32fcd65ca4..605c5fe977 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java @@ -73,7 +73,7 @@ public class BatchTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitS pendingPack.clear(); for (int i = startIdx; i < endIdx; i++) { IdMsgPair pair = orderedMsgList.get(i); - pendingPack.put(pair.uuid, pair.msg); + pendingPack.put(pair.uuid(), pair.msg()); } tmpPack = new LinkedHashMap<>(pendingPack); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java index 9f48c7bb70..5da8b85dc4 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java @@ -34,7 +34,7 @@ public class BurstTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitS if (log.isDebugEnabled()) { log.debug("[{}] submitting [{}] messages to rule engine", queueName, orderedMsgList.size()); } - orderedMsgList.forEach(pair -> msgConsumer.accept(pair.uuid, pair.msg)); + orderedMsgList.forEach(pair -> msgConsumer.accept(pair.uuid(), pair.msg())); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java index 81d2854dac..a0e440f7cd 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java @@ -15,19 +15,9 @@ */ package org.thingsboard.server.service.queue.processing; -import lombok.Getter; +import com.google.protobuf.GeneratedMessageV3; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import java.util.UUID; -public class IdMsgPair { - @Getter - final UUID uuid; - @Getter - final TbProtoQueueMsg msg; - - public IdMsgPair(UUID uuid, TbProtoQueueMsg msg) { - this.uuid = uuid; - this.msg = msg; - } -} +public record IdMsgPair(UUID uuid, TbProtoQueueMsg msg) {} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java index cfc3bb4a38..5acb629824 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java @@ -51,7 +51,7 @@ public abstract class SequentialByEntityIdTbRuleEngineSubmitStrategy extends Abs entityIdToListMap.forEach((entityId, queue) -> { IdMsgPair msg = queue.peek(); if (msg != null) { - msgConsumer.accept(msg.uuid, msg.msg); + msgConsumer.accept(msg.uuid(), msg.msg()); } }); } @@ -71,13 +71,13 @@ public abstract class SequentialByEntityIdTbRuleEngineSubmitStrategy extends Abs IdMsgPair next = null; synchronized (queue) { IdMsgPair expected = queue.peek(); - if (expected != null && expected.uuid.equals(id)) { + if (expected != null && expected.uuid().equals(id)) { queue.poll(); next = queue.peek(); } } if (next != null) { - msgConsumer.accept(next.uuid, next.msg); + msgConsumer.accept(next.uuid(), next.msg()); } } } @@ -87,9 +87,9 @@ public abstract class SequentialByEntityIdTbRuleEngineSubmitStrategy extends Abs msgToEntityIdMap.clear(); entityIdToListMap.clear(); for (IdMsgPair pair : orderedMsgList) { - EntityId entityId = getEntityId(pair.msg.getValue()); + EntityId entityId = getEntityId(pair.msg().getValue()); if (entityId != null) { - msgToEntityIdMap.put(pair.uuid, entityId); + msgToEntityIdMap.put(pair.uuid(), entityId); entityIdToListMap.computeIfAbsent(entityId, id -> new LinkedList<>()).add(pair); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java index 64be9bd378..be32f155dd 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java @@ -60,11 +60,11 @@ public class SequentialTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSu int idx = msgIdx.get(); if (idx < listSize) { IdMsgPair pair = orderedMsgList.get(idx); - expectedMsgId = pair.uuid; + expectedMsgId = pair.uuid(); if (log.isDebugEnabled()) { - log.debug("[{}] submitting [{}] message to rule engine", queueName, pair.msg); + log.debug("[{}] submitting [{}] message to rule engine", queueName, pair.msg()); } - msgConsumer.accept(pair.uuid, pair.msg); + msgConsumer.accept(pair.uuid(), pair.msg()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index 8ba98b817a..3733b49f1f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -42,6 +42,7 @@ import org.thingsboard.server.queue.common.consumer.TbQueueConsumerTask.Consumer import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.service.queue.TbMsgPackCallback; import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; +import org.thingsboard.server.service.queue.TbMsgPackProcessingContextFactory; import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; @@ -67,13 +68,15 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager { Integer partitionId = tpi != null ? tpi.getPartition().orElse(-1) : null; @@ -82,6 +85,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager submitMessage(packCtx, id, msg)); final boolean timeout = !packCtx.await(queue.getPackProcessingTimeout(), TimeUnit.MILLISECONDS); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index ef0d83d564..611eae26fe 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -197,6 +197,8 @@ usage: enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}" # Statistics reporting interval, set to send summarized data every 10 seconds by default interval: "${USAGE_STATS_REPORT_INTERVAL:60}" + # Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates + urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}" # Amount of statistic messages in pack pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}" check: @@ -1910,6 +1912,9 @@ queue: print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:60000}" # Max length of the error message that is printed by statistics max-error-message-length: "${TB_QUEUE_RULE_ENGINE_MAX_ERROR_MESSAGE_LENGTH:4096}" + prometheus-stats: + # Enable/disable Prometheus statistics for individual Rule Engine message processing (records time in ms for success/failure). + enabled: "${TB_QUEUE_RULE_ENGINE_PROMETHEUS_STATS_ENABLED:false}" # After a queue is deleted (or the profile's isolation option was disabled), Rule Engine will continue reading related topics during this period before deleting the actual topics topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:15}" # Size of the thread pool that handles such operations as partition changes, config updates, queue deletion @@ -2036,7 +2041,7 @@ management: web: exposure: # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). - include: '${METRICS_ENDPOINTS_EXPOSE:info}' + include: "${METRICS_ENDPOINTS_EXPOSE:info}" health: elasticsearch: # Enable the org.springframework.boot.actuate.elasticsearch.ElasticsearchRestClientHealthIndicator.doHealthCheck diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 71cbed6d10..e8fcd6a723 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -123,6 +123,8 @@ import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.job.Job; import org.thingsboard.server.common.data.job.JobType; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.notification.Notification; import org.thingsboard.server.common.data.notification.NotificationDeliveryMethod; import org.thingsboard.server.common.data.notification.NotificationType; @@ -177,6 +179,7 @@ import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -430,6 +433,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { tenantProfileService.deleteTenantProfiles(TenantId.SYS_TENANT_ID); jdbcTemplate.execute("TRUNCATE TABLE notification"); + jdbcTemplate.execute("TRUNCATE TABLE audit_log"); log.debug("Executed web test teardown"); } @@ -717,6 +721,10 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { return assetProfile; } + protected Device createDevice(String name) throws Exception { + return createDevice(name, "default", null, null); + } + protected Device createDevice(String name, String accessToken) throws Exception { return createDevice(name, "default", null, accessToken); } @@ -730,7 +738,11 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { deviceData.setTransportConfiguration(new DefaultDeviceTransportConfiguration()); deviceData.setConfiguration(new DefaultDeviceConfiguration()); device.setDeviceData(deviceData); - return doPost("/api/device?accessToken=" + accessToken, device, Device.class); + if (accessToken != null) { + return doPost("/api/device?accessToken=" + accessToken, device, Device.class); + } else { + return doPost("/api/device", device, Device.class); + } } protected Device assignDeviceToCustomer(DeviceId deviceId, CustomerId customerId) { @@ -1218,7 +1230,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { Awaitility.await("CF state for entity actor ready to refresh dynamic arguments").atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> { CalculatedFieldState calculatedFieldState = statesMap.get(cfId); boolean isReady = calculatedFieldState != null && ((GeofencingCalculatedFieldState) calculatedFieldState).getLastScheduledRefreshTs() < - System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(scheduledUpdateInterval); + System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(scheduledUpdateInterval); log.warn("entityId {}, cfId {}, state ready to refresh == {}", entityId, cfId, isReady); return isReady; }); @@ -1410,7 +1422,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { protected List findJobs(List types, List entities) throws Exception { return doGetTypedWithPageLink("/api/jobs?types=" + types.stream().map(Enum::name).collect(Collectors.joining(",")) + - "&entities=" + entities.stream().map(UUID::toString).collect(Collectors.joining(",")) + "&", + "&entities=" + entities.stream().map(UUID::toString).collect(Collectors.joining(",")) + "&", new TypeReference>() {}, new PageLink(100, 0, null, new SortOrder("createdTime", SortOrder.Direction.DESC))).getData(); } @@ -1424,12 +1436,37 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { protected void postTelemetry(EntityId entityId, String payload) throws Exception { doPostAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + - "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(payload), 30_000L).andExpect(status().isOk()); + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(payload), 30_000L).andExpect(status().isOk()); + } + + protected void postTelemetry(EntityId entityId, TsKvEntry entry) throws Exception { + var values = JacksonUtil.newObjectNode(); + JacksonUtil.addKvEntry(values, entry); + + var payload = JacksonUtil.newObjectNode() + .put("ts", entry.getTs()) + .set("values", values); + + var url = "/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/timeseries/any"; + doPostAsync(url, payload, 30_000L).andExpect(status().isOk()); } protected void postAttributes(EntityId entityId, AttributeScope scope, String payload) throws Exception { doPostAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + - "/attributes/" + scope, JacksonUtil.toJsonNode(payload), 30_000L).andExpect(status().isOk()); + "/attributes/" + scope, JacksonUtil.toJsonNode(payload), 30_000L).andExpect(status().isOk()); + } + + protected void postAttributes(EntityId entityId, AttributeScope scope, KvEntry... attributes) throws Exception { + postAttributes(entityId, scope, Arrays.asList(attributes)); + } + + protected void postAttributes(EntityId entityId, AttributeScope scope, Collection attributes) throws Exception { + var url = "/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/attributes/" + scope; + var payload = JacksonUtil.newObjectNode(); + for (KvEntry entry : attributes) { + JacksonUtil.addKvEntry(payload, entry); + } + doPostAsync(url, payload, 30_000L).andExpect(status().isOk()); } protected CalculatedField saveCalculatedField(CalculatedField calculatedField) { @@ -1438,7 +1475,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { protected PageData getEntityCalculatedFields(EntityId entityId, CalculatedFieldType type, PageLink pageLink) throws Exception { return doGetTypedWithPageLink("/api/" + entityId.getEntityType() + "/" + entityId.getId() + "/calculatedFields" + - (type != null ? "?type=" + type.name() + "&" : "?"), new TypeReference<>() {}, pageLink); + (type != null ? "?type=" + type.name() + "&" : "?"), new TypeReference<>() {}, pageLink); } protected PageData getCalculatedFieldNames(CalculatedFieldType type, PageLink pageLink) throws Exception { @@ -1451,11 +1488,11 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { List entities, List names) throws Exception { return doGetTypedWithPageLink("/api/calculatedFields?" + - (type != null ? "types=" + type + "&" : "") + - (entityType != null ? "entityType=" + entityType + "&" : "") + - (entities != null ? "entities=" + String.join(",", - entities.stream().map(UUID::toString).toList()) + "&" : "") + - (names != null ? names.stream().map(name -> "name=" + name + "&").collect(Collectors.joining("")) : ""), + (type != null ? "types=" + type + "&" : "") + + (entityType != null ? "entityType=" + entityType + "&" : "") + + (entities != null ? "entities=" + String.join(",", + entities.stream().map(UUID::toString).toList()) + "&" : "") + + (names != null ? names.stream().map(name -> "name=" + name + "&").collect(Collectors.joining("")) : ""), new TypeReference>() {}, new PageLink(10)).getData(); } diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java index 7332ee26c6..07b68210bb 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java @@ -1732,11 +1732,4 @@ public class DeviceControllerTest extends AbstractControllerTest { assertThat(fifthDevice.getName()).isEqualTo("My unique device_2"); } - private Device createDevice(String name) { - Device device = new Device(); - device.setName(name); - device.setType("default"); - return doPost("/api/device", device, Device.class); - } - } diff --git a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java index f15ab09ab2..6c1a63194c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java @@ -23,6 +23,7 @@ import org.springframework.test.context.TestPropertySource; import org.thingsboard.server.common.data.edqs.EdqsState; import org.thingsboard.server.common.data.edqs.EdqsState.EdqsApiMode; import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; +import org.awaitility.core.ThrowingRunnable; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; @@ -86,6 +87,11 @@ public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest { result -> result == expectedResult); } + @Override + protected void verifyAvailableKeysByQueryV2(ThrowingRunnable assertion) { + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(assertion); + } + @Test public void testEdqsState() throws Exception { loginSysAdmin(); diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index e1173ef9a7..835f00985b 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -17,7 +17,11 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.BooleanNode; +import com.fasterxml.jackson.databind.node.DoubleNode; +import com.fasterxml.jackson.databind.node.IntNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import org.awaitility.core.ThrowingRunnable; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -43,12 +47,21 @@ import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataPageLink; import org.thingsboard.server.common.data.query.AlarmDataQuery; import org.thingsboard.server.common.data.query.AliasEntityId; +import org.thingsboard.server.common.data.query.AvailableEntityKeysV2; +import org.thingsboard.server.common.data.query.AvailableEntityKeysV2.KeyInfo; import org.thingsboard.server.common.data.query.DeviceTypeFilter; import org.thingsboard.server.common.data.query.DynamicValue; import org.thingsboard.server.common.data.query.DynamicValueSourceType; @@ -84,6 +97,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import java.util.stream.Collectors; @@ -1329,7 +1343,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { //assign dashboard doPost("/api/customer/" + savedCustomer.getId().getId().toString() - + "/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class); + + "/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class); // check entity data query by customer User customerUser = new User(); @@ -1454,6 +1468,10 @@ public class EntityQueryControllerTest extends AbstractControllerTest { return result; } + protected void verifyAvailableKeysByQueryV2(ThrowingRunnable assertion) throws Throwable { + assertion.run(); + } + private KeyFilter getEntityFieldEqualFilter(String keyName, String value) { return getEntityFieldKeyFilter(keyName, value, StringFilterPredicate.StringOperation.EQUAL); } @@ -1494,4 +1512,221 @@ public class EntityQueryControllerTest extends AbstractControllerTest { return nameFilter; } + // --- findAvailableEntityKeysV2 tests --- + + @Test + public void testFindAvailableKeysByQueryV2() throws Throwable { + // GIVEN — two devices matched by query; a third device should not be matched + var device1 = createDevice("Test device 1"); + var device2 = createDevice("Test device 2"); + var unmatchedDevice = createDevice("Unmatched device"); + + // unmatched device has unique keys that must NOT appear in the result + postTelemetry(unmatchedDevice.getId(), new BasicTsKvEntry(9000, new DoubleDataEntry("unmatchedTs", 999.0))); + postAttributes(unmatchedDevice.getId(), AttributeScope.SHARED_SCOPE, new StringDataEntry("unmatchedAttr", "nope")); + + // device1: timeseries1 (Double) with two data points, and timeseries2 older data point + postTelemetry(device1.getId(), new BasicTsKvEntry(1000, new DoubleDataEntry("timeseries1", 10.0))); + postTelemetry(device1.getId(), new BasicTsKvEntry(2000, new DoubleDataEntry("timeseries1", 20.5))); + postTelemetry(device1.getId(), new BasicTsKvEntry(1000, new LongDataEntry("timeseries2", 100L))); + + // device2: timeseries2 (Long) with a newer data point, and timeseries3 only on this device + postTelemetry(device2.getId(), new BasicTsKvEntry(3000, new LongDataEntry("timeseries2", 300L))); + postTelemetry(device2.getId(), new BasicTsKvEntry(5000, new DoubleDataEntry("timeseries3", 99.9))); + + // device1: SHARED_SCOPE attributes + postAttributes(device1.getId(), AttributeScope.SHARED_SCOPE, + new BooleanDataEntry("sharedAttribute1", true), new DoubleDataEntry("sharedAttribute2", 3.14)); + + // device2: CLIENT_SCOPE attributes (saved via service to bypass API restriction) + attributesService.save(tenantId, device2.getId(), AttributeScope.CLIENT_SCOPE, List.of( + new BaseAttributeKvEntry(new JsonDataEntry("clientAttribute1", "{\"key\":\"val\"}"), System.currentTimeMillis()), + new BaseAttributeKvEntry(new BooleanDataEntry("clientAttribute2", false), System.currentTimeMillis()) + )).get(); + + // device1 also has SERVER_SCOPE attributes (should be omitted by scope filter) + postAttributes(device1.getId(), AttributeScope.SERVER_SCOPE, + new StringDataEntry("serverAttribute1", "sv1"), new LongDataEntry("serverAttribute2", 42L)); + + // WHEN — query matches both devices; request timeseries + only SHARED and CLIENT attribute scopes + DeviceTypeFilter filter = new DeviceTypeFilter(); + filter.setDeviceTypes(List.of("default")); + filter.setDeviceNameFilter("Test device"); + EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, null); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, List.of(), null, null); + + // THEN + verifyAvailableKeysByQueryV2(() -> { + AvailableEntityKeysV2 result = findAvailableEntityKeysByQueryV2(query, + true, true, List.of(AttributeScope.SHARED_SCOPE, AttributeScope.CLIENT_SCOPE), true); + + assertThat(result.entityTypes()).containsExactly(EntityType.DEVICE); + + // timeseries: keys collected from both devices, samples contain the freshest data points + assertThat(result.timeseries()).extracting(KeyInfo::key) + .containsExactly("timeseries1", "timeseries2", "timeseries3"); + assertThat(result.timeseries()).allSatisfy(ki -> assertThat(ki.sample()).isNotNull()); + assertKeySample(result.timeseries(), "timeseries1", new DoubleNode(20.5), 2000); // from device1 + assertKeySample(result.timeseries(), "timeseries2", new IntNode(300), 3000); // from device2 (newer) + assertKeySample(result.timeseries(), "timeseries3", new DoubleNode(99.9), 5000); // only on device2 + + // SERVER_SCOPE must be fully omitted from the response + assertThat(result.attributes()).containsOnlyKeys(AttributeScope.SHARED_SCOPE, AttributeScope.CLIENT_SCOPE); + + // SHARED_SCOPE: from device1 (alphabetical order) + assertThat(result.attributes().get(AttributeScope.SHARED_SCOPE)) + .extracting(KeyInfo::key).containsExactly("sharedAttribute1", "sharedAttribute2"); + assertKeySample(result.attributes().get(AttributeScope.SHARED_SCOPE), "sharedAttribute1", BooleanNode.TRUE); + assertKeySample(result.attributes().get(AttributeScope.SHARED_SCOPE), "sharedAttribute2", new DoubleNode(3.14)); + + // CLIENT_SCOPE: from device2 (alphabetical order) + assertThat(result.attributes().get(AttributeScope.CLIENT_SCOPE)) + .extracting(KeyInfo::key).containsExactly("clientAttribute1", "clientAttribute2"); + assertKeySample(result.attributes().get(AttributeScope.CLIENT_SCOPE), "clientAttribute1", JacksonUtil.toJsonNode("{\"key\":\"val\"}")); + assertKeySample(result.attributes().get(AttributeScope.CLIENT_SCOPE), "clientAttribute2", BooleanNode.FALSE); + }); + } + + @Test + public void testFindAvailableKeysByQueryV2_withoutSamples() throws Throwable { + // GIVEN + var device = createDevice("Test device"); + postTelemetry(device.getId(), new BasicTsKvEntry(System.currentTimeMillis(), new DoubleDataEntry("temperature", 10.0))); + postAttributes(device.getId(), AttributeScope.SERVER_SCOPE, new StringDataEntry("firmware", "v1.0")); + + // THEN + verifyAvailableKeysByQueryV2(() -> { + AvailableEntityKeysV2 result = findAvailableEntityKeysByQueryV2( + buildDeviceQuery("Test device"), true, true, null, false); + + assertThat(result.timeseries()).allSatisfy(ki -> assertThat(ki.sample()).isNull()); + assertThat(result.attributes().get(AttributeScope.SERVER_SCOPE)) + .allSatisfy(ki -> assertThat(ki.sample()).isNull()); + }); + } + + @Test + public void testFindAvailableKeysByQueryV2_timeseriesOnly() throws Throwable { + // GIVEN + var device = createDevice("Test device"); + postTelemetry(device.getId(), new BasicTsKvEntry(System.currentTimeMillis(), new DoubleDataEntry("temperature", 10.0))); + postAttributes(device.getId(), AttributeScope.SERVER_SCOPE, new StringDataEntry("firmware", "v1.0")); + + // THEN + verifyAvailableKeysByQueryV2(() -> { + AvailableEntityKeysV2 result = findAvailableEntityKeysByQueryV2( + buildDeviceQuery("Test device"), true, false, null, false); + + assertThat(result.timeseries()).extracting(KeyInfo::key).contains("temperature"); + assertThat(result.attributes()).isNull(); + }); + } + + @Test + public void testFindAvailableKeysByQueryV2_attributesOnly() throws Throwable { + // GIVEN + var device = createDevice("Test device"); + postTelemetry(device.getId(), new BasicTsKvEntry(System.currentTimeMillis(), new DoubleDataEntry("temperature", 10.0))); + postAttributes(device.getId(), AttributeScope.SERVER_SCOPE, new StringDataEntry("firmware", "v1.0")); + + // THEN + verifyAvailableKeysByQueryV2(() -> { + AvailableEntityKeysV2 result = findAvailableEntityKeysByQueryV2( + buildDeviceQuery("Test device"), false, true, null, false); + + assertThat(result.timeseries()).isNull(); + assertThat(result.attributes().get(AttributeScope.SERVER_SCOPE)) + .extracting(KeyInfo::key).contains("firmware"); + }); + } + + @Test + public void testFindAvailableKeysByQueryV2_noMatchingEntities() throws Throwable { + // THEN + verifyAvailableKeysByQueryV2(() -> { + AvailableEntityKeysV2 result = findAvailableEntityKeysByQueryV2( + buildDeviceQuery("NonExistentDevice_" + UUID.randomUUID()), true, true, null, true); + + assertThat(result.entityTypes()).isEmpty(); + assertThat(result.timeseries()).isEmpty(); + assertThat(result.attributes()).isEmpty(); + }); + } + + @Test + public void testFindAvailableKeysByQueryV2_assetUsesServerScopeOnly() throws Throwable { + // GIVEN + var asset = new Asset(); + asset.setName("Test asset"); + asset.setType("default"); + asset = doPost("/api/asset", asset, Asset.class); + postAttributes(asset.getId(), AttributeScope.SERVER_SCOPE, new StringDataEntry("location", "warehouse")); + + // WHEN + var filter = new SingleEntityFilter(); + filter.setSingleEntity(AliasEntityId.fromEntityId(asset.getId())); + var query = new EntityDataQuery(filter, new EntityDataPageLink(1, 0, null, null), Collections.emptyList(), null, null); + + // THEN + verifyAvailableKeysByQueryV2(() -> { + AvailableEntityKeysV2 result = findAvailableEntityKeysByQueryV2(query, false, true, null, false); + + assertThat(result.entityTypes()).containsExactly(EntityType.ASSET); + assertThat(result.attributes()).containsOnlyKeys(AttributeScope.SERVER_SCOPE); + assertThat(result.attributes().get(AttributeScope.SERVER_SCOPE)) + .extracting(KeyInfo::key).containsExactly("location"); + }); + } + + @Test + public void testFindAvailableKeysByQueryV2_rejectsWhenNoKeyTypeRequested() throws Exception { + // WHEN / THEN + EntityDataQuery query = buildDeviceQuery("NonExistent"); + + doPostAsync("/api/v2/entitiesQuery/find/keys?includeTimeseries=false&includeAttributes=false", + query, 30_000L).andExpect(status().isBadRequest()); + } + + protected AvailableEntityKeysV2 findAvailableEntityKeysByQueryV2(EntityDataQuery query, + boolean includeTimeseries, boolean includeAttributes, + List scopes, boolean includeSamples) throws Exception { + StringBuilder url = new StringBuilder("/api/v2/entitiesQuery/find/keys?") + .append("includeTimeseries=").append(includeTimeseries) + .append("&includeAttributes=").append(includeAttributes) + .append("&includeSamples=").append(includeSamples); + if (scopes != null) { + for (AttributeScope scope : scopes) { + url.append("&scopes=").append(scope); + } + } + return doPostAsyncWithTypedResponse(url.toString(), query, + new TypeReference<>() {}, status().isOk()); + } + + private static void assertKeySample(List keys, String expectedKey, JsonNode expectedValue, long expectedTs) { + KeyInfo keyInfo = findKeyInfo(keys, expectedKey); + assertThat(keyInfo.sample()).isNotNull(); + assertThat(keyInfo.sample().value()).isEqualTo(expectedValue); + assertThat(keyInfo.sample().ts()).isEqualTo(expectedTs); + } + + private static void assertKeySample(List keys, String expectedKey, JsonNode expectedValue) { + KeyInfo keyInfo = findKeyInfo(keys, expectedKey); + assertThat(keyInfo.sample()).isNotNull(); + assertThat(keyInfo.sample().value()).isEqualTo(expectedValue); + assertThat(keyInfo.sample().ts()).isGreaterThan(0); + } + + private static KeyInfo findKeyInfo(List keys, String key) { + return keys.stream() + .filter(ki -> ki.key().equals(key)).findFirst().orElseThrow(); + } + + private static EntityDataQuery buildDeviceQuery(String deviceName) { + var filter = new DeviceTypeFilter(); + filter.setDeviceTypes(Collections.singletonList("default")); + filter.setDeviceNameFilter(deviceName); + return new EntityDataQuery(filter, new EntityDataPageLink(1, 0, null, null), Collections.emptyList(), null, null); + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java index b0c76a16a8..b4ce7179d2 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java @@ -633,13 +633,6 @@ public class EntityRelationControllerTest extends AbstractControllerTest { deleteDifferentTenant(); } - private Device createDevice(String name) { - var device = new Device(); - device.setName(name); - device.setType("default"); - return doPost("/api/device", device, Device.class); - } - private ResultActions getRelation(EntityRelation relation) throws Exception { return doGet("/api/relation?" + "fromId=" + relation.getFrom().getId() + diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java index 2454dae974..a2e622c8b5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java @@ -407,20 +407,15 @@ public class TenantProfileControllerTest extends AbstractControllerTest { testBroadcastEntityStateChangeEventNeverTenantProfile(); } - private void awaitAuditLog(String awaitMessage, TenantProfileId tenantProfileId, ActionType expectedAction) throws Exception { + private void awaitAuditLog(String awaitMessage, TenantProfileId tenantProfileId, ActionType expectedAction) { Awaitility.await(awaitMessage) .atMost(TIMEOUT, TimeUnit.SECONDS) - .until(() -> - doGetTypedWithTimePageLink( - "/api/audit/logs/entity/TENANT_PROFILE/" + tenantProfileId.getId() + "?", - new TypeReference>() { - }, - new TimePageLink(5)) - .getData() - .stream() - .anyMatch(log -> log.getActionType() == expectedAction) + .until(() -> doGetTypedWithTimePageLink( + "/api/audit/logs?", + new TypeReference>() {}, + new TimePageLink(100)).getData().stream() + .anyMatch(log -> log.getEntityId().equals(tenantProfileId) && log.getActionType() == expectedAction) ); - } private TenantProfile createTenantProfile(String name) { diff --git a/application/src/test/java/org/thingsboard/server/service/apiusage/ApiUsageTest.java b/application/src/test/java/org/thingsboard/server/service/apiusage/ApiUsageTest.java index da73600671..4cd41c314e 100644 --- a/application/src/test/java/org/thingsboard/server/service/apiusage/ApiUsageTest.java +++ b/application/src/test/java/org/thingsboard/server/service/apiusage/ApiUsageTest.java @@ -19,6 +19,8 @@ import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.TestPropertySource; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.ApiUsageStateValue; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.SaveDeviceWithCredentialsRequest; @@ -30,6 +32,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; +import org.thingsboard.server.common.stats.TbApiUsageReportClient; import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.controller.TbUrlConstants; import org.thingsboard.server.dao.service.DaoSqlTest; @@ -46,6 +49,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @TestPropertySource(properties = { "usage.stats.report.enabled=true", "usage.stats.report.interval=2", + "usage.stats.report.urgent_interval=1", "usage.stats.gauge_report_interval=1", }) public class ApiUsageTest extends AbstractControllerTest { @@ -54,9 +58,12 @@ public class ApiUsageTest extends AbstractControllerTest { private User tenantAdmin; private static final int MAX_DP_ENABLE_VALUE = 12; + private static final int MAX_SMS_ENABLE_VALUE = 10; private static final double WARN_THRESHOLD_VALUE = 0.5; @Autowired private ApiUsageStateService apiUsageStateService; + @Autowired + private TbApiUsageReportClient apiUsageReportClient; @Before public void beforeTest() throws Exception { @@ -82,7 +89,7 @@ public class ApiUsageTest extends AbstractControllerTest { } @Test - public void testTelemetryApiCall() throws Exception { + public void testDbStorageApiUsage() throws Exception { Device device = createDevice(); assertNotNull(device); String telemetryPayload = "{\"temperature\":25, \"humidity\":60}"; @@ -94,7 +101,8 @@ public class ApiUsageTest extends AbstractControllerTest { doPostAsync(url, telemetryPayload, String.class, status().isOk()); } - await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> assertEquals(ApiUsageStateValue.WARNING, apiUsageStateService.findTenantApiUsageState(tenantId).getDbStorageState())); + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> + assertEquals(ApiUsageStateValue.WARNING, getUsageState().getDbStorageState())); long VALUE_DISABLE = (long) (MAX_DP_ENABLE_VALUE - (MAX_DP_ENABLE_VALUE * WARN_THRESHOLD_VALUE)) / 2; @@ -104,10 +112,35 @@ public class ApiUsageTest extends AbstractControllerTest { await().atMost(TIMEOUT, TimeUnit.SECONDS) .untilAsserted(() -> { - assertEquals(ApiUsageStateValue.DISABLED, apiUsageStateService.findTenantApiUsageState(tenantId).getDbStorageState()); + assertEquals(ApiUsageStateValue.DISABLED, getUsageState().getDbStorageState()); }); } + @Test + public void testSmsApiUsage() { + long smsWarnThreshold = (long) (MAX_SMS_ENABLE_VALUE * WARN_THRESHOLD_VALUE); + + for (int i = 0; i < smsWarnThreshold; i++) { + apiUsageReportClient.report(tenantId, null, ApiUsageRecordKey.SMS_EXEC_COUNT); + } + + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> + assertEquals(ApiUsageStateValue.WARNING, getUsageState().getSmsExecState())); + + long smsDisableCount = MAX_SMS_ENABLE_VALUE - smsWarnThreshold; + + for (int i = 0; i < smsDisableCount; i++) { + apiUsageReportClient.report(tenantId, null, ApiUsageRecordKey.SMS_EXEC_COUNT); + } + + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> + assertEquals(ApiUsageStateValue.DISABLED, getUsageState().getSmsExecState())); + } + + private ApiUsageState getUsageState() { + return apiUsageStateService.findTenantApiUsageState(tenantId); + } + private TenantProfile createTenantProfile() { TenantProfile tenantProfile = new TenantProfile(); tenantProfile.setName("Tenant Profile"); @@ -116,6 +149,8 @@ public class ApiUsageTest extends AbstractControllerTest { TenantProfileData tenantProfileData = new TenantProfileData(); DefaultTenantProfileConfiguration config = DefaultTenantProfileConfiguration.builder() .maxDPStorageDays(MAX_DP_ENABLE_VALUE) + .maxSms(MAX_SMS_ENABLE_VALUE) + .smsEnabled(true) .warnThreshold(WARN_THRESHOLD_VALUE) .build(); diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesAggregationCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesAggregationCalculatedFieldStateTest.java index cbde708ff8..02b063a499 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesAggregationCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesAggregationCalculatedFieldStateTest.java @@ -234,6 +234,8 @@ public class RelatedEntitiesAggregationCalculatedFieldStateTest { config.setUseLatestTs(true); + config.setScheduledUpdateInterval(10); + calculatedField.setConfiguration(config); calculatedField.setVersion(1L); return calculatedField; diff --git a/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java b/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java index 3ea827d330..8d349bb858 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java @@ -15,14 +15,17 @@ */ package org.thingsboard.server.service.queue; -import lombok.extern.slf4j.Slf4j; -import org.junit.After; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.junit.MockitoJUnitRunner; +import com.google.common.util.concurrent.MoreExecutors; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; @@ -35,30 +38,241 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertTrue; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.then; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -@Slf4j -@RunWith(MockitoJUnitRunner.class) -public class TbMsgPackProcessingContextTest { +@ExtendWith(MockitoExtension.class) +class TbMsgPackProcessingContextTest { + + TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); + + @Mock + TbRuleEngineSubmitStrategy submitStrategy; + @Mock + TbProtoQueueMsg mockMsg; + + ConcurrentMap> pendingMap; - public static final int TIMEOUT = 10; ExecutorService executorService; - @After - public void tearDown() { + @BeforeEach + void setup() { + pendingMap = new ConcurrentHashMap<>(); + lenient().when(submitStrategy.getPendingMap()).thenReturn(pendingMap); + } + + @AfterEach + void tearDown() { if (executorService != null) { - executorService.shutdownNow(); + MoreExecutors.shutdownAndAwaitTermination(executorService, 5, TimeUnit.SECONDS); } } @Test - public void testHighConcurrencyCase() throws InterruptedException { - //log.warn("preparing the test..."); + void testAwait_shouldReturnTrue_whenOnSuccessIsCalledBeforeTimeout() throws InterruptedException { + // GIVEN - a context with one pending message + executorService = Executors.newSingleThreadExecutor(); + + UUID msgId = UUID.randomUUID(); + pendingMap.put(msgId, mockMsg); + var context = new TbMsgPackProcessingContext("test-queue", submitStrategy, false); + + // WHEN - onSuccess() is called in another thread before timeout + executorService.submit(() -> { + try { + Thread.sleep(100); + context.onSuccess(msgId); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + + // THEN - await() should return true (successful completion) + boolean result = context.await(5000, TimeUnit.MILLISECONDS); + assertThat(result).as("await() should return true when latch is counted down before timeout").isTrue(); + + // Verify the message was moved to success map + assertThat(context.getSuccessMap()).containsKey(msgId); + assertThat(context.getPendingMap()).isEmpty(); + assertThat(context.getExceptionsMap()).isEmpty(); + + + // Verify submit strategy was notified about successful message processing + then(submitStrategy).should().onSuccess(msgId); + } + + @Test + void testAwait_shouldReturnTrue_whenOnFailureIsCalledBeforeTimeout() throws InterruptedException { + // GIVEN - a context with one pending message + executorService = Executors.newSingleThreadExecutor(); + + UUID msgId = UUID.randomUUID(); + pendingMap.put(msgId, mockMsg); + var context = new TbMsgPackProcessingContext("test-queue", submitStrategy, false); + + var exception = new RuleEngineException("Test exception"); + + // WHEN - onFailure() is called in another thread before timeout + executorService.submit(() -> { + try { + Thread.sleep(100); + context.onFailure(tenantId, msgId, exception); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + + // THEN - await() should return true (successful completion, even if message processing failed) + boolean result = context.await(5000, TimeUnit.MILLISECONDS); + assertThat(result).as("await() should return true when latch is counted down before timeout").isTrue(); + + // Verify the exception was added to exceptions map + assertThat(context.getSuccessMap()).isEmpty(); + assertThat(context.getPendingMap()).isEmpty(); + assertThat(context.getExceptionsMap()).containsEntry(tenantId, exception); + } + + @Test + void testAwait_shouldReturnFalse_whenTimeoutOccurs() throws InterruptedException { + // GIVEN - a context with one pending message and no processing + UUID msgId = UUID.randomUUID(); + pendingMap.put(msgId, mockMsg); + var context = new TbMsgPackProcessingContext("test-queue", submitStrategy, false); + + // WHEN - await() is called with short timeout and no message processing happens + long startTime = System.nanoTime(); + boolean result = context.await(100, TimeUnit.MILLISECONDS); + long elapsedTime = System.nanoTime() - startTime; + + // THEN - await() should return false (timeout occurred) + assertThat(result).as("await() should return false when timeout occurs").isFalse(); + assertThat(elapsedTime).as("await() should wait for at least the timeout duration").isGreaterThanOrEqualTo(100L); + + // Message should still be in pending map + assertThat(context.getSuccessMap()).isEmpty(); + assertThat(context.getPendingMap()).containsKey(msgId); + assertThat(context.getExceptionsMap()).isEmpty(); + } + + @Test + void testAwait_shouldHandleMultiplePendingMessages() throws InterruptedException { + // GIVEN - a context with multiple pending messages + executorService = Executors.newSingleThreadExecutor(); + + UUID msgId1 = UUID.randomUUID(); + UUID msgId2 = UUID.randomUUID(); + UUID msgId3 = UUID.randomUUID(); + + pendingMap.put(msgId1, mockMsg); + pendingMap.put(msgId2, mockMsg); + pendingMap.put(msgId3, mockMsg); + + var context = new TbMsgPackProcessingContext("test-queue", submitStrategy, false); + + // WHEN - messages are processed one by one + executorService.submit(() -> { + try { + Thread.sleep(50); + context.onSuccess(msgId1); + Thread.sleep(50); + context.onSuccess(msgId2); + Thread.sleep(50); + context.onSuccess(msgId3); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + + // THEN - await() should return true only after all messages are processed + boolean result = context.await(5000, TimeUnit.MILLISECONDS); + assertThat(result).as("await() should return true after all messages are processed").isTrue(); + + // All messages should be in success map + assertThat(context.getSuccessMap()).containsKeys(msgId1, msgId2, msgId3); + assertThat(context.getPendingMap()).isEmpty(); + assertThat(context.getExceptionsMap()).isEmpty(); + } + + @Test + void testAwait_shouldNotCountDownPrematurely_withMultipleMessages() throws InterruptedException { + // GIVEN - a context with multiple pending messages + executorService = Executors.newSingleThreadExecutor(); + + UUID msgId1 = UUID.randomUUID(); + UUID msgId2 = UUID.randomUUID(); + + pendingMap.put(msgId1, mockMsg); + pendingMap.put(msgId2, mockMsg); + + var context = new TbMsgPackProcessingContext("test-queue", submitStrategy, false); + + // WHEN - only one message is processed + executorService.submit(() -> { + try { + Thread.sleep(100); + context.onSuccess(msgId1); + // msgId2 still in processing + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + + // THEN: await should timeout because not all messages were processed + boolean result = context.await(2000, TimeUnit.MILLISECONDS); + assertThat(result).as("await() should timeout when not all messages are processed").isFalse(); + + // One message in success, one still pending + assertThat(context.getSuccessMap()).containsOnlyKeys(msgId1); + assertThat(context.getPendingMap()).containsOnlyKeys(msgId2); + assertThat(context.getExceptionsMap()).isEmpty(); + } + + @Test + void testAwait_shouldHandleMixedSuccessAndFailure() throws InterruptedException { + // GIVEN - multiple messages + executorService = Executors.newSingleThreadExecutor(); + + UUID msgId1 = UUID.randomUUID(); + UUID msgId2 = UUID.randomUUID(); + + pendingMap.put(msgId1, mockMsg); + pendingMap.put(msgId2, mockMsg); + + var context = new TbMsgPackProcessingContext("test-queue", submitStrategy, false); + + var exception = new RuleEngineException("Test exception"); + + // WHEN - one succeeds, one fails + executorService.submit(() -> { + try { + Thread.sleep(50); + context.onSuccess(msgId1); + Thread.sleep(50); + context.onFailure(tenantId, msgId2, exception); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + + // THEN - await() should complete successfully + boolean result = context.await(5000, TimeUnit.MILLISECONDS); + assertThat(result).as("await() should return true when all messages are processed").isTrue(); + + assertThat(context.getSuccessMap()).containsOnlyKeys(msgId1); + assertThat(context.getPendingMap()).isEmpty(); + assertThat(context.getExceptionsMap()).containsEntry(tenantId, exception); + } + + @Test + void testHighConcurrencyCase() throws InterruptedException { int msgCount = 1000; int parallelCount = 5; executorService = Executors.newFixedThreadPool(parallelCount, ThingsBoardThreadFactory.forName(getClass().getSimpleName() + "-test-scope")); @@ -76,28 +290,24 @@ public class TbMsgPackProcessingContextTest { final CountDownLatch startLatch = new CountDownLatch(1); final CountDownLatch finishLatch = new CountDownLatch(parallelCount); for (int i = 0; i < parallelCount; i++) { - //final String taskName = "" + uuid + " " + i; executorService.submit(() -> { - //log.warn("ready {}", taskName); readyLatch.countDown(); try { startLatch.await(); } catch (InterruptedException e) { - Assert.fail("failed to await"); + fail("failed to await"); } - //log.warn("go {}", taskName); - context.onSuccess(uuid); - finishLatch.countDown(); }); } - assertTrue(readyLatch.await(TIMEOUT, TimeUnit.SECONDS)); + assertTrue(readyLatch.await(10, TimeUnit.SECONDS)); Thread.yield(); startLatch.countDown(); //run all-at-once submitted tasks - assertTrue(finishLatch.await(TIMEOUT, TimeUnit.SECONDS)); + assertTrue(finishLatch.await(10, TimeUnit.SECONDS)); } - assertTrue(context.await(TIMEOUT, TimeUnit.SECONDS)); + assertTrue(context.await(10, TimeUnit.SECONDS)); verify(strategyMock, times(msgCount)).onSuccess(any(UUID.class)); } + } diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/RuleEngineConsumerLoopTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/RuleEngineConsumerLoopTest.java new file mode 100644 index 0000000000..20ea6fa881 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/RuleEngineConsumerLoopTest.java @@ -0,0 +1,281 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.ruleengine; + +import com.google.common.util.concurrent.MoreExecutors; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.data.queue.ProcessingStrategy; +import org.thingsboard.server.common.data.queue.ProcessingStrategyType; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.data.queue.SubmitStrategy; +import org.thingsboard.server.common.data.queue.SubmitStrategyType; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.memory.DefaultInMemoryStorage; +import org.thingsboard.server.queue.memory.InMemoryStorage; +import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; +import org.thingsboard.server.service.queue.TbMsgPackProcessingContextFactory; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; +import org.thingsboard.server.service.stats.RuleEngineStatisticsService; + +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class RuleEngineConsumerLoopTest { + + TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + + InMemoryStorage storage; + + @Mock + ActorSystemContext actorContext; + @Mock + StatsFactory statsFactory; + @Mock + TbRuleEngineQueueFactory queueFactory; + @Mock + RuleEngineStatisticsService statisticsService; + @Mock + TbServiceInfoProvider serviceInfoProvider; + @Mock + PartitionService partitionService; + @Mock + TbQueueProducerProvider producerProvider; + @Mock + TbQueueAdmin queueAdmin; + @Mock + TbMsgPackProcessingContextFactory packProcessingContextFactory; + @Mock + TbMsgPackProcessingContext packCtx; + + Queue mainQueue; + + TbQueueConsumer> consumer; + + TbRuleEngineConsumerContext ruleEngineConsumerContext; + TbRuleEngineQueueConsumerManager consumerManager; + + ExecutorService consumersExecutor; + ScheduledExecutorService scheduler; + ExecutorService mgmtExecutor; + + @BeforeEach + void setup() throws InterruptedException { + consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer")); + scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("tb-rule-engine-consumer-scheduler"); + mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(1, "tb-rule-engine-mgmt"); + + mainQueue = new Queue(); + mainQueue.setTenantId(TenantId.SYS_TENANT_ID); + mainQueue.setName("Main"); + mainQueue.setTopic("tb_rule_engine.main"); + mainQueue.setPollInterval(25); + mainQueue.setPartitions(1); + mainQueue.setConsumerPerPartition(false); + mainQueue.setPackProcessingTimeout(2000L); + + var submitStrategy = new SubmitStrategy(); + submitStrategy.setType(SubmitStrategyType.BURST); + submitStrategy.setBatchSize(1000); + mainQueue.setSubmitStrategy(submitStrategy); + + var processingStrategy = new ProcessingStrategy(); + processingStrategy.setType(ProcessingStrategyType.SKIP_ALL_FAILURES); + processingStrategy.setRetries(3); + processingStrategy.setFailurePercentage(0.0); + processingStrategy.setPauseBetweenRetries(3); + processingStrategy.setMaxPauseBetweenRetries(3); + mainQueue.setProcessingStrategy(processingStrategy); + + storage = new DefaultInMemoryStorage(); + + consumer = spy(new InMemoryTbQueueConsumer<>(storage, mainQueue.getTopic())); + given(queueFactory.createToRuleEngineMsgConsumer(eq(mainQueue), isNull())).willReturn(consumer); + + ruleEngineConsumerContext = new TbRuleEngineConsumerContext( + actorContext, statsFactory, new TbRuleEngineSubmitStrategyFactory(), new TbRuleEngineProcessingStrategyFactory(), + queueFactory, statisticsService, serviceInfoProvider, partitionService, producerProvider, queueAdmin + ); + ruleEngineConsumerContext.setPollDuration(25); + ruleEngineConsumerContext.setPackProcessingTimeout(2000); + ruleEngineConsumerContext.setStatsEnabled(false); // true by default + ruleEngineConsumerContext.setPrometheusStatsEnabled(false); + ruleEngineConsumerContext.setTopicDeletionDelayInSec(15); + ruleEngineConsumerContext.setMgmtThreadPoolSize(12); + + // Tell the (mock) context factory to return (mock) message pack context + given(packProcessingContextFactory.create( + eq(mainQueue.getName()), + any(TbRuleEngineSubmitStrategy.class), + eq(false) + )).willAnswer(invocation -> { + TbRuleEngineSubmitStrategy realStrategy = invocation.getArgument(1); + when(packCtx.getPendingMap()).thenAnswer(i -> realStrategy.getPendingMap()); + when(packCtx.getFailedMap()).thenReturn(new ConcurrentHashMap<>()); + return packCtx; + }); + + // Tell the (mock) context's await() to return 'false' (always timeout) immediately + given(packCtx.await(anyLong(), any(TimeUnit.class))).willReturn(false); + + consumerManager = TbRuleEngineQueueConsumerManager.create() + .ctx(ruleEngineConsumerContext) + .queueKey(new QueueKey(ServiceType.TB_RULE_ENGINE, mainQueue)) + .consumerExecutor(consumersExecutor) + .scheduler(scheduler) + .taskExecutor(mgmtExecutor) + .packProcessingContextFactory(packProcessingContextFactory) + .build(); + } + + @AfterEach + void destroy() { + MoreExecutors.shutdownAndAwaitTermination(scheduler, Duration.ofSeconds(30)); + MoreExecutors.shutdownAndAwaitTermination(mgmtExecutor, Duration.ofSeconds(30)); + MoreExecutors.shutdownAndAwaitTermination(consumersExecutor, Duration.ofSeconds(30)); + } + + @Test + void consumerLoopTest_verifyOperationsOrder() throws InterruptedException { + // Create partition + var partition = TopicPartitionInfo.builder() + .tenantId(TenantId.SYS_TENANT_ID) + .topic(mainQueue.getTopic()) + .partition(0) + .myPartition(true) + .useInternalPartition(false) + .build(); + + // Put 10k messages to the queue + for (int i = 0; i < 10_000; i++) { + var tbMsg = TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(deviceId) + .data("{\"temperature\":123}") + .metaData(TbMsgMetaData.EMPTY) + .build(); + + var toRuleEngineMsg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTbMsgProto(TbMsg.toProto(tbMsg)) + .addAllRelationTypes(Set.of("Success")) + .build(); + + storage.put(partition.getFullTopicName(), new TbProtoQueueMsg<>(UUID.randomUUID(), toRuleEngineMsg, new DefaultTbQueueMsgHeaders())); + } + + // Count how many polls were made + var totalPolls = new AtomicInteger(0); + var emptyPolls = new AtomicInteger(0); + doAnswer(invocation -> { + totalPolls.incrementAndGet(); + @SuppressWarnings("unchecked") + var messages = (List) invocation.callRealMethod(); + if (messages.isEmpty()) { + emptyPolls.incrementAndGet(); + } + return messages; + }).when(consumer).poll(mainQueue.getPollInterval()); + + // Count how many commits were made + var totalCommits = new AtomicInteger(0); + doAnswer(invocation -> { + totalCommits.incrementAndGet(); + return invocation.callRealMethod(); + }).when(consumer).commit(); + + // Initialize consumer + consumerManager.init(mainQueue); + + // Assign partition to the consumer + consumerManager.update(Set.of(partition)); + + // Give some time for the consumer to get all messages + await().atMost(Duration.ofSeconds(10L)).until(() -> storage.getLagTotal() == 0); + + // Stop consumer + consumerManager.stop(); + consumerManager.awaitStop(); + + // Determine number of non-empty consumer iterations made, since polling does not stop immediately after consuming all messages and may do a few empty polls + int nonEmptyPolls = totalPolls.get() - emptyPolls.get(); + + // Verify that there is 10 polls and 10 matching commits + // Each poll consumes 1k messages and queue has 10k total, so that means 10k total msgs / 1k msgs per poll = 10 polls + assertThat(nonEmptyPolls).isEqualTo(10).isEqualTo(totalCommits.get()); + + // Verify that poll-await-commit cycle happened in order with correct await timeout + InOrder inOrder = inOrder(consumer, packCtx); + for (int i = 0; i < nonEmptyPolls; i++) { + inOrder.verify(consumer).poll(mainQueue.getPollInterval()); + inOrder.verify(packCtx).await(mainQueue.getPackProcessingTimeout(), TimeUnit.MILLISECONDS); + inOrder.verify(consumer).commit(); + } + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java index 228b987cf0..e50698200b 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java @@ -59,6 +59,7 @@ import org.thingsboard.server.queue.provider.KafkaMonolithQueueFactory; import org.thingsboard.server.queue.provider.KafkaTbRuleEngineQueueFactory; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.service.queue.TbMsgPackProcessingContextFactory; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; import org.thingsboard.server.service.stats.RuleEngineStatisticsService; @@ -194,6 +195,7 @@ public class TbRuleEngineQueueConsumerManagerTest { .consumerExecutor(consumersExecutor) .scheduler(scheduler) .taskExecutor(mgmtExecutor) + .packProcessingContextFactory(new TbMsgPackProcessingContextFactory.DefaultTbMsgPackProcessingContextFactory()) .build(); } diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineStrategyTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineStrategyTest.java index 3b33d91348..ca47af7b19 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineStrategyTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineStrategyTest.java @@ -45,6 +45,7 @@ import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.consumer.TbQueueConsumerTask.ConsumerKey; import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.service.queue.TbMsgPackProcessingContextFactory; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; @@ -196,6 +197,7 @@ public class TbRuleEngineStrategyTest { var consumerManager = TbRuleEngineQueueConsumerManager.create() .ctx(ruleEngineConsumerContext) .queueKey(queueKey) + .packProcessingContextFactory(new TbMsgPackProcessingContextFactory.DefaultTbMsgPackProcessingContextFactory()) .build(); consumerManager.init(queue); diff --git a/application/src/test/java/org/thingsboard/server/service/sms/DefaultSmsServiceTest.java b/application/src/test/java/org/thingsboard/server/service/sms/DefaultSmsServiceTest.java index 445be6363a..28ce4ea46a 100644 --- a/application/src/test/java/org/thingsboard/server/service/sms/DefaultSmsServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sms/DefaultSmsServiceTest.java @@ -52,6 +52,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @TestPropertySource(properties = { "usage.stats.report.enabled=true", "usage.stats.report.interval=1", + "usage.stats.report.urgent_interval=1" }) public class DefaultSmsServiceTest extends AbstractControllerTest { @SpyBean diff --git a/application/src/test/java/org/thingsboard/server/service/stats/DevicesStatisticsTest.java b/application/src/test/java/org/thingsboard/server/service/stats/DevicesStatisticsTest.java index 31283a1165..e207c2325c 100644 --- a/application/src/test/java/org/thingsboard/server/service/stats/DevicesStatisticsTest.java +++ b/application/src/test/java/org/thingsboard/server/service/stats/DevicesStatisticsTest.java @@ -41,6 +41,7 @@ import static org.awaitility.Awaitility.await; @TestPropertySource(properties = { "usage.stats.report.enabled=true", "usage.stats.report.interval=2", + "usage.stats.report.urgent_interval=1", "usage.stats.gauge_report_interval=1", "usage.stats.devices.report_interval=3", "state.defaultStateCheckIntervalInSec=3", diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/FwLwM2MDevice.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/FwLwM2MDevice.java index 979a2b8c96..9265893fa2 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/FwLwM2MDevice.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/FwLwM2MDevice.java @@ -140,45 +140,61 @@ public class FwLwM2MDevice extends BaseInstanceEnabler implements Destroyable { } private void startDownloading() { + long delay = 0; + // Step 1: state = 1 scheduler.schedule(() -> { - try { - state.set(1); - fireResourceChange(3); - Thread.sleep(100); - state.set(2); - fireResourceChange(3); - } catch (Exception e) { - } - }, 100, TimeUnit.MILLISECONDS); + state.set(1); + fireResourceChange(3); + log.info("Downloading started: state=[{}]", state.get()); + }, delay, TimeUnit.MILLISECONDS); + + delay += 100; // next step after 100 ms + + // Step 2: state = 2 + scheduler.schedule(() -> { + state.set(2); + fireResourceChange(3); + log.info("Downloading in progress: state=[{}]", state.get()); + }, delay, TimeUnit.MILLISECONDS); } + private void startUpdating(LwM2mServer identity) { scheduler.schedule(() -> { try { + // Update state + result state.set(3); fireResourceChange(3); - Thread.sleep(100); + updateResult.set(1); fireResourceChange(5); - this.pkgName = TITLE; - fireResourceChange(6); - this.pkgVersion = TARGET_FW_VERSION; - fireResourceChange(7); + if (this.leshanClient != null) { log.info("Stop/reboot LwM2M client {}", this.leshanClient.getEndpoint(identity)); this.leshanClient.stop(false); + log.info("Start after update fw LwM2M client {}", this.leshanClient.getEndpoint(identity)); this.leshanClient.start(); - this.pkgName = this.pkgNameDef; - this.pkgVersion = this.pkgVersionDef; + + // Delayed reset pkgName/pkgVersion, after reboot + registration + scheduler.schedule(() -> { + this.pkgName = this.pkgNameDef; + fireResourceChange(6); + + this.pkgVersion = this.pkgVersionDef; + fireResourceChange(7); + + log.info("FW resources updating to new values: pkgName=[{}], pkgVersion=[{}]", + this.pkgName, this.pkgVersion); + }, 15, TimeUnit.SECONDS); // 15 sec — safe timing } } catch (Exception e) { + log.error("Error during firmware update", e); } - }, 100, TimeUnit.MILLISECONDS); + }, 0, TimeUnit.SECONDS); // start immediately, without further delay } protected void setLeshanClient(LeshanClient leshanClient) { this.leshanClient = leshanClient; } - } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/SwLwM2MDevice.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/SwLwM2MDevice.java index 76b9827f5a..6db2cf2e9b 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/SwLwM2MDevice.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/SwLwM2MDevice.java @@ -33,6 +33,8 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import static org.thingsboard.server.controller.AbstractWebTest.TIMEOUT; + @Slf4j public class SwLwM2MDevice extends BaseInstanceEnabler implements Destroyable { @@ -85,10 +87,7 @@ public class SwLwM2MDevice extends BaseInstanceEnabler implements Destroyable { log.info("Write on Device resource /{}/{}/{}", getModel().id, getId(), resourceId); switch (resourceId) { - case 2: - startDownloading(); - return WriteResponse.success(); - case 3: + case 2, 3: startDownloading(); return WriteResponse.success(); default: @@ -123,25 +122,34 @@ public class SwLwM2MDevice extends BaseInstanceEnabler implements Destroyable { } private void startDownloading() { + long delay = 0; + + // Step 1: start downloading scheduler.schedule(() -> { - try { - state.set(1); - updateResult.set(1); - fireResourceChange(7); - fireResourceChange(9); - Thread.sleep(100); - state.set(2); - fireResourceChange(7); - Thread.sleep(100); - state.set(3); - fireResourceChange(7); - Thread.sleep(100); - updateResult.set(3); - fireResourceChange(9); - } catch (Exception e) { - - } - }, 100, TimeUnit.MILLISECONDS); + state.set(1); + updateResult.set(1); + fireResourceChange(7); + fireResourceChange(9); + }, delay, TimeUnit.MILLISECONDS); + + delay += 100; + + // Step 2: downloading in progress + scheduler.schedule(() -> { + state.set(2); + fireResourceChange(7); + }, delay, TimeUnit.MILLISECONDS); + + delay += 100; + + // Step 3: downloading finished + scheduler.schedule(() -> { + state.set(3); + fireResourceChange(7); + + updateResult.set(3); + fireResourceChange(9); + }, delay, TimeUnit.MILLISECONDS); } private void startUpdating() { @@ -150,7 +158,13 @@ public class SwLwM2MDevice extends BaseInstanceEnabler implements Destroyable { updateResult.set(2); fireResourceChange(7); fireResourceChange(9); + + // Optional: delayed log about FW update + scheduler.schedule(() -> { + log.info("FW resources updating to new values: state=[{}], updateResult=[{}]", + state.get(), updateResult.get()); + }, 500, TimeUnit.MILLISECONDS); + }, 100, TimeUnit.MILLISECONDS); } - } diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/AbstractMqttV5ClientSparkplugTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/AbstractMqttV5ClientSparkplugTest.java index 6fb08f9eea..4e4d0debf9 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/AbstractMqttV5ClientSparkplugTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/AbstractMqttV5ClientSparkplugTest.java @@ -159,14 +159,17 @@ public abstract class AbstractMqttV5ClientSparkplugTest extends AbstractMqttInte protected List connectClientWithCorrectAccessTokenWithNDEATHCreatedDevices(int cntDevices, long ts) throws Exception { List devices = new ArrayList<>(); clientWithCorrectNodeAccessTokenWithNDEATH(); - MetricDataType metricDataType = Int32; - String key = "Node Metric int32"; + String keyInt = "Node Metric int32"; int valueDeviceInt32 = 1024; - SparkplugBProto.Payload.Metric metric = createMetric(valueDeviceInt32, ts, key, metricDataType, -1L); + SparkplugBProto.Payload.Metric metricInt = createMetric(valueDeviceInt32, ts, keyInt, Int32, -1L); + String keyStringEmpty = "Node Metric String Empty"; + String valueDeviceStringEmpty = ""; + SparkplugBProto.Payload.Metric metricStringEmpty = createMetric(valueDeviceStringEmpty, ts, keyStringEmpty, MetricDataType.String, -1L); SparkplugBProto.Payload.Builder payloadBirthNode = SparkplugBProto.Payload.newBuilder() .setTimestamp(ts) .setSeq(getBdSeqNum()); - payloadBirthNode.addMetrics(metric); + payloadBirthNode.addMetrics(metricInt); + payloadBirthNode.addMetrics(metricStringEmpty); payloadBirthNode.setTimestamp(ts); if (client.isConnected()) { client.publish(TOPIC_ROOT_SPB_V_1_0 + "/" + groupId + "/" + SparkplugMessageType.NBIRTH.name() + "/" + edgeNode, @@ -174,14 +177,14 @@ public abstract class AbstractMqttV5ClientSparkplugTest extends AbstractMqttInte } valueDeviceInt32 = 4024; - metric = createMetric(valueDeviceInt32, ts, metricBirthName_Int32, metricBirthDataType_Int32, -1L); + metricInt = createMetric(valueDeviceInt32, ts, metricBirthName_Int32, metricBirthDataType_Int32, -1L); for (int i = 0; i < cntDevices; i++) { SparkplugBProto.Payload.Builder payloadBirthDevice = SparkplugBProto.Payload.newBuilder() .setTimestamp(ts) .setSeq(getSeqNum()); String deviceName = deviceId + "_" + i; - payloadBirthDevice.addMetrics(metric); + payloadBirthDevice.addMetrics(metricInt); if (client.isConnected()) { client.publish(TOPIC_ROOT_SPB_V_1_0 + "/" + groupId + "/" + SparkplugMessageType.DBIRTH.name() + "/" + edgeNode + "/" + deviceName, payloadBirthDevice.build().toByteArray(), 0, false); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java index 3166c64835..c5e359e2d5 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java @@ -27,9 +27,6 @@ import java.util.Collection; import java.util.List; import java.util.Optional; -/** - * @author Andrew Shvayka - */ public interface AttributesService { ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, String attributeKey); @@ -48,7 +45,13 @@ public interface AttributesService { List findAllKeysByEntityIds(TenantId tenantId, List entityIds); - List findAllKeysByEntityIds(TenantId tenantId, List entityIds, AttributeScope scope); + List findAllKeysByEntityIdsAndScope(TenantId tenantId, List entityIds, AttributeScope scope); + + ListenableFuture> findAllKeysByEntityIdsAndScopeAsync(TenantId tenantId, List entityIds, AttributeScope scope); + + List findLatestByEntityIdsAndScope(TenantId tenantId, List entityIds, AttributeScope scope); + + ListenableFuture> findLatestByEntityIdsAndScopeAsync(TenantId tenantId, List entityIds, AttributeScope scope); int removeAllByEntityId(TenantId tenantId, EntityId entityId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java index 3a67c31b57..fdd722a266 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.entity; import com.google.common.util.concurrent.FluentFuture; +import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; @@ -51,4 +52,6 @@ public interface EntityService { PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query); + ListenableFuture> findEntityDataByQueryAsync(TenantId tenantId, CustomerId customerId, EntityDataQuery query); + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java index 5bf2c63176..6d5727474a 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java @@ -30,9 +30,6 @@ import java.util.Collection; import java.util.List; import java.util.Optional; -/** - * @author Andrew Shvayka - */ public interface TimeseriesService { ListenableFuture> findAllByQueries(TenantId tenantId, EntityId entityId, List queries); @@ -65,6 +62,10 @@ public interface TimeseriesService { ListenableFuture> findAllKeysByEntityIdsAsync(TenantId tenantId, List entityIds); + List findLatestByEntityIds(TenantId tenantId, List entityIds); + + ListenableFuture> findLatestByEntityIdsAsync(TenantId tenantId, List entityIds); + void cleanup(long systemTtl); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java index 5a679ce2cf..ceb657374b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java @@ -25,8 +25,8 @@ public enum ApiUsageRecordKey { RE_EXEC_COUNT(ApiFeature.RE, "ruleEngineExecutionCount", "ruleEngineExecutionLimit", "Rule Engine execution"), JS_EXEC_COUNT(ApiFeature.JS, "jsExecutionCount", "jsExecutionLimit", "JavaScript execution"), TBEL_EXEC_COUNT(ApiFeature.TBEL, "tbelExecutionCount", "tbelExecutionLimit", "Tbel execution"), - EMAIL_EXEC_COUNT(ApiFeature.EMAIL, "emailCount", "emailLimit", "email message"), - SMS_EXEC_COUNT(ApiFeature.SMS, "smsCount", "smsLimit", "SMS message"), + EMAIL_EXEC_COUNT(ApiFeature.EMAIL, "emailCount", "emailLimit", "email message", true, true), + SMS_EXEC_COUNT(ApiFeature.SMS, "smsCount", "smsLimit", "SMS message", true, true), CREATED_ALARMS_COUNT(ApiFeature.ALARM, "createdAlarmsCount", "createdAlarmsLimit", "alarm"), ACTIVE_DEVICES("activeDevicesCount"), INACTIVE_DEVICES("inactiveDevicesCount"); @@ -50,21 +50,24 @@ public enum ApiUsageRecordKey { private final String unitLabel; @Getter private final boolean counter; + @Getter + private final boolean urgent; // urgent keys are reported at a shorter interval for quicker usage state updates ApiUsageRecordKey(ApiFeature apiFeature, String apiCountKey, String apiLimitKey, String unitLabel) { - this(apiFeature, apiCountKey, apiLimitKey, unitLabel, true); + this(apiFeature, apiCountKey, apiLimitKey, unitLabel, true, false); } ApiUsageRecordKey(String apiCountKey) { - this(null, apiCountKey, null, null, false); + this(null, apiCountKey, null, null, false, false); } - ApiUsageRecordKey(ApiFeature apiFeature, String apiCountKey, String apiLimitKey, String unitLabel, boolean counter) { + ApiUsageRecordKey(ApiFeature apiFeature, String apiCountKey, String apiLimitKey, String unitLabel, boolean counter, boolean urgent) { this.apiFeature = apiFeature; this.apiCountKey = apiCountKey; this.apiLimitKey = apiLimitKey; this.unitLabel = unitLabel; this.counter = counter; + this.urgent = urgent; } public static ApiUsageRecordKey[] getKeys(ApiFeature feature) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScheduledUpdateSupportedCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScheduledUpdateSupportedCalculatedFieldConfiguration.java index f5b45c6d5c..f78ff72af0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScheduledUpdateSupportedCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScheduledUpdateSupportedCalculatedFieldConfiguration.java @@ -22,14 +22,14 @@ public interface ScheduledUpdateSupportedCalculatedFieldConfiguration extends Ca boolean isScheduledUpdateEnabled(); @PositiveOrZero - int getScheduledUpdateInterval(); + Integer getScheduledUpdateInterval(); - void setScheduledUpdateInterval(int interval); + void setScheduledUpdateInterval(Integer interval); default void validate(long minAllowedScheduledUpdateInterval) { if (getScheduledUpdateInterval() < minAllowedScheduledUpdateInterval) { - throw new IllegalArgumentException("Scheduled update interval is less than configured " + - "minimum allowed interval in tenant profile: " + minAllowedScheduledUpdateInterval); + throw new IllegalArgumentException("Scheduled update interval (" + getScheduledUpdateInterval() + + " seconds) is less than minimum allowed interval in tenant profile: " + minAllowedScheduledUpdateInterval + " seconds"); } } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java index d905248aa5..a1296a9f0c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java @@ -45,7 +45,7 @@ public class RelatedEntitiesAggregationCalculatedFieldConfiguration implements A private Output output; private boolean useLatestTs; - private int scheduledUpdateInterval; + private Integer scheduledUpdateInterval; @Override public CalculatedFieldType getType() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfiguration.java index cb408c5dbf..9718d996e6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfiguration.java @@ -48,7 +48,7 @@ public class GeofencingCalculatedFieldConfiguration implements ArgumentsBasedCal private Map zoneGroups; private boolean scheduledUpdateEnabled; - private int scheduledUpdateInterval; + private Integer scheduledUpdateInterval; @NotNull private Output output; @@ -88,6 +88,9 @@ public class GeofencingCalculatedFieldConfiguration implements ArgumentsBasedCal @Override public void validate() { + if (scheduledUpdateEnabled && scheduledUpdateInterval == null) { + throw new IllegalArgumentException("Refresh interval is required when periodic zone group refresh is enabled."); + } zoneGroups.forEach((key, value) -> value.validate(key)); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeysV2.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeysV2.java new file mode 100644 index 0000000000..536b70e4ff --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeysV2.java @@ -0,0 +1,99 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.query; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.JsonNode; +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Schema; +import org.jspecify.annotations.Nullable; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +@Schema( + description = """ + Contains unique time series and attribute key names discovered from entities matching a query, + optionally including a sample value for each key.""" +) +@JsonInclude(JsonInclude.Include.NON_NULL) +public record AvailableEntityKeysV2( + @Schema( + description = "Set of entity types found among the matched entities.", + example = "[\"DEVICE\", \"ASSET\"]", + requiredMode = Schema.RequiredMode.REQUIRED + ) + Set entityTypes, + + @ArraySchema( + arraySchema = @Schema( + description = """ + List of unique time series keys available on the matched entities, sorted alphabetically. + Omitted when timeseries keys were not requested.""", + nullable = true + ), + schema = @Schema(implementation = KeyInfo.class) + ) + @Nullable List timeseries, + + @Schema( + description = """ + Map of attribute scope to the list of unique attribute keys available on the matched entities. + Only scopes supported by the matched entity types are included. + Omitted when attribute keys were not requested or when none of the requested scopes apply to the matched entity types.""", + nullable = true + ) + @Nullable Map> attributes +) { + + @Schema(description = "Key name with an optional sample value.") + @JsonInclude(JsonInclude.Include.NON_NULL) + public record KeyInfo( + @Schema( + description = "Key name.", + example = "temperature", + requiredMode = Schema.RequiredMode.REQUIRED + ) + String key, + + @Schema( + description = "Most recent sample value for this key across the matched entities. Omitted when samples were not requested.", + nullable = true + ) + @Nullable KeySample sample + ) {} + + @Schema(description = "Most recent value and its timestamp.") + public record KeySample( + @Schema( + description = "Timestamp in milliseconds since epoch.", example = "1707000000000", + requiredMode = Schema.RequiredMode.REQUIRED + ) + long ts, + + @Schema( + description = "Sample value.", + example = "23.5", + requiredMode = Schema.RequiredMode.REQUIRED, + implementation = Object.class + ) + JsonNode value + ) {} + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index a04816365b..297f92cffb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -17,6 +17,7 @@ package org.thingsboard.server.common.data.tenant.profile; import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.constraints.Positive; +import jakarta.validation.constraints.PositiveOrZero; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -173,6 +174,7 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura @Schema(example = "10") private long maxArgumentsPerCF = 10; @Schema(example = "10") + @PositiveOrZero private int minAllowedScheduledUpdateIntervalInSecForCF = 10; @Builder.Default @Schema(example = "2") diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/ScheduledUpdateSupportedCalculatedFieldConfigurationTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/ScheduledUpdateSupportedCalculatedFieldConfigurationTest.java index 23e0f1add3..a2df32b483 100644 --- a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/ScheduledUpdateSupportedCalculatedFieldConfigurationTest.java +++ b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/ScheduledUpdateSupportedCalculatedFieldConfigurationTest.java @@ -47,8 +47,8 @@ public class ScheduledUpdateSupportedCalculatedFieldConfigurationTest { assertThatThrownBy(() -> cfg.validate(minAllowedInterval)) .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Scheduled update interval is less than configured " + - "minimum allowed interval in tenant profile: " + minAllowedInterval); + .hasMessage("Scheduled update interval (1 seconds) is less than " + + "minimum allowed interval in tenant profile: " + minAllowedInterval + " seconds"); } } diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfigurationTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfigurationTest.java index 2f391435b2..2c68bcd8c5 100644 --- a/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfigurationTest.java +++ b/common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfigurationTest.java @@ -28,6 +28,7 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates.ENTITY_ID_LATITUDE_ARGUMENT_KEY; @@ -103,4 +104,17 @@ public class GeofencingCalculatedFieldConfigurationTest { assertThat(allowedZonesArgument.getRefEntityKey()).isEqualTo(new ReferencedEntityKey("perimeter", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); } + @Test + void validateShouldThrowWhenScheduledUpdateEnabledButIntervalNotSet() { + var cfg = new GeofencingCalculatedFieldConfiguration(); + cfg.setEntityCoordinates(mock(EntityCoordinates.class)); + cfg.setZoneGroups(Map.of("zone", mock(ZoneGroupConfiguration.class))); + cfg.setScheduledUpdateEnabled(true); + cfg.setScheduledUpdateInterval(null); + + assertThatThrownBy(cfg::validate) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Refresh interval is required when periodic zone group refresh is enabled."); + } + } diff --git a/common/edge-api/src/main/proto/edge.proto b/common/edge-api/src/main/proto/edge.proto index f5918cbf29..472ce8e8dd 100644 --- a/common/edge-api/src/main/proto/edge.proto +++ b/common/edge-api/src/main/proto/edge.proto @@ -46,12 +46,12 @@ enum EdgeVersion { V_4_2_0 = 12; V_4_3_0 = 13; V_4_2_1_2 = 14; - V_4_2_1_3 = 420; + V_4_2_2 = 4220; V_4_3_0_1 = 15; - V_4_3_0_2 = 430; - V_4_4_0 = 440; + V_4_3_1 = 4310; + V_4_4_0 = 4400; - V_LATEST = 999; + V_LATEST = 99999; } /** diff --git a/common/message/src/test/java/org/thingsboard/server/common/msg/tools/TbRateLimitsTest.java b/common/message/src/test/java/org/thingsboard/server/common/msg/tools/TbRateLimitsTest.java index 39840d8f1c..0ec7ce0d76 100644 --- a/common/message/src/test/java/org/thingsboard/server/common/msg/tools/TbRateLimitsTest.java +++ b/common/message/src/test/java/org/thingsboard/server/common/msg/tools/TbRateLimitsTest.java @@ -42,7 +42,7 @@ public class TbRateLimitsTest { assertThat(rateLimits.tryConsume()).as("new token is available").isFalse(); int expectedRefillTime = (int) (((double) period / capacity) * 1000); - int gap = 500; + int gap = 1000; for (int i = 0; i < capacity; i++) { await("token refill for rate limit " + rateLimitConfig) @@ -71,7 +71,7 @@ public class TbRateLimitsTest { assertThat(rateLimits.tryConsume()).as("new token is available").isFalse(); int expectedRefillTime = period * 1000; - int gap = 500; + int gap = 1000; await("tokens refill for rate limit " + rateLimitConfig) .pollInterval(new FixedPollInterval(10, TimeUnit.MILLISECONDS)) diff --git a/common/proto/src/main/java/org/thingsboard/server/common/adaptor/ProtoConverter.java b/common/proto/src/main/java/org/thingsboard/server/common/adaptor/ProtoConverter.java index 6dc9387a37..a47caae954 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/adaptor/ProtoConverter.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/adaptor/ProtoConverter.java @@ -156,11 +156,7 @@ public class ProtoConverter { case BOOLEAN_V: case LONG_V: case DOUBLE_V: - break; case STRING_V: - if (StringUtils.isEmpty(keyValueProto.getStringV())) { - throw new IllegalArgumentException("Value is empty for key: " + key + "!"); - } break; case JSON_V: try { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageReportClient.java b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageReportClient.java index b6a785316b..50e131103b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageReportClient.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageReportClient.java @@ -63,8 +63,10 @@ public class DefaultTbApiUsageReportClient implements TbApiUsageReportClient { private boolean enabled; @Value("${usage.stats.report.enabled_per_customer:false}") private boolean enabledPerCustomer; - @Value("${usage.stats.report.interval:10}") + @Value("${usage.stats.report.interval:60}") private int interval; + @Value("${usage.stats.report.urgent_interval:10}") + private int urgentInterval; @Value("${usage.stats.report.pack_size:1024}") private int packSize; @@ -83,20 +85,30 @@ public class DefaultTbApiUsageReportClient implements TbApiUsageReportClient { for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { stats.put(key, new ConcurrentHashMap<>()); } + Random random = new Random(); scheduler.scheduleWithFixedDelay(() -> { try { - reportStats(); + reportStats(false); } catch (Exception e) { log.warn("Failed to report statistics: ", e); } - }, new Random().nextInt(interval), interval, TimeUnit.SECONDS); + }, random.nextInt(interval), interval, TimeUnit.SECONDS); + scheduler.scheduleWithFixedDelay(() -> { + try { + reportStats(true); + } catch (Exception e) { + log.warn("Failed to report urgent statistics: ", e); + } + }, random.nextInt(urgentInterval), urgentInterval, TimeUnit.SECONDS); } } - private void reportStats() { + private void reportStats(boolean urgent) { ConcurrentMap report = new ConcurrentHashMap<>(); for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { + if (key.isUrgent() != urgent) continue; + ConcurrentMap statsForKey = stats.get(key); statsForKey.forEach((reportLevel, statsValue) -> { long value = statsValue.get(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java index 2241b20e14..7c563debd6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java @@ -55,6 +55,12 @@ public interface AttributesDao { List findAllKeysByEntityIdsAndScope(TenantId tenantId, List entityIds, AttributeScope scope); + ListenableFuture> findAllKeysByEntityIdsAndScopeAsync(TenantId tenantId, List entityIds, AttributeScope scope); + + List findLatestByEntityIdsAndScope(TenantId tenantId, List entityIds, AttributeScope scope); + + ListenableFuture> findLatestByEntityIdsAndScopeAsync(TenantId tenantId, List entityIds, AttributeScope scope); + List> removeAllByEntityId(TenantId tenantId, EntityId entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 117c47b61c..ca4c937d4d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -45,9 +45,6 @@ import java.util.Optional; import static org.thingsboard.server.dao.attributes.AttributeUtils.validate; -/** - * @author Andrew Shvayka - */ @Service @ConditionalOnProperty(prefix = "cache.attributes", value = "enabled", havingValue = "false", matchIfMissing = true) @Primary @@ -92,7 +89,7 @@ public class BaseAttributesService implements AttributesService { } @Override - public List findAllKeysByEntityIds(TenantId tenantId, List entityIds, AttributeScope scope) { + public List findAllKeysByEntityIdsAndScope(TenantId tenantId, List entityIds, AttributeScope scope) { if (scope == null) { return attributesDao.findAllKeysByEntityIds(tenantId, entityIds); } else { @@ -100,6 +97,21 @@ public class BaseAttributesService implements AttributesService { } } + @Override + public ListenableFuture> findAllKeysByEntityIdsAndScopeAsync(TenantId tenantId, List entityIds, AttributeScope scope) { + return attributesDao.findAllKeysByEntityIdsAndScopeAsync(tenantId, entityIds, scope); + } + + @Override + public List findLatestByEntityIdsAndScope(TenantId tenantId, List entityIds, AttributeScope scope) { + return attributesDao.findLatestByEntityIdsAndScope(tenantId, entityIds, scope); + } + + @Override + public ListenableFuture> findLatestByEntityIdsAndScopeAsync(TenantId tenantId, List entityIds, AttributeScope scope) { + return attributesDao.findLatestByEntityIdsAndScopeAsync(tenantId, entityIds, scope); + } + @Override public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { validate(entityId, scope); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index c150eabfa2..1c99dfc495 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -66,6 +66,7 @@ import static org.thingsboard.server.dao.attributes.AttributeUtils.validate; @Primary @Slf4j public class CachedAttributesService implements AttributesService { + private static final String STATS_NAME = "attributes.cache"; public static final String LOCAL_CACHE_TYPE = "caffeine"; @@ -212,7 +213,7 @@ public class CachedAttributesService implements AttributesService { } @Override - public List findAllKeysByEntityIds(TenantId tenantId, List entityIds, AttributeScope scope) { + public List findAllKeysByEntityIdsAndScope(TenantId tenantId, List entityIds, AttributeScope scope) { if (scope == null) { return attributesDao.findAllKeysByEntityIds(tenantId, entityIds); } else { @@ -220,6 +221,21 @@ public class CachedAttributesService implements AttributesService { } } + @Override + public ListenableFuture> findAllKeysByEntityIdsAndScopeAsync(TenantId tenantId, List entityIds, AttributeScope scope) { + return attributesDao.findAllKeysByEntityIdsAndScopeAsync(tenantId, entityIds, scope); + } + + @Override + public List findLatestByEntityIdsAndScope(TenantId tenantId, List entityIds, AttributeScope scope) { + return attributesDao.findLatestByEntityIdsAndScope(tenantId, entityIds, scope); + } + + @Override + public ListenableFuture> findLatestByEntityIdsAndScopeAsync(TenantId tenantId, List entityIds, AttributeScope scope) { + return attributesDao.findLatestByEntityIdsAndScopeAsync(tenantId, entityIds, scope); + } + @Override public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { validate(entityId, scope); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 00c5ec372e..075e2d8a9d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -26,18 +26,21 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldFilter; import org.thingsboard.server.common.data.cf.CalculatedFieldInfo; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.entity.AbstractEntityService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.service.validator.CalculatedFieldDataValidator; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import java.util.EnumSet; import java.util.List; @@ -62,6 +65,7 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements private final EntityService entityService; private final CalculatedFieldDao calculatedFieldDao; private final CalculatedFieldDataValidator calculatedFieldDataValidator; + private final ApiLimitService apiLimitService; @Override public CalculatedField save(CalculatedField calculatedField) { @@ -70,6 +74,7 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements @Override public CalculatedField save(CalculatedField calculatedField, boolean doValidate) { + setConfigurationDefaults(calculatedField); CalculatedField oldCalculatedField = null; if (doValidate) { oldCalculatedField = calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId); @@ -79,6 +84,15 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return doSave(calculatedField, oldCalculatedField); } + private void setConfigurationDefaults(CalculatedField calculatedField) { + if (calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration config + && config.getScheduledUpdateInterval() == null) { + int minScheduledUpdateInterval = (int) apiLimitService.getLimit( + calculatedField.getTenantId(), DefaultTenantProfileConfiguration::getMinAllowedScheduledUpdateIntervalInSecForCF + ); + config.setScheduledUpdateInterval(minScheduledUpdateInterval); + } + } private CalculatedField doSave(CalculatedField calculatedField, CalculatedField oldCalculatedField) { try { diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index dc950cebd4..b20535dbf4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -16,6 +16,9 @@ package org.thingsboard.server.dao.entity; import com.google.common.util.concurrent.FluentFuture; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; @@ -55,6 +58,7 @@ import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.common.stats.EdqsStatsService; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.sql.JpaExecutorService; import java.util.ArrayList; import java.util.Collections; @@ -104,6 +108,9 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe @Autowired private EdqsStatsService edqsStatsService; + @Autowired + private JpaExecutorService jpaExecutorService; + @Override public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query) { log.trace("Executing countEntitiesByQuery, tenantId [{}], customerId [{}], query [{}]", tenantId, customerId, query); @@ -142,41 +149,78 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe EdqsResponse response = processEdqsRequest(tenantId, customerId, request); result = response.getEntityDataQueryResult(); } else { - if (!isValidForOptimization(query)) { - result = entityQueryDao.findEntityDataByQuery(tenantId, customerId, query); - } else { - // 1 step - find entity data by filter and sort columns - PageData entityDataByQuery = findEntityIdsByFilterAndSorterColumns(tenantId, customerId, query); - if (entityDataByQuery == null || entityDataByQuery.getData().isEmpty()) { - result = entityDataByQuery; - } else { - // 2 step - find entity data by entity ids from the 1st step - List entities = fetchEntityDataByIdsFromInitialQuery(tenantId, customerId, query, entityDataByQuery.getData()); - result = new PageData<>(entities, entityDataByQuery.getTotalPages(), entityDataByQuery.getTotalElements(), entityDataByQuery.hasNext()); - } - } + result = findEntityDataByQueryInternal(tenantId, customerId, query); } edqsStatsService.reportEntityDataQuery(tenantId, query, System.nanoTime() - startNs); return result; } + @Override + public ListenableFuture> findEntityDataByQueryAsync(TenantId tenantId, CustomerId customerId, EntityDataQuery query) { + log.trace("Executing findEntityDataByQueryAsync, tenantId [{}], customerId [{}], query [{}]", tenantId, customerId, query); + + try { + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); + validateEntityDataQuery(query); + } catch (Exception e) { + return Futures.immediateFailedFuture(e); + } + + if (edqsService.isApiEnabled() && validForEdqs(query) && !tenantId.isSysTenantId()) { + EdqsRequest request = EdqsRequest.builder() + .entityDataQuery(query) + .build(); + long startNs = System.nanoTime(); + return Futures.transform(processEdqsRequestAsync(tenantId, customerId, request), response -> { + edqsStatsService.reportEntityDataQuery(tenantId, query, System.nanoTime() - startNs); + return response.getEntityDataQueryResult(); + }, MoreExecutors.directExecutor()); + } + + return jpaExecutorService.submit(() -> { + long startNs = System.nanoTime(); + PageData result = findEntityDataByQueryInternal(tenantId, customerId, query); + edqsStatsService.reportEntityDataQuery(tenantId, query, System.nanoTime() - startNs); + return result; + }); + } + + private PageData findEntityDataByQueryInternal(TenantId tenantId, CustomerId customerId, EntityDataQuery query) { + if (!isValidForOptimization(query)) { + return entityQueryDao.findEntityDataByQuery(tenantId, customerId, query); + } + // 1 step - find entity data by filter and sort columns + PageData entityDataByQuery = findEntityIdsByFilterAndSorterColumns(tenantId, customerId, query); + if (entityDataByQuery == null || entityDataByQuery.getData().isEmpty()) { + return entityDataByQuery; + } + // 2 step - find entity data by entity ids from the 1st step + List entities = fetchEntityDataByIdsFromInitialQuery(tenantId, customerId, query, entityDataByQuery.getData()); + return new PageData<>(entities, entityDataByQuery.getTotalPages(), entityDataByQuery.getTotalElements(), entityDataByQuery.hasNext()); + } + private boolean validForEdqs(EntityCountQuery query) { // for compatibility with PE return true; } private EdqsResponse processEdqsRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) { - EdqsResponse response; try { - log.debug("[{}] Sending request to EDQS: {}", tenantId, request); - response = edqsApiService.processRequest(tenantId, customerId, request).get(); + return processEdqsRequestAsync(tenantId, customerId, request).get(); } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } - log.debug("[{}] Received response from EDQS: {}", tenantId, response); - if (response.getError() != null) { - throw new RuntimeException(response.getError()); - } - return response; + } + + private ListenableFuture processEdqsRequestAsync(TenantId tenantId, CustomerId customerId, EdqsRequest request) { + log.debug("[{}] Sending request to EDQS: {}", tenantId, request); + return Futures.transform(edqsApiService.processRequest(tenantId, customerId, request), response -> { + log.debug("[{}] Received response from EDQS: {}", tenantId, response); + if (response.getError() != null) { + throw new RuntimeException(response.getError()); + } + return response; + }, MoreExecutors.directExecutor()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java index 44a38bf445..971c06c7a0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java @@ -65,6 +65,12 @@ import static org.thingsboard.server.dao.model.ModelConstants.VERSION_COLUMN; query = SearchTsKvLatestRepository.FIND_ALL_BY_ENTITY_ID_QUERY, resultSetMapping = "tsKvLatestFindMapping", resultClass = TsKvLatestEntity.class + ), + @NamedNativeQuery( + name = SearchTsKvLatestRepository.FIND_LATEST_BY_ENTITY_IDS, + query = SearchTsKvLatestRepository.FIND_LATEST_BY_ENTITY_IDS_QUERY, + resultSetMapping = "tsKvLatestFindMapping", + resultClass = TsKvLatestEntity.class ) }) public final class TsKvLatestEntity extends AbstractTsKvEntity { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java index aa7f0490d7..8a3405549e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java @@ -20,6 +20,14 @@ import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.dao.model.sql.AttributeKvCompositeKey; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; @@ -60,6 +68,19 @@ public interface AttributeKvRepository extends JpaRepository findAllKeysByEntityIdsAndAttributeType(@Param("entityIds") List entityIds, @Param("attributeType") int attributeType); + @Query(value = """ + SELECT DISTINCT ON (a.attribute_key) + kd.key AS strKey, + a.bool_v AS boolV, a.str_v AS strV, a.long_v AS longV, + a.dbl_v AS dblV, a.json_v AS jsonV, + a.last_update_ts AS lastUpdateTs, a.version AS version + FROM attribute_kv a + INNER JOIN key_dictionary kd ON a.attribute_key = kd.key_id + WHERE a.entity_id IN :entityIds AND a.attribute_type = :attributeType + ORDER BY a.attribute_key, a.last_update_ts DESC""", nativeQuery = true) + List findLatestByEntityIdsAndAttributeType(@Param("entityIds") List entityIds, + @Param("attributeType") int attributeType); + @Query(value = "SELECT attribute_key, attribute_type, entity_id, bool_v, dbl_v, json_v, last_update_ts, long_v, str_v, version FROM attribute_kv WHERE (entity_id, attribute_type, attribute_key) > " + "(:entityId, :attributeType, :attributeKey) ORDER BY entity_id, attribute_type, attribute_key LIMIT :batchSize", nativeQuery = true) List findNextBatch(@Param("entityId") UUID entityId, @@ -67,4 +88,40 @@ public interface AttributeKvRepository extends JpaRepository> findAllKeysByEntityIdsAndScopeAsync(TenantId tenantId, List entityIds, AttributeScope scope) { + return service.submit(() -> findAllKeysByEntityIdsAndScope(tenantId, entityIds, scope)); + } + + @Override + public List findLatestByEntityIdsAndScope(TenantId tenantId, List entityIds, AttributeScope scope) { + if (CollectionUtils.isEmpty(entityIds)) { + return Collections.emptyList(); + } + var uniqueIds = entityIds.stream().map(EntityId::getId).distinct().toList(); + return attributeKvRepository.findLatestByEntityIdsAndAttributeType(uniqueIds, scope.getId()) + .stream() + .map(AttributeKvRepository.AttributeKvProjection::toAttributeKvEntry) + .toList(); + } + + @Override + public ListenableFuture> findLatestByEntityIdsAndScopeAsync(TenantId tenantId, List entityIds, AttributeScope scope) { + return service.submit(() -> findLatestByEntityIdsAndScope(tenantId, entityIds, scope)); + } + @Override public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, AttributeKvEntry attribute) { AttributeKvEntity entity = new AttributeKvEntity(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java index eabbdfed80..143ab53e18 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java @@ -172,4 +172,14 @@ public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseries return sqlDao.findAllKeysByEntityIdsAsync(tenantId, entityIds); } + @Override + public List findLatestByEntityIds(TenantId tenantId, List entityIds) { + return sqlDao.findLatestByEntityIds(tenantId, entityIds); + } + + @Override + public ListenableFuture> findLatestByEntityIdsAsync(TenantId tenantId, List entityIds) { + return sqlDao.findLatestByEntityIdsAsync(tenantId, entityIds); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index d807794ddc..b33ceca63b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -22,6 +22,7 @@ import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -54,6 +55,7 @@ import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; import org.thingsboard.server.dao.util.SqlTsLatestAnyDao; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -189,6 +191,21 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme return service.submit(() -> findAllKeysByEntityIds(tenantId, entityIds)); } + @Override + public List findLatestByEntityIds(TenantId tenantId, List entityIds) { + if (CollectionUtils.isEmpty(entityIds)) { + return Collections.emptyList(); + } + return DaoUtil.convertDataList( + searchTsKvLatestRepository.findLatestByEntityIds(entityIds.stream().map(EntityId::getId).toList()) + ); + } + + @Override + public ListenableFuture> findLatestByEntityIdsAsync(TenantId tenantId, List entityIds) { + return service.submit(() -> findLatestByEntityIds(tenantId, entityIds)); + } + private ListenableFuture getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query, Long version) { ListenableFuture> future = findNewLatestEntryFuture(tenantId, entityId, query); return Futures.transformAsync(future, entryList -> { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java index 0ab1994059..8b3a66612c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java @@ -34,6 +34,20 @@ public class SearchTsKvLatestRepository { " ts_kv_latest.bool_v AS boolValue, ts_kv_latest.long_v AS longValue, ts_kv_latest.dbl_v AS doubleValue, ts_kv_latest.json_v AS jsonValue, ts_kv_latest.ts AS ts, ts_kv_latest.version AS version FROM ts_kv_latest " + "INNER JOIN key_dictionary ON ts_kv_latest.key = key_dictionary.key_id WHERE ts_kv_latest.entity_id = cast(:id AS uuid)"; + public static final String FIND_LATEST_BY_ENTITY_IDS = "findLatestByEntityIds"; + + public static final String FIND_LATEST_BY_ENTITY_IDS_QUERY = """ + SELECT DISTINCT ON (ts_kv_latest.key) + ts_kv_latest.entity_id AS entityId, ts_kv_latest.key AS key, + key_dictionary.key AS strKey, ts_kv_latest.str_v AS strValue, + ts_kv_latest.bool_v AS boolValue, ts_kv_latest.long_v AS longValue, + ts_kv_latest.dbl_v AS doubleValue, ts_kv_latest.json_v AS jsonValue, + ts_kv_latest.ts AS ts, ts_kv_latest.version AS version + FROM ts_kv_latest + INNER JOIN key_dictionary ON ts_kv_latest.key = key_dictionary.key_id + WHERE ts_kv_latest.entity_id IN (:entityIds) + ORDER BY ts_kv_latest.key, ts_kv_latest.ts DESC"""; + @PersistenceContext private EntityManager entityManager; @@ -43,4 +57,10 @@ public class SearchTsKvLatestRepository { .getResultList(); } + public List findLatestByEntityIds(List entityIds) { + return entityManager.createNamedQuery(FIND_LATEST_BY_ENTITY_IDS, TsKvLatestEntity.class) + .setParameter("entityIds", entityIds) + .getResultList(); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index 747197470c..70e04eb5e3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -55,9 +55,6 @@ import java.util.stream.Collectors; import static org.thingsboard.server.common.data.StringUtils.isBlank; -/** - * @author Andrew Shvayka - */ @Service @Slf4j public class BaseTimeseriesService implements TimeseriesService { @@ -161,6 +158,16 @@ public class BaseTimeseriesService implements TimeseriesService { return timeseriesLatestDao.findAllKeysByEntityIdsAsync(tenantId, entityIds); } + @Override + public List findLatestByEntityIds(TenantId tenantId, List entityIds) { + return timeseriesLatestDao.findLatestByEntityIds(tenantId, entityIds); + } + + @Override + public ListenableFuture> findLatestByEntityIdsAsync(TenantId tenantId, List entityIds) { + return timeseriesLatestDao.findLatestByEntityIdsAsync(tenantId, entityIds); + } + @Override public void cleanup(long systemTtl) { timeseriesDao.cleanup(systemTtl); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java index 6bcc88c2df..b3cc8496a0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -104,6 +104,16 @@ public class CassandraBaseTimeseriesLatestDao extends AbstractCassandraBaseTimes return Futures.immediateFuture(Collections.emptyList()); } + @Override + public List findLatestByEntityIds(TenantId tenantId, List entityIds) { + return Collections.emptyList(); + } + + @Override + public ListenableFuture> findLatestByEntityIdsAsync(TenantId tenantId, List entityIds) { + return Futures.immediateFuture(Collections.emptyList()); + } + @Override public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getLatestStmt().bind()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java index 26e784b760..c7582944a9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java @@ -52,4 +52,13 @@ public interface TimeseriesLatestDao { ListenableFuture> findAllKeysByEntityIdsAsync(TenantId tenantId, List entityIds); + /** + * For each unique timeseries key across the given entities, returns the single most recent {@link TsKvEntry} + * (i.e. the entry with the highest timestamp). If the same key exists on multiple entities, + * only the freshest value is kept. Useful for discovering available keys together with a representative sample value. + */ + List findLatestByEntityIds(TenantId tenantId, List entityIds); + + ListenableFuture> findLatestByEntityIdsAsync(TenantId tenantId, List entityIds); + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java index 9a467071ef..5ee9af6292 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java @@ -93,11 +93,13 @@ public abstract class AbstractServiceTest { @Autowired protected EntityServiceRegistry entityServiceRegistry; + protected Tenant tenant; protected TenantId tenantId; @Before public void beforeAbstractService() { - tenantId = createTenant().getId(); + tenant = createTenant(); + tenantId = tenant.getId(); } @After diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index ba914c6d35..c9524ba7b5 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.dao.service; +import org.apache.commons.lang3.RandomUtils; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; @@ -29,6 +31,10 @@ import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.TimeSeriesOutput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.ZoneGroupConfiguration; @@ -39,6 +45,7 @@ import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; +import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.exception.DataValidationException; import java.util.ArrayList; @@ -59,6 +66,8 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { private DeviceService deviceService; @Autowired private TbTenantProfileCache tbTenantProfileCache; + @Autowired + private TenantProfileService tenantProfileService; @Test public void testSaveCalculatedField() { @@ -82,8 +91,6 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { assertThat(updatedCalculatedField.getName()).isEqualTo(savedCalculatedField.getName()); assertThat(updatedCalculatedField.getVersion()).isEqualTo(savedCalculatedField.getVersion() + 1); - - calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); } @Test @@ -113,11 +120,11 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { int min = tbTenantProfileCache.get(tenantId) .getDefaultProfileConfiguration() .getMinAllowedScheduledUpdateIntervalInSecForCF(); - int valueFromConfig = min - 10; // Enable scheduling with an interval below tenant min cfg.setScheduledUpdateEnabled(true); - cfg.setScheduledUpdateInterval(valueFromConfig); + int invalidInterval = RandomUtils.insecure().randomInt(1, min); + cfg.setScheduledUpdateInterval(invalidInterval); // Create & save Calculated Field CalculatedField cf = new CalculatedField(); @@ -131,8 +138,8 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { assertThatThrownBy(() -> calculatedFieldService.save(cf)) .isInstanceOf(DataValidationException.class) .hasCauseInstanceOf(IllegalArgumentException.class) - .hasMessageStartingWith("Scheduled update interval is less than configured " + - "minimum allowed interval in tenant profile: "); + .hasMessage("Scheduled update interval (" + invalidInterval + + " seconds) is less than minimum allowed interval in tenant profile: " + min + " seconds"); } @Test @@ -233,8 +240,67 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { int savedInterval = geofencingCalculatedFieldConfiguration.getScheduledUpdateInterval(); assertThat(savedInterval).isEqualTo(valueFromConfig); + } - calculatedFieldService.deleteCalculatedField(tenantId, saved.getId()); + @Test + public void testSaveGeofencingCalculatedField_shouldAcceptZeroScheduledUpdateIntervalWhenTenantProfileAllows() { + // GIVEN + var device = createTestDevice(); + + // Store original value and update tenant profile to allow 0 as min scheduled update interval + TenantProfile tenantProfile = tenantProfileService.findTenantProfileById(tenantId, tenant.getTenantProfileId()); + int originalMinScheduledUpdateInterval = tenantProfile.getDefaultProfileConfiguration().getMinAllowedScheduledUpdateIntervalInSecForCF(); + tenantProfile.getDefaultProfileConfiguration().setMinAllowedScheduledUpdateIntervalInSecForCF(0); + tenantProfileService.saveTenantProfile(tenantId, tenantProfile); + tbTenantProfileCache.evict(tenantProfile.getId()); + + try { + // Build a valid Geofencing configuration + var cfg = new GeofencingCalculatedFieldConfiguration(); + + // Coordinates: TS_LATEST, no dynamic source + var entityCoordinates = new EntityCoordinates("latitude", "longitude"); + cfg.setEntityCoordinates(entityCoordinates); + + // Zone-group argument (ATTRIBUTE) — make it DYNAMIC so scheduling is enabled + var zoneGroupConfiguration = new ZoneGroupConfiguration("allowed", REPORT_TRANSITION_EVENTS_AND_PRESENCE_STATUS, false); + var dynamicSourceConfiguration = new RelationPathQueryDynamicSourceConfiguration(); + dynamicSourceConfiguration.setLevels(List.of(new RelationPathLevel(EntitySearchDirection.FROM, EntityRelation.CONTAINS_TYPE))); + zoneGroupConfiguration.setRefDynamicSourceConfiguration(dynamicSourceConfiguration); + cfg.setZoneGroups(Map.of("allowed", zoneGroupConfiguration)); + + // Enable scheduling with interval = 0 + cfg.setScheduledUpdateEnabled(true); + cfg.setScheduledUpdateInterval(0); + + // Create Calculated Field + var cf = new CalculatedField(); + cf.setTenantId(tenantId); + cf.setEntityId(device.getId()); + cf.setType(CalculatedFieldType.GEOFENCING); + cf.setName("GF zero scheduled update interval test"); + cf.setConfigurationVersion(0); + cf.setConfiguration(cfg); + + var out = new AttributesOutput(); + out.setScope(AttributeScope.SERVER_SCOPE); + cfg.setOutput(out); + + // WHEN + CalculatedField saved = calculatedFieldService.save(cf); + + // THEN + assertThat(saved).isNotNull(); + assertThat(saved.getConfiguration()).isInstanceOf(GeofencingCalculatedFieldConfiguration.class); + + var savedConfig = (GeofencingCalculatedFieldConfiguration) saved.getConfiguration(); + assertThat(savedConfig.getScheduledUpdateInterval()).isEqualTo(0); + } finally { + // Restore original tenant profile value + tenantProfile.getProfileConfiguration().orElseThrow().setMinAllowedScheduledUpdateIntervalInSecForCF(originalMinScheduledUpdateInterval); + tenantProfileService.saveTenantProfile(tenantId, tenantProfile); + tbTenantProfileCache.evict(tenantProfile.getId()); + } } @Test @@ -254,8 +320,6 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { CalculatedField fetchedCalculatedField = calculatedFieldService.findById(tenantId, savedCalculatedField.getId()); assertThat(fetchedCalculatedField).isEqualTo(savedCalculatedField); - - calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); } @Test @@ -267,6 +331,156 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { assertThat(calculatedFieldService.findById(tenantId, savedCalculatedField.getId())).isNull(); } + @Test + public void testSaveRelatedEntitiesAggregationCF_shouldUseMinScheduledUpdateIntervalFromTenantProfileWhenNotSet() { + // GIVEN + var device = createTestDevice(); + + var cfg = new RelatedEntitiesAggregationCalculatedFieldConfiguration(); + cfg.setRelation(new RelationPathLevel(EntitySearchDirection.FROM, EntityRelation.CONTAINS_TYPE)); + + var argument = new Argument(); + argument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + cfg.setArguments(Map.of("temp", argument)); + + var metric = new AggMetric(); + metric.setFunction(AggFunction.AVG); + metric.setInput(new AggKeyInput("temp")); + cfg.setMetrics(Map.of("avgTemp", metric)); + + var output = new TimeSeriesOutput(); + output.setName("avgTemperature"); + cfg.setOutput(output); + + int minDeduplicationInterval = (int) tbTenantProfileCache.get(tenantId) + .getDefaultProfileConfiguration() + .getMinAllowedDeduplicationIntervalInSecForCF(); + cfg.setDeduplicationIntervalInSec(minDeduplicationInterval); + + // Do NOT set scheduledUpdateInterval - it should default to tenant profile min value + + var cf = new CalculatedField(); + cf.setTenantId(tenantId); + cf.setEntityId(device.getId()); + cf.setType(CalculatedFieldType.RELATED_ENTITIES_AGGREGATION); + cf.setName("Related Entities Aggregation CF - default scheduled interval test"); + cf.setConfigurationVersion(0); + cf.setConfiguration(cfg); + + // WHEN + CalculatedField saved = calculatedFieldService.save(cf); + + // THEN + assertThat(saved).isNotNull(); + assertThat(saved.getConfiguration()).isInstanceOf(RelatedEntitiesAggregationCalculatedFieldConfiguration.class); + + var savedConfig = (RelatedEntitiesAggregationCalculatedFieldConfiguration) saved.getConfiguration(); + int expectedMinScheduledUpdateInterval = tbTenantProfileCache.get(tenantId) + .getDefaultProfileConfiguration() + .getMinAllowedScheduledUpdateIntervalInSecForCF(); + + assertThat(savedConfig.getScheduledUpdateInterval()).isEqualTo(expectedMinScheduledUpdateInterval); + } + + @Test + public void testSaveRelatedEntitiesAggregationCF_shouldThrowWhenScheduledUpdateIntervalLessThanMinAllowed() { + // GIVEN + var device = createTestDevice(); + + var cfg = new RelatedEntitiesAggregationCalculatedFieldConfiguration(); + cfg.setRelation(new RelationPathLevel(EntitySearchDirection.FROM, EntityRelation.CONTAINS_TYPE)); + + var argument = new Argument(); + argument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + cfg.setArguments(Map.of("temp", argument)); + + var metric = new AggMetric(); + metric.setFunction(AggFunction.AVG); + metric.setInput(new AggKeyInput("temp")); + cfg.setMetrics(Map.of("avgTemp", metric)); + + var output = new TimeSeriesOutput(); + output.setName("avgTemperature"); + cfg.setOutput(output); + + int minDeduplicationInterval = (int) tbTenantProfileCache.get(tenantId) + .getDefaultProfileConfiguration() + .getMinAllowedDeduplicationIntervalInSecForCF(); + cfg.setDeduplicationIntervalInSec(minDeduplicationInterval); + + int minScheduledUpdateInterval = tbTenantProfileCache.get(tenantId) + .getDefaultProfileConfiguration() + .getMinAllowedScheduledUpdateIntervalInSecForCF(); + int invalidInterval = RandomUtils.insecure().randomInt(1, minScheduledUpdateInterval); + cfg.setScheduledUpdateInterval(invalidInterval); + + var cf = new CalculatedField(); + cf.setTenantId(tenantId); + cf.setEntityId(device.getId()); + cf.setType(CalculatedFieldType.RELATED_ENTITIES_AGGREGATION); + cf.setName("Related Entities Aggregation CF - invalid scheduled interval test"); + cf.setConfigurationVersion(0); + cf.setConfiguration(cfg); + + // WHEN-THEN + assertThatThrownBy(() -> calculatedFieldService.save(cf)) + .isInstanceOf(DataValidationException.class) + .hasCauseInstanceOf(IllegalArgumentException.class) + .hasMessage("Scheduled update interval (" + invalidInterval + + " seconds) is less than minimum allowed interval in tenant profile: " + minScheduledUpdateInterval + " seconds"); + } + + @Test + public void testSaveRelatedEntitiesAggregationCF_shouldAcceptValidScheduledUpdateInterval() { + // GIVEN + var device = createTestDevice(); + + var cfg = new RelatedEntitiesAggregationCalculatedFieldConfiguration(); + cfg.setRelation(new RelationPathLevel(EntitySearchDirection.FROM, EntityRelation.CONTAINS_TYPE)); + + var argument = new Argument(); + argument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + cfg.setArguments(Map.of("temp", argument)); + + var metric = new AggMetric(); + metric.setFunction(AggFunction.AVG); + metric.setInput(new AggKeyInput("temp")); + cfg.setMetrics(Map.of("avgTemp", metric)); + + var output = new TimeSeriesOutput(); + output.setName("avgTemperature"); + cfg.setOutput(output); + + int minDeduplicationInterval = (int) tbTenantProfileCache.get(tenantId) + .getDefaultProfileConfiguration() + .getMinAllowedDeduplicationIntervalInSecForCF(); + cfg.setDeduplicationIntervalInSec(minDeduplicationInterval); + + int minScheduledUpdateInterval = tbTenantProfileCache.get(tenantId) + .getDefaultProfileConfiguration() + .getMinAllowedScheduledUpdateIntervalInSecForCF(); + int customScheduledUpdateInterval = minScheduledUpdateInterval + 100; + cfg.setScheduledUpdateInterval(customScheduledUpdateInterval); + + var cf = new CalculatedField(); + cf.setTenantId(tenantId); + cf.setEntityId(device.getId()); + cf.setType(CalculatedFieldType.RELATED_ENTITIES_AGGREGATION); + cf.setName("Related Entities Aggregation CF - valid scheduled interval test"); + cf.setConfigurationVersion(0); + cf.setConfiguration(cfg); + + // WHEN + CalculatedField saved = calculatedFieldService.save(cf); + + // THEN + assertThat(saved).isNotNull(); + assertThat(saved.getConfiguration()).isInstanceOf(RelatedEntitiesAggregationCalculatedFieldConfiguration.class); + + var savedConfig = (RelatedEntitiesAggregationCalculatedFieldConfiguration) saved.getConfiguration(); + assertThat(savedConfig.getScheduledUpdateInterval()).isEqualTo(customScheduledUpdateInterval); + } + private CalculatedField saveValidCalculatedField() { Device device = createTestDevice(); CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java index d3aa0fde0f..373112293d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java @@ -23,7 +23,6 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.awaitility.Awaitility; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.AttributeScope; @@ -31,8 +30,12 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.dao.attributes.AttributesDao; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.service.AbstractServiceTest; @@ -40,6 +43,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.concurrent.Executors; @@ -48,9 +52,6 @@ import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; -/** - * @author Andrew Shvayka - */ @Slf4j public abstract class BaseAttributesServiceTest extends AbstractServiceTest { @@ -60,9 +61,8 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { @Autowired private AttributesService attributesService; - @Before - public void before() { - } + @Autowired + private AttributesDao attributesDao; @Test public void saveAndFetch() throws Exception { @@ -223,7 +223,7 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { saveAttribute(tenantId, deviceId, AttributeScope.SERVER_SCOPE, "key2", "123"); Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { - List keys = attributesService.findAllKeysByEntityIds(tenantId, List.of(deviceId), AttributeScope.SERVER_SCOPE); + List keys = attributesService.findAllKeysByEntityIdsAndScope(tenantId, List.of(deviceId), AttributeScope.SERVER_SCOPE); assertThat(keys).containsOnly("key1", "key2"); }); } @@ -241,6 +241,84 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { }); } + @Test + public void findLatestByEntityIdsAndScope_returnsOneEntryPerKey() { + var device1 = new DeviceId(UUID.randomUUID()); + var device2 = new DeviceId(UUID.randomUUID()); + + // Both devices have "temperature", device2 has a newer ts + saveAttribute(tenantId, device1, AttributeScope.SERVER_SCOPE, 1000, new DoubleDataEntry("temperature", 20.0)); + saveAttribute(tenantId, device2, AttributeScope.SERVER_SCOPE, 2000, new DoubleDataEntry("temperature", 25.0)); + // Only device1 has "humidity" + saveAttribute(tenantId, device1, AttributeScope.SERVER_SCOPE, 1500, new LongDataEntry("humidity", 60L)); + // Only device2 has "active" + saveAttribute(tenantId, device2, AttributeScope.SERVER_SCOPE, 3000, new BooleanDataEntry("active", true)); + + List results = attributesDao.findLatestByEntityIdsAndScope(tenantId, List.of(device1, device2), AttributeScope.SERVER_SCOPE); + Map byKey = results.stream().collect(Collectors.toMap(AttributeKvEntry::getKey, e -> e)); + + Assert.assertEquals(3, byKey.size()); + + // "temperature" should pick device2's value (ts=2000 > ts=1000) + AttributeKvEntry temp = byKey.get("temperature"); + Assert.assertNotNull(temp); + Assert.assertEquals(25.0, temp.getDoubleValue().orElseThrow(), 0.0); + Assert.assertEquals(2000, temp.getLastUpdateTs()); + + // "humidity" — only device1 has it + AttributeKvEntry humidity = byKey.get("humidity"); + Assert.assertNotNull(humidity); + Assert.assertEquals(60L, (long) humidity.getLongValue().orElseThrow()); + Assert.assertEquals(1500, humidity.getLastUpdateTs()); + + // "active" — only device2 has it + AttributeKvEntry active = byKey.get("active"); + Assert.assertNotNull(active); + Assert.assertEquals(true, active.getBooleanValue().orElseThrow()); + Assert.assertEquals(3000, active.getLastUpdateTs()); + } + + @Test + public void findLatestByEntityIdsAndScope_emptyList() { + List results = attributesDao.findLatestByEntityIdsAndScope(tenantId, List.of(), AttributeScope.SERVER_SCOPE); + Assert.assertTrue(results.isEmpty()); + } + + @Test + public void findLatestByEntityIdsAndScope_singleEntity() throws Exception { + var device = new DeviceId(UUID.randomUUID()); + saveAttribute(tenantId, device, AttributeScope.SERVER_SCOPE, 1000, new StringDataEntry("key1", "value1")); + saveAttribute(tenantId, device, AttributeScope.SERVER_SCOPE, 2000, new StringDataEntry("key2", "value2")); + + // sync + List results = attributesDao.findLatestByEntityIdsAndScope(tenantId, List.of(device), AttributeScope.SERVER_SCOPE); + Assert.assertEquals(2, results.size()); + Map byKey = results.stream().collect(Collectors.toMap(AttributeKvEntry::getKey, e -> e)); + Assert.assertEquals("value1", byKey.get("key1").getStrValue().orElseThrow()); + Assert.assertEquals(1000, byKey.get("key1").getLastUpdateTs()); + Assert.assertEquals("value2", byKey.get("key2").getStrValue().orElseThrow()); + Assert.assertEquals(2000, byKey.get("key2").getLastUpdateTs()); + + // async — same result + List asyncResults = attributesDao.findLatestByEntityIdsAndScopeAsync(tenantId, List.of(device), AttributeScope.SERVER_SCOPE).get(); + Assert.assertEquals(results, asyncResults); + } + + @Test + public void findLatestByEntityIdsAndScope_filtersScope() { + var device = new DeviceId(UUID.randomUUID()); + saveAttribute(tenantId, device, AttributeScope.SERVER_SCOPE, 1000, new StringDataEntry("serverKey", "sv")); + saveAttribute(tenantId, device, AttributeScope.CLIENT_SCOPE, 1000, new StringDataEntry("clientKey", "cv")); + + List serverResults = attributesDao.findLatestByEntityIdsAndScope(tenantId, List.of(device), AttributeScope.SERVER_SCOPE); + Assert.assertEquals(1, serverResults.size()); + Assert.assertEquals("serverKey", serverResults.get(0).getKey()); + + List clientResults = attributesDao.findLatestByEntityIdsAndScope(tenantId, List.of(device), AttributeScope.CLIENT_SCOPE); + Assert.assertEquals(1, clientResults.size()); + Assert.assertEquals("clientKey", clientResults.get(0).getKey()); + } + private void testConcurrentFetchAndUpdate(TenantId tenantId, DeviceId deviceId, ListeningExecutorService pool) throws Exception { var scope = AttributeScope.SERVER_SCOPE; var key = "TEST"; @@ -313,6 +391,15 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { } } + private void saveAttribute(TenantId tenantId, DeviceId deviceId, AttributeScope scope, long ts, KvEntry value) { + try { + attributesService.save(tenantId, deviceId, scope, Collections.singletonList(new BaseAttributeKvEntry(ts, value))).get(10, TimeUnit.SECONDS); + } catch (Exception e) { + log.warn("Failed to save attribute", e.getCause()); + throw new RuntimeException(e); + } + } + private void equalsIgnoreVersion(AttributeKvEntry expected, AttributeKvEntry actual) { Assert.assertEquals(expected.getKey(), actual.getKey()); Assert.assertEquals(expected.getValue(), actual.getValue()); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidatorTest.java index 170b452fee..373344cb6d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidatorTest.java @@ -16,29 +16,35 @@ package org.thingsboard.server.dao.service.validator; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.test.context.bean.override.mockito.MockitoBean; +import org.springframework.test.context.bean.override.mockito.MockitoSpyBean; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.dao.tenant.TenantProfileDao; import org.thingsboard.server.dao.tenant.TenantProfileService; +import org.thingsboard.server.exception.DataValidationException; import java.util.UUID; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.verify; @SpringBootTest(classes = TenantProfileDataValidator.class) class TenantProfileDataValidatorTest { - @MockBean + @MockitoBean TenantProfileDao tenantProfileDao; - @MockBean + @MockitoBean TenantProfileService tenantProfileService; - @SpyBean + @MockitoSpyBean TenantProfileDataValidator validator; + TenantId tenantId = TenantId.fromUUID(UUID.fromString("9ef79cdf-37a8-4119-b682-2e7ed4e018da")); @Test @@ -53,4 +59,44 @@ class TenantProfileDataValidatorTest { verify(validator).validateString("Tenant profile name", tenantProfile.getName()); } + @ParameterizedTest + @ValueSource(ints = {-1, -100, Integer.MIN_VALUE}) + void minAllowedScheduledUpdateIntervalInSecForCF_shouldRejectNegativeValues(int value) { + // GIVEN + var config = new DefaultTenantProfileConfiguration(); + config.setMinAllowedScheduledUpdateIntervalInSecForCF(value); + + var tenantProfileData = new TenantProfileData(); + tenantProfileData.setConfiguration(config); + + var tenantProfile = new TenantProfile(); + tenantProfile.setName("Test"); + tenantProfile.setProfileData(tenantProfileData); + + // WHEN/THEN + assertThatThrownBy(() -> validator.validate(tenantProfile, __ -> TenantId.SYS_TENANT_ID)) + .isInstanceOf(DataValidationException.class) + .hasMessageContaining("minAllowedScheduledUpdateIntervalInSecForCF") + .hasMessageContaining("must be greater than or equal to 0"); + } + + @ParameterizedTest + @ValueSource(ints = {0, 1, 60, Integer.MAX_VALUE}) + void minAllowedScheduledUpdateIntervalInSecForCF_shouldAcceptValidValues(int value) { + // GIVEN + var config = new DefaultTenantProfileConfiguration(); + config.setMinAllowedScheduledUpdateIntervalInSecForCF(value); + + var tenantProfileData = new TenantProfileData(); + tenantProfileData.setConfiguration(config); + + var tenantProfile = new TenantProfile(); + tenantProfile.setName("Test"); + tenantProfile.setProfileData(tenantProfileData); + + // WHEN/THEN + assertThatCode(() -> validator.validate(tenantProfile, __ -> TenantId.SYS_TENANT_ID)) + .doesNotThrowAnyException(); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDaoTest.java index 96f3390b13..02abf1e5fd 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDaoTest.java @@ -22,6 +22,9 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.service.AbstractServiceTest; @@ -30,7 +33,9 @@ import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -102,6 +107,69 @@ public class SqlTimeseriesLatestDaoTest extends AbstractServiceTest { } } + @Test + public void findLatestByEntityIds_returnsOneEntryPerKey() throws Exception { + DeviceId device1 = new DeviceId(UUID.randomUUID()); + DeviceId device2 = new DeviceId(UUID.randomUUID()); + + // Both devices have "temperature" key, device2 has a newer ts + timeseriesLatestDao.saveLatest(tenantId, device1, new BasicTsKvEntry(1000, new DoubleDataEntry("temperature", 20.0))).get(); + timeseriesLatestDao.saveLatest(tenantId, device2, new BasicTsKvEntry(2000, new DoubleDataEntry("temperature", 25.0))).get(); + // Only device1 has "humidity" + timeseriesLatestDao.saveLatest(tenantId, device1, new BasicTsKvEntry(1500, new LongDataEntry("humidity", 60L))).get(); + // Only device2 has "active" + timeseriesLatestDao.saveLatest(tenantId, device2, new BasicTsKvEntry(3000, new BooleanDataEntry("active", true))).get(); + + List results = timeseriesLatestDao.findLatestByEntityIds(tenantId, List.of(device1, device2)); + Map byKey = results.stream().collect(Collectors.toMap(TsKvEntry::getKey, e -> e)); + + assertEquals(3, byKey.size()); + + // "temperature" should pick device2's value (ts=2000 > ts=1000) + TsKvEntry temp = byKey.get("temperature"); + assertNotNull(temp); + assertEquals(25.0, temp.getDoubleValue().orElseThrow()); + assertEquals(2000, temp.getTs()); + + // "humidity" — only device1 has it + TsKvEntry humidity = byKey.get("humidity"); + assertNotNull(humidity); + assertEquals(60L, humidity.getLongValue().orElseThrow()); + assertEquals(1500, humidity.getTs()); + + // "active" — only device2 has it + TsKvEntry active = byKey.get("active"); + assertNotNull(active); + assertEquals(true, active.getBooleanValue().orElseThrow()); + assertEquals(3000, active.getTs()); + } + + @Test + public void findLatestByEntityIds_emptyList() { + List results = timeseriesLatestDao.findLatestByEntityIds(tenantId, List.of()); + assertTrue(results.isEmpty()); + } + + @Test + public void findLatestByEntityIds_singleEntity() throws Exception { + DeviceId device = new DeviceId(UUID.randomUUID()); + timeseriesLatestDao.saveLatest(tenantId, device, new BasicTsKvEntry(1000, new StringDataEntry("key1", "value1"))).get(); + timeseriesLatestDao.saveLatest(tenantId, device, new BasicTsKvEntry(2000, new StringDataEntry("key2", "value2"))).get(); + + // sync + List results = timeseriesLatestDao.findLatestByEntityIds(tenantId, List.of(device)); + assertEquals(2, results.size()); + Map byKey = results.stream().collect(Collectors.toMap(TsKvEntry::getKey, e -> e)); + assertEquals("value1", byKey.get("key1").getStrValue().orElseThrow()); + assertEquals(1000, byKey.get("key1").getTs()); + assertEquals("value2", byKey.get("key2").getStrValue().orElseThrow()); + assertEquals(2000, byKey.get("key2").getTs()); + + // async — same result + List asyncResults = timeseriesLatestDao.findLatestByEntityIdsAsync(tenantId, List.of(device)).get(); + assertEquals(results, asyncResults); + } + private TsKvEntry createEntry(String key, long ts) { return new BasicTsKvEntry(ts, new StringDataEntry(key, RandomStringUtils.random(10))); } diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index a2a6ca6814..010994dc88 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -210,7 +210,7 @@ management: web: exposure: # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). - include: '${METRICS_ENDPOINTS_EXPOSE:info}' + include: "${METRICS_ENDPOINTS_EXPOSE:info}" health: elasticsearch: # Enable the org.springframework.boot.actuate.elasticsearch.ElasticsearchRestClientHealthIndicator.doHealthCheck diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 4f5dc7ef63..56e15d4b4d 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -215,6 +215,8 @@ usage: enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}" # Interval of reporting the statistics. By default, the summarized statistics are sent every 10 seconds interval: "${USAGE_STATS_REPORT_INTERVAL:60}" + # Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates + urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}" # Amount of statistic messages in pack pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}" @@ -232,7 +234,7 @@ management: web: exposure: # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). - include: '${METRICS_ENDPOINTS_EXPOSE:info}' + include: "${METRICS_ENDPOINTS_EXPOSE:info}" # Service common properties service: diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 6c97a23a90..f133527ddc 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -94,6 +94,8 @@ import org.thingsboard.server.common.data.asset.AssetSearchQuery; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.audit.AuditLog; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldInfo; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.device.DeviceSearchQuery; import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.domain.DomainInfo; @@ -169,6 +171,7 @@ import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; import org.thingsboard.server.common.data.query.AvailableEntityKeys; +import org.thingsboard.server.common.data.query.AvailableEntityKeysV2; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; @@ -1895,6 +1898,10 @@ public class RestClient implements Closeable { }).getBody(); } + /** + * @deprecated Use {@link #findAvailableEntityKeysV2(EntityDataQuery, boolean, boolean, Set, boolean)} instead. + */ + @Deprecated(forRemoval = true) public AvailableEntityKeys findAvailableEntityKeysByQuery(EntityDataQuery query, boolean includeTimeseries, boolean includeAttributes, AttributeScope scope) { var uri = UriComponentsBuilder.fromUriString(baseURL) .path("/api/entitiesQuery/find/keys") @@ -1906,6 +1913,22 @@ public class RestClient implements Closeable { return restTemplate.exchange(uri, HttpMethod.POST, new HttpEntity<>(query), new ParameterizedTypeReference() {}).getBody(); } + @SneakyThrows(URISyntaxException.class) + public AvailableEntityKeysV2 findAvailableEntityKeysV2( + EntityDataQuery query, boolean includeTimeseries, boolean includeAttributes, Set scopes, boolean includeSamples + ) { + var builder = new URIBuilder(baseURL).appendPath("/api/v2/entitiesQuery/find/keys") + .addParameter("includeTimeseries", String.valueOf(includeTimeseries)) + .addParameter("includeAttributes", String.valueOf(includeAttributes)) + .addParameter("includeSamples", String.valueOf(includeSamples)); + if (scopes != null) { + for (AttributeScope scope : scopes) { + builder.addParameter("scopes", scope.name()); + } + } + return restTemplate.exchange(builder.build(), HttpMethod.POST, new HttpEntity<>(query), new ParameterizedTypeReference() {}).getBody(); + } + public PageData findAlarmDataByQuery(AlarmDataQuery query) { return restTemplate.exchange( baseURL + "/api/alarmsQuery/find", @@ -4368,6 +4391,46 @@ public class RestClient implements Closeable { } + @SneakyThrows(URISyntaxException.class) + public PageData getCalculatedFields(PageLink pageLink, + Set types, + EntityType entityType, + Set entities, + Set names) { + var urlBuilder = new URIBuilder(baseURL).appendPath("/api/calculatedFields"); + urlBuilder.addParameter("pageSize", String.valueOf(pageLink.getPageSize())); + urlBuilder.addParameter("page", String.valueOf(pageLink.getPage())); + if (!isEmpty(pageLink.getTextSearch())) { + urlBuilder.addParameter("textSearch", pageLink.getTextSearch()); + } + if (pageLink.getSortOrder() != null) { + urlBuilder.addParameter("sortProperty", pageLink.getSortOrder().getProperty()); + urlBuilder.addParameter("sortOrder", pageLink.getSortOrder().getDirection().name()); + } + if (!CollectionUtils.isEmpty(types)) { + for (CalculatedFieldType type : types) { + urlBuilder.addParameter("types", type.name()); + } + } + if (entityType != null) { + urlBuilder.addParameter("entityType", entityType.name()); + } + if (!CollectionUtils.isEmpty(entities)) { + for (UUID entity : entities) { + urlBuilder.addParameter("entities", entity.toString()); + } + } + if (!CollectionUtils.isEmpty(names)) { + for (String name : names) { + urlBuilder.addParameter("name", name); + } + } + return restTemplate.exchange( + urlBuilder.build(), + HttpMethod.GET, HttpEntity.EMPTY, + new ParameterizedTypeReference>() {}).getBody(); + } + public void deleteCalculatedField(CalculatedFieldId calculatedFieldId) { restTemplate.delete(baseURL + "/api/calculatedField/{calculatedFieldId}", calculatedFieldId.getId()); } diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index dde415bd91..376075a0a4 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -421,6 +421,8 @@ usage: enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}" # Interval of reporting the statistics. By default, the summarized statistics are sent every 10 seconds interval: "${USAGE_STATS_REPORT_INTERVAL:60}" + # Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates + urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}" # Amount of statistic messages in pack pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}" @@ -435,7 +437,7 @@ management: web: exposure: # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). - include: '${METRICS_ENDPOINTS_EXPOSE:info}' + include: "${METRICS_ENDPOINTS_EXPOSE:info}" # Notification system parameters notification_system: diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 39f7d15fb9..1f6a251324 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -370,6 +370,8 @@ usage: enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}" # Interval of reporting the statistics. By default, the summarized statistics are sent every 10 seconds interval: "${USAGE_STATS_REPORT_INTERVAL:60}" + # Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates + urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}" # Amount of statistic messages in pack pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}" @@ -384,7 +386,7 @@ management: web: exposure: # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). - include: '${METRICS_ENDPOINTS_EXPOSE:info}' + include: "${METRICS_ENDPOINTS_EXPOSE:info}" # Notification system parameters notification_system: diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index 21687bb697..b4d8d6c7eb 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -471,6 +471,8 @@ usage: enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}" # Interval of reporting the statistics. By default, the summarized statistics are sent every 10 seconds interval: "${USAGE_STATS_REPORT_INTERVAL:60}" + # Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates + urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}" # Amount of statistic messages in pack pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}" @@ -485,7 +487,7 @@ management: web: exposure: # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). - include: '${METRICS_ENDPOINTS_EXPOSE:info}' + include: "${METRICS_ENDPOINTS_EXPOSE:info}" # Notification system parameters notification_system: diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 4a7571a734..0b96c57c13 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -404,6 +404,8 @@ usage: enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}" # Interval of reporting the statistics. By default, the summarized statistics are sent every 10 seconds interval: "${USAGE_STATS_REPORT_INTERVAL:60}" + # Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates + urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}" # Amount of statistic messages in pack pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}" @@ -418,7 +420,7 @@ management: web: exposure: # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). - include: '${METRICS_ENDPOINTS_EXPOSE:info}' + include: "${METRICS_ENDPOINTS_EXPOSE:info}" # Notification system parameters notification_system: diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index b98ad2a9e9..71524ba47e 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -359,6 +359,8 @@ usage: enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}" # Interval of reporting the statistics. By default, the summarized statistics are sent every 10 seconds interval: "${USAGE_STATS_REPORT_INTERVAL:60}" + # Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates + urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}" # Amount of statistic messages in pack pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}" @@ -373,7 +375,7 @@ management: web: exposure: # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). - include: '${METRICS_ENDPOINTS_EXPOSE:info}' + include: "${METRICS_ENDPOINTS_EXPOSE:info}" # Notification system parameters notification_system: diff --git a/ui-ngx/patches/@iplab+ngx-color-picker+20.0.0.patch b/ui-ngx/patches/@iplab+ngx-color-picker+20.0.0.patch index 211a3c8b0b..b0087a67e9 100644 --- a/ui-ngx/patches/@iplab+ngx-color-picker+20.0.0.patch +++ b/ui-ngx/patches/@iplab+ngx-color-picker+20.0.0.patch @@ -1,46 +1,13 @@ diff --git a/node_modules/@iplab/ngx-color-picker/fesm2022/iplab-ngx-color-picker.mjs b/node_modules/@iplab/ngx-color-picker/fesm2022/iplab-ngx-color-picker.mjs -index a372799..a3d709a 100644 +index a372799..f64a6f8 100644 --- a/node_modules/@iplab/ngx-color-picker/fesm2022/iplab-ngx-color-picker.mjs +++ b/node_modules/@iplab/ngx-color-picker/fesm2022/iplab-ngx-color-picker.mjs -@@ -1129,11 +1129,11 @@ class RgbaComponent { - this.color.set(newColor); +@@ -516,7 +516,7 @@ class Color { + const s = (color.saturation / 100) * (l <= 1 ? l : 2 - l); + const value = (l + s) / 2; + const saturation = (2 * s) / (l + s) || 0; +- return new Hsva(hue, saturation, value, color.alpha); ++ return new Hsva(hue, saturation * 100, value * 100, color.alpha); } - static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "20.0.4", ngImport: i0, type: RgbaComponent, deps: [], target: i0.ɵɵFactoryTarget.Component }); } -- static { this.ɵcmp = i0.ɵɵngDeclareComponent({ minVersion: "17.0.0", version: "20.0.4", type: RgbaComponent, isStandalone: true, selector: "rgba-input-component", inputs: { color: { classPropertyName: "color", publicName: "color", isSignal: true, isRequired: true, transformFunction: null }, labelVisible: { classPropertyName: "labelVisible", publicName: "label", isSignal: true, isRequired: false, transformFunction: null }, isAlphaVisible: { classPropertyName: "isAlphaVisible", publicName: "alpha", isSignal: true, isRequired: false, transformFunction: null } }, outputs: { color: "colorChange" }, ngImport: i0, template: "
\r\n \r\n @if (labelVisible()) {\r\n R\r\n }\r\n
\r\n
\r\n \r\n @if (labelVisible()) {\r\n G\r\n }\r\n
\r\n
\r\n \r\n @if (labelVisible()) {\r\n B\r\n }\r\n
\r\n@if (isAlphaVisible()) {\r\n
\r\n \r\n @if (labelVisible()) {\r\n A\r\n }\r\n
\r\n}", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n", ""], dependencies: [{ kind: "directive", type: ColorPickerInputDirective, selector: "[inputChange]", inputs: ["min", "max"], outputs: ["inputChange"] }], changeDetection: i0.ChangeDetectionStrategy.OnPush }); } -+ static { this.ɵcmp = i0.ɵɵngDeclareComponent({ minVersion: "17.0.0", version: "20.0.4", type: RgbaComponent, isStandalone: true, selector: "rgba-input-component", inputs: { color: { classPropertyName: "color", publicName: "color", isSignal: true, isRequired: true, transformFunction: null }, labelVisible: { classPropertyName: "labelVisible", publicName: "label", isSignal: true, isRequired: false, transformFunction: null }, isAlphaVisible: { classPropertyName: "isAlphaVisible", publicName: "alpha", isSignal: true, isRequired: false, transformFunction: null } }, outputs: { color: "colorChange" }, ngImport: i0, template: "
\r\n \r\n @if (labelVisible()) {\r\n R\r\n }\r\n
\r\n
\r\n \r\n @if (labelVisible()) {\r\n G\r\n }\r\n
\r\n
\r\n \r\n @if (labelVisible()) {\r\n B\r\n }\r\n
\r\n@if (isAlphaVisible()) {\r\n
\r\n \r\n @if (labelVisible()) {\r\n A\r\n }\r\n
\r\n}", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n", ""], dependencies: [{ kind: "directive", type: ColorPickerInputDirective, selector: "[inputChange]", inputs: ["min", "max"], outputs: ["inputChange"] }], changeDetection: i0.ChangeDetectionStrategy.OnPush }); } - } - i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "20.0.4", ngImport: i0, type: RgbaComponent, decorators: [{ - type: Component, -- args: [{ selector: `rgba-input-component`, changeDetection: ChangeDetectionStrategy.OnPush, imports: [ColorPickerInputDirective], template: "
\r\n \r\n @if (labelVisible()) {\r\n R\r\n }\r\n
\r\n
\r\n \r\n @if (labelVisible()) {\r\n G\r\n }\r\n
\r\n
\r\n \r\n @if (labelVisible()) {\r\n B\r\n }\r\n
\r\n@if (isAlphaVisible()) {\r\n
\r\n \r\n @if (labelVisible()) {\r\n A\r\n }\r\n
\r\n}", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n"] }] -+ args: [{ selector: `rgba-input-component`, changeDetection: ChangeDetectionStrategy.OnPush, imports: [ColorPickerInputDirective], template: "
\r\n \r\n @if (labelVisible()) {\r\n R\r\n }\r\n
\r\n
\r\n \r\n @if (labelVisible()) {\r\n G\r\n }\r\n
\r\n
\r\n \r\n @if (labelVisible()) {\r\n B\r\n }\r\n
\r\n@if (isAlphaVisible()) {\r\n
\r\n \r\n @if (labelVisible()) {\r\n A\r\n }\r\n
\r\n}", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n"] }] - }] }); - - class HslaComponent { -@@ -1155,11 +1155,11 @@ class HslaComponent { - this.color.set(newColor); - } - static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "20.0.4", ngImport: i0, type: HslaComponent, deps: [], target: i0.ɵɵFactoryTarget.Component }); } -- static { this.ɵcmp = i0.ɵɵngDeclareComponent({ minVersion: "17.0.0", version: "20.0.4", type: HslaComponent, isStandalone: true, selector: "hsla-input-component", inputs: { color: { classPropertyName: "color", publicName: "color", isSignal: true, isRequired: true, transformFunction: null }, labelVisible: { classPropertyName: "labelVisible", publicName: "label", isSignal: true, isRequired: false, transformFunction: null }, isAlphaVisible: { classPropertyName: "isAlphaVisible", publicName: "alpha", isSignal: true, isRequired: false, transformFunction: null } }, outputs: { color: "colorChange" }, ngImport: i0, template: "
\r\n \r\n @if (labelVisible()) {\r\n H\r\n }\r\n
\r\n
\r\n \r\n @if (labelVisible()) {\r\n S\r\n }\r\n
\r\n
\r\n \r\n @if (labelVisible()) {\r\n L\r\n }\r\n
\r\n@if (isAlphaVisible()) {\r\n
\r\n \r\n @if (labelVisible()) {\r\n A\r\n }\r\n
\r\n}", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n", ""], dependencies: [{ kind: "directive", type: ColorPickerInputDirective, selector: "[inputChange]", inputs: ["min", "max"], outputs: ["inputChange"] }], changeDetection: i0.ChangeDetectionStrategy.OnPush }); } -+ static { this.ɵcmp = i0.ɵɵngDeclareComponent({ minVersion: "17.0.0", version: "20.0.4", type: HslaComponent, isStandalone: true, selector: "hsla-input-component", inputs: { color: { classPropertyName: "color", publicName: "color", isSignal: true, isRequired: true, transformFunction: null }, labelVisible: { classPropertyName: "labelVisible", publicName: "label", isSignal: true, isRequired: false, transformFunction: null }, isAlphaVisible: { classPropertyName: "isAlphaVisible", publicName: "alpha", isSignal: true, isRequired: false, transformFunction: null } }, outputs: { color: "colorChange" }, ngImport: i0, template: "
\r\n \r\n @if (labelVisible()) {\r\n H\r\n }\r\n
\r\n
\r\n \r\n @if (labelVisible()) {\r\n S\r\n }\r\n
\r\n
\r\n \r\n @if (labelVisible()) {\r\n L\r\n }\r\n
\r\n@if (isAlphaVisible()) {\r\n
\r\n \r\n @if (labelVisible()) {\r\n A\r\n }\r\n
\r\n}", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n", ""], dependencies: [{ kind: "directive", type: ColorPickerInputDirective, selector: "[inputChange]", inputs: ["min", "max"], outputs: ["inputChange"] }], changeDetection: i0.ChangeDetectionStrategy.OnPush }); } - } - i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "20.0.4", ngImport: i0, type: HslaComponent, decorators: [{ - type: Component, -- args: [{ selector: `hsla-input-component`, changeDetection: ChangeDetectionStrategy.OnPush, imports: [ColorPickerInputDirective], template: "
\r\n \r\n @if (labelVisible()) {\r\n H\r\n }\r\n
\r\n
\r\n \r\n @if (labelVisible()) {\r\n S\r\n }\r\n
\r\n
\r\n \r\n @if (labelVisible()) {\r\n L\r\n }\r\n
\r\n@if (isAlphaVisible()) {\r\n
\r\n \r\n @if (labelVisible()) {\r\n A\r\n }\r\n
\r\n}", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n"] }] -+ args: [{ selector: `hsla-input-component`, changeDetection: ChangeDetectionStrategy.OnPush, imports: [ColorPickerInputDirective], template: "
\r\n \r\n @if (labelVisible()) {\r\n H\r\n }\r\n
\r\n
\r\n \r\n @if (labelVisible()) {\r\n S\r\n }\r\n
\r\n
\r\n \r\n @if (labelVisible()) {\r\n L\r\n }\r\n
\r\n@if (isAlphaVisible()) {\r\n
\r\n \r\n @if (labelVisible()) {\r\n A\r\n }\r\n
\r\n}", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n"] }] - }] }); - - class HexComponent { -@@ -1190,11 +1190,11 @@ class HexComponent { - } - } - static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "20.0.4", ngImport: i0, type: HexComponent, deps: [], target: i0.ɵɵFactoryTarget.Component }); } -- static { this.ɵcmp = i0.ɵɵngDeclareComponent({ minVersion: "17.0.0", version: "20.0.4", type: HexComponent, isStandalone: true, selector: "hex-input-component", inputs: { color: { classPropertyName: "color", publicName: "color", isSignal: true, isRequired: true, transformFunction: null }, labelVisible: { classPropertyName: "labelVisible", publicName: "label", isSignal: true, isRequired: false, transformFunction: null }, prefixValue: { classPropertyName: "prefixValue", publicName: "prefix", isSignal: true, isRequired: false, transformFunction: null } }, outputs: { color: "colorChange" }, ngImport: i0, template: "
\r\n \r\n @if (labelVisible()) {\r\n HEX\r\n }\r\n
", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n", ""], changeDetection: i0.ChangeDetectionStrategy.OnPush }); } -+ static { this.ɵcmp = i0.ɵɵngDeclareComponent({ minVersion: "17.0.0", version: "20.0.4", type: HexComponent, isStandalone: true, selector: "hex-input-component", inputs: { color: { classPropertyName: "color", publicName: "color", isSignal: true, isRequired: true, transformFunction: null }, labelVisible: { classPropertyName: "labelVisible", publicName: "label", isSignal: true, isRequired: false, transformFunction: null }, prefixValue: { classPropertyName: "prefixValue", publicName: "prefix", isSignal: true, isRequired: false, transformFunction: null } }, outputs: { color: "colorChange" }, ngImport: i0, template: "
\r\n \r\n @if (labelVisible()) {\r\n HEX\r\n }\r\n
", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n", ""], changeDetection: i0.ChangeDetectionStrategy.OnPush }); } - } - i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "20.0.4", ngImport: i0, type: HexComponent, decorators: [{ - type: Component, -- args: [{ selector: `hex-input-component`, changeDetection: ChangeDetectionStrategy.OnPush, standalone: true, template: "
\r\n \r\n @if (labelVisible()) {\r\n HEX\r\n }\r\n
", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n"] }] -+ args: [{ selector: `hex-input-component`, changeDetection: ChangeDetectionStrategy.OnPush, standalone: true, template: "
\r\n \r\n @if (labelVisible()) {\r\n HEX\r\n }\r\n
", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n"] }] - }] }); - - const OpacityAnimation = trigger('opacityAnimation', [ + rgbaToHsva(color) { + const red = color.red / 255; diff --git a/ui-ngx/patches/ngx-hm-carousel+19.0.0.patch b/ui-ngx/patches/ngx-hm-carousel+19.0.0.patch new file mode 100644 index 0000000000..0dfee707e1 --- /dev/null +++ b/ui-ngx/patches/ngx-hm-carousel+19.0.0.patch @@ -0,0 +1,31 @@ +diff --git a/node_modules/ngx-hm-carousel/fesm2022/ngx-hm-carousel.mjs b/node_modules/ngx-hm-carousel/fesm2022/ngx-hm-carousel.mjs +index 117f782..70e49a7 100644 +--- a/node_modules/ngx-hm-carousel/fesm2022/ngx-hm-carousel.mjs ++++ b/node_modules/ngx-hm-carousel/fesm2022/ngx-hm-carousel.mjs +@@ -1,5 +1,5 @@ + import * as i0 from '@angular/core'; +-import { Directive, inject, ViewContainerRef, TemplateRef, input, PLATFORM_ID, DestroyRef, Renderer2, NgZone, ChangeDetectorRef, viewChild, ElementRef, contentChildren, contentChild, computed, signal, effect, forwardRef, Component, ChangeDetectionStrategy } from '@angular/core'; ++import { Directive, inject, ViewContainerRef, TemplateRef, input, PLATFORM_ID, DestroyRef, Renderer2, NgZone, ChangeDetectorRef, viewChild, ElementRef, contentChildren, contentChild, computed, signal, effect, forwardRef, Component, ChangeDetectionStrategy, afterNextRender } from '@angular/core'; + import { DOCUMENT, isPlatformBrowser, NgTemplateOutlet, AsyncPipe } from '@angular/common'; + import { toObservable, takeUntilDestroyed } from '@angular/core/rxjs-interop'; + import { NG_VALUE_ACCESSOR } from '@angular/forms'; +@@ -340,7 +340,7 @@ class NgxHmCarouselComponent { + }); + }); + }); +- const effectRef = effect(() => { ++ afterNextRender(() => { + this.rootElm = this.container().nativeElement; + this.containerElm = this.rootElm.children[0]; + this.init(); +@@ -365,10 +365,6 @@ class NgxHmCarouselComponent { + ]) + .pipe(takeUntilDestroyed(this._destroyRef)) + .subscribe(); +- // only exec once +- effectRef.destroy(); +- }, { +- allowSignalWrites: true, + }); + } + ngOnDestroy() { diff --git a/ui-ngx/src/app/modules/home/components/alarm-rules/alarm-rules-table-config.ts b/ui-ngx/src/app/modules/home/components/alarm-rules/alarm-rules-table-config.ts index cadac572c6..039ee7d5fd 100644 --- a/ui-ngx/src/app/modules/home/components/alarm-rules/alarm-rules-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/alarm-rules/alarm-rules-table-config.ts @@ -106,6 +106,8 @@ export class AlarmRulesTableConfig extends EntityTableConfig { this.entityType = EntityType.API_KEY; this.detailsPanelEnabled = false; - this.addAsTextButton = true; this.pageMode = false; this.entityTranslations = entityTypeTranslations.get(EntityType.API_KEY); diff --git a/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.html b/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.html index 6a797a37aa..c9247f5549 100644 --- a/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.html +++ b/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.html @@ -142,7 +142,7 @@ diff --git a/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.ts b/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.ts index 9187b69b73..b26eee40cf 100644 --- a/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.ts @@ -40,7 +40,7 @@ import { MatDialog } from '@angular/material/dialog'; import { DialogService } from '@core/services/dialog.service'; import { Direction, SortOrder } from '@shared/models/page/sort-order'; import { forkJoin, merge, Observable, Subject } from 'rxjs'; -import { debounceTime, distinctUntilChanged, takeUntil } from 'rxjs/operators'; +import { debounceTime, distinctUntilChanged, take, takeUntil } from 'rxjs/operators'; import { EntityId } from '@shared/models/id/entity-id'; import { AttributeData, @@ -187,6 +187,7 @@ export class AttributeTableComponent extends PageComponent implements AfterViewI textSearch = this.fb.control('', {nonNullable: true}); private destroy$ = new Subject(); + selectAllModel: boolean = false; constructor(protected store: Store, private attributeService: AttributeService, @@ -223,6 +224,14 @@ export class AttributeTableComponent extends PageComponent implements AfterViewI }); }); this.widgetResize$.observe(this.elementRef.nativeElement); + + this.dataSource.selection.changed.pipe( + takeUntil(this.destroy$) + ).subscribe(() => { + this.dataSource.isAllSelected().pipe(take(1)).subscribe(allSelected => { + this.selectAllModel = allSelected; + }); + }); } ngOnDestroy() { @@ -237,6 +246,7 @@ export class AttributeTableComponent extends PageComponent implements AfterViewI this.attributeScope = attributeScope; this.mode = 'default'; this.paginator.pageIndex = 0; + this.selectAllModel = false; this.updateData(true); } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index efb767f92e..aac49a3f37 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -110,6 +110,8 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig -
- - - - - - - - + - - -
@@ -118,6 +94,72 @@ matTooltipPosition="above"> search +
+ + + + + + + + + + + + + + + + +
+ + + diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.scss b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.scss index 38ab6294ea..3a1e3f6a1b 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.scss +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.scss @@ -43,7 +43,7 @@ } } - > .mat-expansion-panel-content { + .mat-expansion-panel-content { > .mat-expansion-panel-body { padding: 0; } diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index cc3fb23da5..829c1414db 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -32,7 +32,8 @@ -
+
-
+
diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/bar-chart-with-labels-basic-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/bar-chart-with-labels-basic-config.component.ts index 9204a901b2..d569bcf37b 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/bar-chart-with-labels-basic-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/bar-chart-with-labels-basic-config.component.ts @@ -45,7 +45,10 @@ import { barChartWithLabelsDefaultSettings, BarChartWithLabelsWidgetSettings } from '@home/components/widget/lib/chart/bar-chart-with-labels-widget.models'; -import { TimeSeriesChartType } from '@home/components/widget/lib/chart/time-series-chart.models'; +import { + TimeSeriesChartType, + updateLatestDataKeys +} from '@home/components/widget/lib/chart/time-series-chart.models'; import { getSourceTbUnitSymbol } from '@shared/models/unit.models'; @Component({ @@ -76,7 +79,7 @@ export class BarChartWithLabelsBasicConfigComponent extends BasicWidgetConfigCom tooltipDatePreviewFn = this._tooltipDatePreviewFn.bind(this); predefinedValues = widgetTitleAutocompleteValues; - + constructor(protected store: Store, protected widgetConfigComponent: WidgetConfigComponent, private $injector: Injector, @@ -167,6 +170,11 @@ export class BarChartWithLabelsBasicConfigComponent extends BasicWidgetConfigCom }); } + protected onConfigChanged(widgetConfig: WidgetConfigComponentData) { + updateLatestDataKeys([widgetConfig.config.settings.yAxis], this.datasource, this.callbacks); + super.onConfigChanged(widgetConfig); + } + protected prepareOutputConfig(config: any): WidgetConfigComponentData { setTimewindowConfig(this.widgetConfig.config, config.timewindowConfig); this.widgetConfig.config.datasources = config.datasources; diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/range-chart-basic-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/range-chart-basic-config.component.ts index 0527045650..95dbabfbfc 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/range-chart-basic-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/range-chart-basic-config.component.ts @@ -47,7 +47,7 @@ import { } from '@home/components/widget/lib/chart/range-chart-widget.models'; import { lineSeriesStepTypes, - lineSeriesStepTypeTranslations + lineSeriesStepTypeTranslations, updateLatestDataKeys } from '@home/components/widget/lib/chart/time-series-chart.models'; import { chartLabelPositions, @@ -289,6 +289,11 @@ export class RangeChartBasicConfigComponent extends BasicWidgetConfigComponent { return this.widgetConfig; } + protected onConfigChanged(widgetConfig: WidgetConfigComponentData) { + updateLatestDataKeys([widgetConfig.config.settings.yAxis], this.datasource, this.callbacks); + super.onConfigChanged(widgetConfig); + } + protected validatorTriggers(): string[] { return ['showTitle', 'showIcon', 'showRangeThresholds', 'fillArea', 'showLine', 'step', 'showPointLabel', 'enablePointLabelBackground', 'showLegend', 'showTooltip', 'tooltipShowDate']; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.ts index 1f63cd8638..0287b775c3 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.ts @@ -134,6 +134,12 @@ export class BarChartWithLabelsWidgetComponent implements OnInit, OnDestroy, Aft } } + public onLatestDataUpdated() { + if (this.timeSeriesChart) { + this.timeSeriesChart.latestUpdated(); + } + } + public onLegendKeyEnter(key: DataKey) { this.timeSeriesChart.keyEnter(key); } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.component.ts index 2412d51a83..4ad45c85a2 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.component.ts @@ -162,6 +162,12 @@ export class RangeChartWidgetComponent implements OnInit, OnDestroy, AfterViewIn } } + public onLatestDataUpdated() { + if (this.timeSeriesChart) { + this.timeSeriesChart.latestUpdated(); + } + } + public toggleRangeItem(item: RangeItem) { item.enabled = !item.enabled; this.timeSeriesChart.toggleVisualMapRange(item.index); diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.models.ts index 4816852768..e815eefe11 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.models.ts @@ -99,6 +99,8 @@ import { TimeSeriesChartTooltipWidgetSettings } from '@home/components/widget/lib/chart/time-series-chart-tooltip.models'; import { TbUnit, TbUnitConverter } from '@shared/models/unit.models'; +import { DataKeyType } from '@shared/models/telemetry/telemetry.models'; +import { DataKeysCallbacks } from '@home/components/widget/lib/settings/common/key/data-keys.component.models'; type TimeSeriesChartDataEntry = [number, any, number, number]; @@ -1495,3 +1497,101 @@ const createSeriesLabelOption = (item: TimeSeriesChartDataItem, show: boolean, } return labelOption; }; + +export const checkLatestDataKeys = (yAxes: TimeSeriesChartYAxes, datasource: Datasource): TimeSeriesChartYAxes => { + const latestKeys = datasource?.latestDataKeys || []; + const result: TimeSeriesChartYAxes = {}; + + for (const [id, axis] of Object.entries(yAxes)) { + axis.min = normalizeAxisLimit(axis.min); + axis.max = normalizeAxisLimit(axis.max); + const minCfg = axis.min; + const maxCfg = axis.max; + + const minValid = !!minCfg && ( + minCfg.type !== ValueSourceType.latestKey || + latestKeys.some(k => isYAxisKey(k, minCfg)) + ); + + const maxValid = !!maxCfg && ( + maxCfg.type !== ValueSourceType.latestKey || + latestKeys.some(k => isYAxisKey(k, maxCfg)) + ); + + if (minValid && maxValid) { + result[id] = axis; + } + } + + return result; +} + +export const updateLatestDataKeys = (yAxes: TimeSeriesChartYAxisSettings[], datasource: Datasource, dataKeyCallbacks: DataKeysCallbacks)=> { + if (datasource) { + let latestKeys = datasource.latestDataKeys; + if (!latestKeys) { + latestKeys = []; + datasource.latestDataKeys = latestKeys; + } + const existingYAxisKeys = latestKeys.filter(k => k.settings?.__yAxisMinKey === true || k.settings?.__yAxisMaxKey === true); + const foundYAxisKeys: DataKey[] = []; + + for(const yAxis of yAxes) { + const min = yAxis.min as ValueSourceConfig; + const max = yAxis.max as ValueSourceConfig; + if (min && min.type === ValueSourceType.latestKey) { + const found = existingYAxisKeys.find(k => isYAxisKey(k, min)); + if (!found) { + const newKey = dataKeyCallbacks.generateDataKey(min.latestKey, min.latestKeyType, + null, true, null); + newKey.settings.__yAxisMinKey = true; + latestKeys.push(newKey); + } else if (foundYAxisKeys.indexOf(found) === -1) { + foundYAxisKeys.push(found); + } + } + if (max && max.type === ValueSourceType.latestKey) { + const found = existingYAxisKeys.find(k => isYAxisKey(k, max)); + if (!found) { + const newKey = dataKeyCallbacks.generateDataKey(max.latestKey, max.latestKeyType, + null, true, null); + newKey.settings.__yAxisMaxKey = true; + latestKeys.push(newKey); + } else if (foundYAxisKeys.indexOf(found) === -1) { + foundYAxisKeys.push(found); + } + } + } + const toRemove = existingYAxisKeys.filter(k => foundYAxisKeys.indexOf(k) === -1); + for (const key of toRemove) { + const index = latestKeys.indexOf(key); + if (index > -1) { + latestKeys.splice(index, 1); + } + } + } +} + +export const isYAxisKey = (d: DataKey, limit: ValueSourceConfig): boolean => { + return (d.type === DataKeyType.function && d.label === limit.latestKey) || + (d.type !== DataKeyType.function && d.name === limit.latestKey && + d.type === limit.latestKeyType); +} + +export const normalizeAxisLimit = (limit: string | number | ValueSourceConfig): ValueSourceConfig => { + if (!limit) { + return { + type: ValueSourceType.constant, + value: null, + entityAlias: null + }; + } else if (typeof limit === 'number' || typeof limit === 'string') { + return { + type: ValueSourceType.constant, + value: Number(limit), + entityAlias: null + }; + } + return limit; +} + diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.ts index 21f5f2d68c..d1928d75f0 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.ts @@ -23,7 +23,7 @@ import { createTimeSeriesYAxis, defaultTimeSeriesChartYAxisSettings, generateChartData, - LineSeriesStepType, + LineSeriesStepType, normalizeAxisLimit, parseThresholdData, TimeSeriesChartAxis, TimeSeriesChartDataItem, @@ -581,8 +581,8 @@ export class TbTimeSeriesChart { yAxisSettingsList.sort((a1, a2) => a1.order - a2.order); const axisLimitDatasources: Datasource[] = []; for (const yAxisSettings of yAxisSettingsList) { - yAxisSettings.min = this.normalizeAxisLimit(yAxisSettings.min); - yAxisSettings.max = this.normalizeAxisLimit(yAxisSettings.max); + yAxisSettings.min = normalizeAxisLimit(yAxisSettings.min); + yAxisSettings.max = normalizeAxisLimit(yAxisSettings.max); const axisSettings = mergeDeep({} as TimeSeriesChartYAxisSettings, defaultTimeSeriesChartYAxisSettings, yAxisSettings); const units = isNotEmptyTbUnits(axisSettings.units) ? axisSettings.units : this.ctx.units; @@ -1080,21 +1080,4 @@ export class TbTimeSeriesChart { this.timeSeriesChart.setOption(this.timeSeriesChartOptions); } } - - private normalizeAxisLimit(limit: string | number | ValueSourceConfig): string | number | ValueSourceConfig { - if (!limit) { - return { - type: ValueSourceType.constant, - value: null, - entityAlias: null - }; - } else if (typeof limit === 'number' || typeof limit === 'string') { - return { - type: ValueSourceType.constant, - value: Number(limit), - entityAlias: null - }; - } - return limit; - } } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/home-page/recent-dashboards-widget.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/home-page/recent-dashboards-widget.component.html index f832b1982e..e8fba081ee 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/home-page/recent-dashboards-widget.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/home-page/recent-dashboards-widget.component.html @@ -24,8 +24,15 @@ {{ 'widgets.recent-dashboards.last' | translate }} {{ 'widgets.recent-dashboards.starred' | translate }} - {{ 'dashboard.add' | translate }} + + @if (hasDevice) { + {{ 'dashboard.add' | translate }} + } @else { + {{ 'dashboard.add' | translate }} + } + diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/home-page/recent-dashboards-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/home-page/recent-dashboards-widget.component.ts index 899ffa5aea..5dfc520856 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/home-page/recent-dashboards-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/home-page/recent-dashboards-widget.component.ts @@ -48,6 +48,13 @@ import { Direction, SortOrder } from '@shared/models/page/sort-order'; import { MatSort } from '@angular/material/sort'; import { DashboardInfo } from '@shared/models/dashboard.models'; import { DashboardAutocompleteComponent } from '@shared/components/dashboard-autocomplete.component'; +import { UtilsService } from '@core/services/utils.service'; +import { Datasource, DatasourceType, widgetType } from '@shared/models/widget.models'; +import { IWidgetSubscription, WidgetSubscriptionOptions } from '@core/api/widget-api.models'; +import { formattedDataFormDatasourceData } from '@core/utils'; +import { AliasFilterType } from '@shared/models/alias.models'; +import { DataKeyType } from '@shared/models/telemetry/telemetry.models'; +import { EntityType } from '@shared/models/entity-type.models'; @Component({ selector: 'tb-recent-dashboards-widget', @@ -78,13 +85,16 @@ export class RecentDashboardsWidgetComponent extends PageComponent implements On starredDashboardValue = null; hasDashboardsAccess = true; + hasDevice = true; dirty = false; public customerId: string; private isFullscreenMode = getCurrentAuthState(this.store).forceFullscreen; + private subscription: IWidgetSubscription; constructor(protected store: Store, private cd: ChangeDetectorRef, + private utils: UtilsService, private userSettingService: UserSettingsService) { super(store); } @@ -96,6 +106,41 @@ export class RecentDashboardsWidgetComponent extends PageComponent implements On this.hasDashboardsAccess = [Authority.TENANT_ADMIN, Authority.CUSTOMER_USER].includes(this.authUser.authority); if (this.hasDashboardsAccess) { this.reload(); + + if (window.location.pathname.startsWith('/home') && this.authUser.authority === Authority.TENANT_ADMIN) { + const ds: Datasource = { + type: DatasourceType.entityCount, + name: '', + entityFilter: { + entityType: EntityType.DEVICE, + type: AliasFilterType.entityType + }, + dataKeys: [this.utils.createKey({ name: 'count'}, DataKeyType.count)] + } + + const apiUsageSubscriptionOptions: WidgetSubscriptionOptions = { + datasources: [ds], + useDashboardTimewindow: false, + type: widgetType.latest, + callbacks: { + onDataUpdated: (subscription) => { + const data = formattedDataFormDatasourceData(subscription.data); + this.hasDevice = (data[0].count || 0) !== 0; + this.cd.detectChanges(); + } + } + }; + this.ctx.subscriptionApi.createSubscription(apiUsageSubscriptionOptions, true).subscribe((subscription) => { + this.subscription = subscription; + }); + } + } + } + + ngOnDestroy() { + super.ngOnDestroy(); + if (this.subscription) { + this.ctx.subscriptionApi.removeSubscription(this.subscription.id); } } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/power-button-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/power-button-widget.models.ts index 0ea5c898bd..4836cc3804 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/power-button-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/power-button-widget.models.ts @@ -337,6 +337,9 @@ export abstract class PowerButtonShape { protected onPowerSymbolLine: Path; private onIcon$: Observable; + private offIcon$: Observable; + protected onIconOffsetX: number = 0; + protected onIconOffsetY: number = 0; protected constructor(protected widgetContext: WidgetContext, protected svgShape: Svg, @@ -362,10 +365,14 @@ export abstract class PowerButtonShape { take(1), map((svgElement) => { const element = new Element(svgElement.firstChild); + const iconGroup = this.svgShape.group(); + element.addTo(iconGroup); const box = element.bbox(); const scale = size / box.height; element.scale(scale); - return element; + const scaledBox = iconGroup.bbox(); + iconGroup.translate(-scaledBox.cx, -scaledBox.cy); + return iconGroup; }), catchError(() => of(null) )); @@ -416,8 +423,15 @@ export abstract class PowerButtonShape { public drawOffShape(centerGroup: G, label: boolean, labelWeight?: string, circleStroke?: boolean) { if (this.icons.offButtonIcon.showIcon) { - this.createIconElement(this.icons.offButtonIcon.icon, this.icons.offButtonIcon.iconSize).subscribe(icon => - this.offPowerSymbolIcon = icon.center(cx, cy).addTo(centerGroup)); + this.offIcon$ = this.createIconElement(this.icons.offButtonIcon.icon, this.icons.offButtonIcon.iconSize) + .pipe(shareReplay(1)); + + this.offIcon$.pipe(take(1)).subscribe(icon => { + this.offPowerSymbolIcon = icon; + icon.translate(cx, cy); + centerGroup.add(icon); + }); + } else { if (label) { this.offLabelShape = this.createOffLabel(labelWeight).addTo(centerGroup); @@ -434,9 +448,12 @@ export abstract class PowerButtonShape { .pipe(shareReplay(1)); this.onIcon$.subscribe(icon => { - this.onPowerSymbolIcon = icon.center(cx, cy); + const iconBox = icon.bbox(); + this.onIconOffsetX = iconBox.cx; + this.onIconOffsetY = iconBox.cy; + this.onPowerSymbolIcon = icon.translate(cx, cy); if (isDefinedAndNotNull(onCenterGroup)) { - this.onPowerSymbolIcon.addTo(onCenterGroup); + onCenterGroup.add(this.onPowerSymbolIcon); } if (isDefinedAndNotNull(mask)) { this.createMask(mask, [this.onPowerSymbolIcon]); @@ -468,7 +485,7 @@ export abstract class PowerButtonShape { public onCenterTimeLine(timeline: Timeline, label: boolean) { if (this.icons.onButtonIcon.showIcon) { if (this.onIcon$) { - this.onIcon$.subscribe(icon => icon.timeline(timeline)) + this.onIcon$.subscribe(icon => icon.timeline(timeline)); } } else { if (label) { @@ -482,7 +499,7 @@ export abstract class PowerButtonShape { public offCenterColor(mainColor: PowerButtonColor, label: boolean) { if (this.icons.offButtonIcon.showIcon) { - this.offPowerSymbolIcon.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); + this.offIcon$.subscribe(icon => icon.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity})) } else { if (label) { this.offLabelShape.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); @@ -495,7 +512,7 @@ export abstract class PowerButtonShape { public onCenterColor(mainColor: PowerButtonColor, label: boolean) { if (this.icons.onButtonIcon.showIcon) { - this.onPowerSymbolIcon.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); + this.onIcon$.subscribe((icon)=> icon.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity})) } else { if (label) { this.onLabelShape.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}); @@ -508,7 +525,12 @@ export abstract class PowerButtonShape { public buttonAnimation(scale: number, label: boolean) { if (this.icons.onButtonIcon.showIcon) { - powerButtonAnimation(this.onPowerSymbolIcon).transform({scale}); + const translateX = cx - this.onIconOffsetX; + const translateY = cy - this.onIconOffsetY; + powerButtonAnimation(this.onPowerSymbolIcon).transform({ + scale: scale, + translate: [translateX, translateY] + }); } else { if (label) { powerButtonAnimation(this.onLabelShape).transform({scale, origin: {x: cx, y: cy}}); diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/bar-chart-with-labels-widget-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/bar-chart-with-labels-widget-settings.component.ts index 6290572770..55ed3d5035 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/bar-chart-with-labels-widget-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/bar-chart-with-labels-widget-settings.component.ts @@ -31,6 +31,7 @@ import { barChartWithLabelsDefaultSettings } from '@home/components/widget/lib/chart/bar-chart-with-labels-widget.models'; import { getSourceTbUnitSymbol } from '@shared/models/unit.models'; +import { updateLatestDataKeys } from '@home/components/widget/lib/chart/time-series-chart.models'; @Component({ selector: 'tb-bar-chart-with-labels-widget-settings', @@ -123,6 +124,11 @@ export class BarChartWithLabelsWidgetSettingsComponent extends WidgetSettingsCom }); } + protected onSettingsChanged(updated: WidgetSettings) { + updateLatestDataKeys([updated.yAxis], this.datasource, this.dataKeyCallbacks); + super.onSettingsChanged(updated); + } + protected validatorTriggers(): string[] { return ['showBarLabel', 'showBarValue', 'showBarBorder', 'showLegend', 'showTooltip', 'tooltipShowDate']; } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/range-chart-widget-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/range-chart-widget-settings.component.ts index 703c621d62..053b6858bc 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/range-chart-widget-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/range-chart-widget-settings.component.ts @@ -30,7 +30,7 @@ import { rangeChartDefaultSettings } from '@home/components/widget/lib/chart/ran import { DateFormatProcessor, DateFormatSettings } from '@shared/models/widget-settings.models'; import { lineSeriesStepTypes, - lineSeriesStepTypeTranslations + lineSeriesStepTypeTranslations, updateLatestDataKeys } from '@home/components/widget/lib/chart/time-series-chart.models'; import { chartLabelPositions, @@ -269,6 +269,11 @@ export class RangeChartWidgetSettingsComponent extends WidgetSettingsComponent { } } + protected onSettingsChanged(updated: WidgetSettings) { + updateLatestDataKeys([updated.yAxis], this.datasource, this.dataKeyCallbacks); + super.onSettingsChanged(updated); + } + private _pointLabelPreviewFn(): string { const units = getSourceTbUnitSymbol(this.widgetConfig.config.units); const decimals: number = this.widgetConfig.config.decimals; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/axis-scale-row.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/axis-scale-row.component.ts index 24037f2170..914eb0abb2 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/axis-scale-row.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/axis-scale-row.component.ts @@ -97,7 +97,7 @@ export class AxisScaleRowComponent implements ControlValueAccessor, OnInit, Vali this.limitForm = this.fb.group({ type: [ValueSourceType.constant], value: [null], - entityAlias: [null] + entityAlias: [null, [Validators.required]] }); this.latestKeyFormControl = this.fb.control(null, [Validators.required]); this.entityKeyFormControl = this.fb.control(null, [Validators.required]); @@ -169,23 +169,24 @@ export class AxisScaleRowComponent implements ControlValueAccessor, OnInit, Vali } private updateValidators() { - const axisTypeControl = this.limitForm.get('type'); - if (axisTypeControl && this.entityKeyFormControl && this.latestKeyFormControl) { - const type = axisTypeControl.value; - if (type === ValueSourceType.latestKey) { - this.latestKeyFormControl.setValidators([Validators.required]); - this.entityKeyFormControl.clearValidators(); - } else if (type === ValueSourceType.entity) { - this.latestKeyFormControl.clearValidators(); - this.limitForm.get('entityAlias').setValidators([Validators.required]); - this.entityKeyFormControl.setValidators([Validators.required]); - } else { - this.latestKeyFormControl.clearValidators(); - this.entityKeyFormControl.clearValidators(); - } - this.latestKeyFormControl.updateValueAndValidity({ emitEvent: false }); - this.entityKeyFormControl.updateValueAndValidity({ emitEvent: false }); - } + const type = this.limitForm.get('type')?.value; + const entityAliasCtr = this.limitForm.get('entityAlias'); + + const isLatestKey = type === ValueSourceType.latestKey; + const isEntity = type === ValueSourceType.entity; + + isLatestKey ? this.latestKeyFormControl.enable({ emitEvent: false }) + : this.latestKeyFormControl.disable({ emitEvent: false }); + + isEntity ? this.entityKeyFormControl.enable({ emitEvent: false }) + : this.entityKeyFormControl.disable({ emitEvent: false }); + + isEntity ? entityAliasCtr.enable({ emitEvent: false }) + : entityAliasCtr.disable({ emitEvent: false }); + + this.latestKeyFormControl.updateValueAndValidity({ emitEvent: false }); + this.entityKeyFormControl.updateValueAndValidity({ emitEvent: false }); + entityAliasCtr.updateValueAndValidity({ emitEvent: false }); } private updateModel() { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-axis-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-axis-settings.component.ts index db0f8534a9..2f23666574 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-axis-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-axis-settings.component.ts @@ -25,7 +25,7 @@ import { Validators } from '@angular/forms'; import { - AxisPosition, defaultXAxisTicksFormat, + AxisPosition, defaultXAxisTicksFormat, normalizeAxisLimit, timeSeriesAxisPositionTranslations, TimeSeriesChartAxisSettings, TimeSeriesChartXAxisSettings, TimeSeriesChartYAxisSettings @@ -138,8 +138,8 @@ export class TimeSeriesChartAxisSettingsComponent implements OnInit, ControlValu this.axisSettingsFormGroup.addControl('ticksGenerator', this.fb.control(null, [])); this.axisSettingsFormGroup.addControl('interval', this.fb.control(null, [Validators.min(0)])); this.axisSettingsFormGroup.addControl('splitNumber', this.fb.control(null, [Validators.min(1)])); - this.axisSettingsFormGroup.addControl('min', this.fb.control(null, [])); - this.axisSettingsFormGroup.addControl('max', this.fb.control(null, [])); + this.axisSettingsFormGroup.addControl('min', this.fb.control(normalizeAxisLimit(null), [])); + this.axisSettingsFormGroup.addControl('max', this.fb.control(normalizeAxisLimit(null), [])); } else if (this.axisType === 'xAxis') { this.axisSettingsFormGroup.addControl('ticksFormat', this.fb.control(null, [])); } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-y-axes-panel.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-y-axes-panel.component.ts index 694e856bfa..662214a1df 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-y-axes-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-y-axes-panel.component.ts @@ -36,13 +36,15 @@ import { Validator } from '@angular/forms'; import { + checkLatestDataKeys, defaultTimeSeriesChartYAxisSettings, getNextTimeSeriesYAxisId, TimeSeriesChartYAxes, TimeSeriesChartYAxisId, TimeSeriesChartYAxisSettings, timeSeriesChartYAxisValid, - timeSeriesChartYAxisValidator + timeSeriesChartYAxisValidator, + updateLatestDataKeys } from '@home/components/widget/lib/chart/time-series-chart.models'; import { mergeDeep } from '@core/utils'; import { CdkDragDrop } from '@angular/cdk/drag-drop'; @@ -50,7 +52,7 @@ import { coerceBoolean } from '@shared/decorators/coercion'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { IAliasController } from '@app/core/public-api'; import { DataKeysCallbacks } from '@home/components/widget/lib/settings/common/key/data-keys.component.models'; -import { DataKey, DataKeyType, Datasource, ValueSourceConfig, ValueSourceType } from '@app/shared/public-api'; +import { Datasource } from '@app/shared/public-api'; @Component({ selector: 'tb-time-series-chart-y-axes-panel', @@ -127,7 +129,7 @@ export class TimeSeriesChartYAxesPanelComponent implements ControlValueAccessor, for (const axis of axes) { yAxes[axis.id] = axis; } - this.updateLatestDataKeys(Object.values(yAxes)); + updateLatestDataKeys(Object.values(yAxes), this.datasource, this.dataKeyCallbacks); this.propagateChange(yAxes); } ); @@ -150,7 +152,7 @@ export class TimeSeriesChartYAxesPanelComponent implements ControlValueAccessor, } writeValue(value: TimeSeriesChartYAxes | undefined): void { - const yAxes: TimeSeriesChartYAxes = this.checkLatestDataKeys(value || {}); + const yAxes: TimeSeriesChartYAxes = checkLatestDataKeys(value || {}, this.datasource); if (!yAxes.default) { yAxes.default = mergeDeep({} as TimeSeriesChartYAxisSettings, defaultTimeSeriesChartYAxisSettings, {id: 'default', order: 0} as TimeSeriesChartYAxisSettings); @@ -197,8 +199,6 @@ export class TimeSeriesChartYAxesPanelComponent implements ControlValueAccessor, const axes: TimeSeriesChartYAxisSettings[] = this.yAxesFormGroup.get('axes').value; axis.id = getNextTimeSeriesYAxisId(axes); axis.order = axes.length; - axis.min = this.normalizeAxisLimit(axis.min); - axis.max = this.normalizeAxisLimit(axis.max); const axesArray = this.yAxesFormGroup.get('axes') as UntypedFormArray; const axisControl = this.fb.control(axis, [timeSeriesChartYAxisValidator]); axesArray.push(axisControl); @@ -212,100 +212,4 @@ export class TimeSeriesChartYAxesPanelComponent implements ControlValueAccessor, return this.fb.array(axesControls); } - private checkLatestDataKeys(yAxes: TimeSeriesChartYAxes): TimeSeriesChartYAxes { - const latestKeys = this.datasource?.latestDataKeys || []; - const result: TimeSeriesChartYAxes = {}; - - for (const [id, axis] of Object.entries(yAxes)) { - axis.min = this.normalizeAxisLimit(axis.min); - axis.max = this.normalizeAxisLimit(axis.max); - const minCfg = axis.min; - const maxCfg = axis.max; - - const minValid = !!minCfg && ( - minCfg.type !== ValueSourceType.latestKey || - latestKeys.some(k => this.isYAxisKey(k, minCfg)) - ); - - const maxValid = !!maxCfg && ( - maxCfg.type !== ValueSourceType.latestKey || - latestKeys.some(k => this.isYAxisKey(k, maxCfg)) - ); - - if (minValid && maxValid) { - result[id] = axis; - } - } - - return result; - } - - private updateLatestDataKeys(yAxes: TimeSeriesChartYAxisSettings[]) { - if (this.datasource) { - let latestKeys = this.datasource.latestDataKeys; - if (!latestKeys) { - latestKeys = []; - this.datasource.latestDataKeys = latestKeys; - } - const existingYAxisKeys = latestKeys.filter(k => k.settings?.__yAxisMinKey === true || k.settings?.__yAxisMaxKey === true); - const foundYAxisKeys: DataKey[] = []; - - for(const yAxis of yAxes) { - const min = yAxis.min as ValueSourceConfig; - const max = yAxis.max as ValueSourceConfig; - if (min.type === ValueSourceType.latestKey) { - const found = existingYAxisKeys.find(k => this.isYAxisKey(k, min)); - if (!found) { - const newKey = this.dataKeyCallbacks.generateDataKey(min.latestKey, min.latestKeyType, - null, true, null); - newKey.settings.__yAxisMinKey = true; - latestKeys.push(newKey); - } else if (foundYAxisKeys.indexOf(found) === -1) { - foundYAxisKeys.push(found); - } - } - if (max.type === ValueSourceType.latestKey) { - const found = existingYAxisKeys.find(k => this.isYAxisKey(k, max)); - if (!found) { - const newKey = this.dataKeyCallbacks.generateDataKey(max.latestKey, max.latestKeyType, - null, true, null); - newKey.settings.__yAxisMaxKey = true; - latestKeys.push(newKey); - } else if (foundYAxisKeys.indexOf(found) === -1) { - foundYAxisKeys.push(found); - } - } - } - const toRemove = existingYAxisKeys.filter(k => foundYAxisKeys.indexOf(k) === -1); - for (const key of toRemove) { - const index = latestKeys.indexOf(key); - if (index > -1) { - latestKeys.splice(index, 1); - } - } - } - } - - private isYAxisKey(d: DataKey, limit: ValueSourceConfig): boolean { - return (d.type === DataKeyType.function && d.label === limit.latestKey) || - (d.type !== DataKeyType.function && d.name === limit.latestKey && - d.type === limit.latestKeyType); - } - - private normalizeAxisLimit(limit: string | number | ValueSourceConfig): ValueSourceConfig { - if (!limit) { - return { - type: ValueSourceType.constant, - value: null, - entityAlias: null - }; - } else if (typeof limit === 'number' || typeof limit === 'string') { - return { - type: ValueSourceType.constant, - value: Number(limit), - entityAlias: null - }; - } - return limit; - } } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-y-axis-row.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-y-axis-row.component.ts index 5298b7f45b..93e91124d5 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-y-axis-row.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-y-axis-row.component.ts @@ -29,7 +29,7 @@ import { } from '@angular/core'; import { ControlValueAccessor, NG_VALUE_ACCESSOR, UntypedFormBuilder, UntypedFormGroup } from '@angular/forms'; import { - AxisPosition, + AxisPosition, normalizeAxisLimit, timeSeriesAxisPositionTranslations, TimeSeriesChartYAxisSettings } from '@home/components/widget/lib/chart/time-series-chart.models'; @@ -136,8 +136,8 @@ export class TimeSeriesChartYAxisRowComponent implements ControlValueAccessor, O writeValue(value: TimeSeriesChartYAxisSettings): void { this.modelValue = value; - const min = this.normalizeLimit(value.min); - const max = this.normalizeLimit(value.max); + const min = normalizeAxisLimit(value.min); + const max = normalizeAxisLimit(value.max); this.axisFormGroup.patchValue({ label: value.label, @@ -252,27 +252,4 @@ export class TimeSeriesChartYAxisRowComponent implements ControlValueAccessor, O entityKeyType: [null, []] }); } - - private normalizeLimit(limit: any) { - const base = { - type: ValueSourceType.constant, - value: null, - latestKey: null, - latestKeyType: null, - entityAlias: null, - entityKey: null, - entityKeyType: null - }; - - if (limit == null) return base; - - if (typeof limit === 'number' || typeof limit === 'string') { - return { ...base, type: ValueSourceType.constant, value: Number(limit) }; - } - - return { - ...base, - ...limit, - }; - } } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/image-cards-select.component.scss b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/image-cards-select.component.scss index 62915cc390..de1901aa4a 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/image-cards-select.component.scss +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/image-cards-select.component.scss @@ -44,7 +44,7 @@ display: none; } } - > .mat-expansion-panel-content { + .mat-expansion-panel-content { > .mat-expansion-panel-body { padding: 0 16px 16px !important; } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/marker-image-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/marker-image-settings.component.ts index 6da6c809e0..989e9ef0c7 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/marker-image-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/marker-image-settings.component.ts @@ -82,11 +82,12 @@ export class MarkerImageSettingsComponent implements ControlValueAccessor { renderer: this.renderer, componentType: MarkerImageSettingsPanelComponent, hostView: this.viewContainerRef, - preferredPlacement: 'left', + preferredPlacement: 'leftOnly', context: { markerImageSettings: this.modelValue, }, - isModal: true + isModal: true, + overlayStyle: {padding: '10px'} }); markerImageSettingsPanelPopover.tbComponentRef.instance.popover = markerImageSettingsPanelPopover; markerImageSettingsPanelPopover.tbComponentRef.instance.markerImageSettingsApplied.subscribe((markerImageSettings) => { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/shape-fill-image-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/shape-fill-image-settings.component.ts index 5c9bbb260f..74f4ddd1eb 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/shape-fill-image-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/shape-fill-image-settings.component.ts @@ -82,11 +82,12 @@ export class ShapeFillImageSettingsComponent implements ControlValueAccessor { renderer: this.renderer, componentType: ShapeFillImageSettingsPanelComponent, hostView: this.viewContainerRef, - preferredPlacement: 'left', + preferredPlacement: 'leftOnly', context: { shapeFillImageSettings: this.modelValue, }, - isModal: true + isModal: true, + overlayStyle: {padding: '10px'} }).tbComponentRef.instance.shapeFillImageSettingsApplied.subscribe((shapeFillImageSettings) => { this.modelValue = shapeFillImageSettings; this.propagateChange(this.modelValue); diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/widget-settings.scss b/ui-ngx/src/app/modules/home/components/widget/lib/settings/widget-settings.scss index 8579f2a872..4d1c34e131 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/widget-settings.scss +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/widget-settings.scss @@ -108,7 +108,7 @@ align-items: center; } - > .mat-expansion-panel-content { + .mat-expansion-panel-content { > .mat-expansion-panel-body { padding: 0; } diff --git a/ui-ngx/src/app/modules/home/models/datasource/attribute-datasource.ts b/ui-ngx/src/app/modules/home/models/datasource/attribute-datasource.ts index eaeade89ae..cc3112442f 100644 --- a/ui-ngx/src/app/modules/home/models/datasource/attribute-datasource.ts +++ b/ui-ngx/src/app/modules/home/models/datasource/attribute-datasource.ts @@ -115,9 +115,11 @@ export class AttributeDatasource implements DataSource { } isAllSelected(): Observable { - const numSelected = this.selection.selected.length; return this.attributesSubject.pipe( - map((attributes) => numSelected === attributes.length) + map((attributes) => { + const numSelected = this.selection.selected.length; + return attributes.length > 0 && numSelected === attributes.length + }) ); } diff --git a/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts b/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts index 913e754fb7..336ac035da 100644 --- a/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts +++ b/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts @@ -175,7 +175,7 @@ export class EntityTableConfig, P extends PageLink = P selectionEnabled = true; searchEnabled = true; addEnabled = true; - addAsTextButton = false; + addAsTextButton = true; entitiesDeleteEnabled = true; detailsPanelEnabled = true; hideDetailsTabsOnEdit = true; diff --git a/ui-ngx/src/app/modules/home/pages/admin/mail-server.component.html b/ui-ngx/src/app/modules/home/pages/admin/mail-server.component.html index 76d255f40e..2ba7adda3b 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/mail-server.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/mail-server.component.html @@ -280,7 +280,7 @@ -
+
admin.oauth2.redirect-uri-template diff --git a/ui-ngx/src/app/modules/home/pages/ai-model/ai-model-table-config.resolve.ts b/ui-ngx/src/app/modules/home/pages/ai-model/ai-model-table-config.resolve.ts index 2a73488d7e..7c78bd7c02 100644 --- a/ui-ngx/src/app/modules/home/pages/ai-model/ai-model-table-config.resolve.ts +++ b/ui-ngx/src/app/modules/home/pages/ai-model/ai-model-table-config.resolve.ts @@ -47,7 +47,6 @@ export class AiModelsTableConfigResolver { ) { this.config.selectionEnabled = true; this.config.entityType = EntityType.AI_MODEL; - this.config.addAsTextButton = true; this.config.rowPointer = true; this.config.detailsPanelEnabled = false; this.config.entityTranslations = entityTypeTranslations.get(EntityType.AI_MODEL); diff --git a/ui-ngx/src/app/modules/home/pages/mobile/applications/mobile-app-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/mobile/applications/mobile-app-table-config.resolver.ts index 13698ac4a5..274edf5933 100644 --- a/ui-ngx/src/app/modules/home/pages/mobile/applications/mobile-app-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/mobile/applications/mobile-app-table-config.resolver.ts @@ -57,7 +57,6 @@ export class MobileAppTableConfigResolver { ) { this.config.selectionEnabled = false; this.config.entityType = EntityType.MOBILE_APP; - this.config.addAsTextButton = true; this.config.entitiesDeleteEnabled = false; this.config.rowPointer = true; this.config.entityTranslations = entityTypeTranslations.get(EntityType.MOBILE_APP); diff --git a/ui-ngx/src/app/modules/home/pages/mobile/bundes/mobile-bundle-table-config.resolve.ts b/ui-ngx/src/app/modules/home/pages/mobile/bundes/mobile-bundle-table-config.resolve.ts index 69be801980..ba20624852 100644 --- a/ui-ngx/src/app/modules/home/pages/mobile/bundes/mobile-bundle-table-config.resolve.ts +++ b/ui-ngx/src/app/modules/home/pages/mobile/bundes/mobile-bundle-table-config.resolve.ts @@ -63,7 +63,6 @@ export class MobileBundleTableConfigResolver { ) { this.config.selectionEnabled = false; this.config.entityType = EntityType.MOBILE_APP_BUNDLE; - this.config.addAsTextButton = true; this.config.rowPointer = true; this.config.detailsPanelEnabled = false; this.config.entityTranslations = entityTypeTranslations.get(EntityType.MOBILE_APP_BUNDLE); diff --git a/ui-ngx/src/app/modules/home/pages/mobile/common/editor-panel.component.ts b/ui-ngx/src/app/modules/home/pages/mobile/common/editor-panel.component.ts index 9f2e850b53..dbc1a5d9c8 100644 --- a/ui-ngx/src/app/modules/home/pages/mobile/common/editor-panel.component.ts +++ b/ui-ngx/src/app/modules/home/pages/mobile/common/editor-panel.component.ts @@ -48,7 +48,7 @@ export class EditorPanelComponent implements OnInit { tinyMceOptions: Partial = { base_url: '/assets/tinymce', suffix: '.min', - plugins: ['link', 'table', 'image', 'imagetools', 'lists', 'fullscreen'], + plugins: ['link', 'table', 'image', 'lists', 'fullscreen'], menubar: 'edit insert view format', toolbar: ['fontfamily fontsize | bold italic underline strikethrough forecolor backcolor', 'alignleft aligncenter alignright alignjustify | bullist | link table image | fullscreen'], diff --git a/ui-ngx/src/app/modules/home/pages/notification/recipient/recipient-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/notification/recipient/recipient-table-config.resolver.ts index c2fc08fb7c..8e729e94ef 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/recipient/recipient-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/notification/recipient/recipient-table-config.resolver.ts @@ -48,7 +48,6 @@ export class RecipientTableConfigResolver { this.config.entityType = EntityType.NOTIFICATION_TARGET; this.config.detailsPanelEnabled = false; - this.config.addAsTextButton = true; this.config.rowPointer = true; this.config.entityTranslations = entityTypeTranslations.get(EntityType.NOTIFICATION_TARGET); diff --git a/ui-ngx/src/app/modules/home/pages/notification/rule/rule-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/notification/rule/rule-table-config.resolver.ts index 066db92c0a..f71197ae95 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/rule/rule-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/notification/rule/rule-table-config.resolver.ts @@ -49,7 +49,6 @@ export class RuleTableConfigResolver { this.config.entityType = EntityType.NOTIFICATION_RULE; this.config.detailsPanelEnabled = false; - this.config.addAsTextButton = true; this.config.rowPointer = true; this.config.entityTranslations = entityTypeTranslations.get(EntityType.NOTIFICATION_RULE); diff --git a/ui-ngx/src/app/modules/home/pages/notification/template/configuration/notification-template-configuration.component.scss b/ui-ngx/src/app/modules/home/pages/notification/template/configuration/notification-template-configuration.component.scss index 34bff387e6..de9c0a06a7 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/template/configuration/notification-template-configuration.component.scss +++ b/ui-ngx/src/app/modules/home/pages/notification/template/configuration/notification-template-configuration.component.scss @@ -65,7 +65,7 @@ :host ::ng-deep { .tb-form-panel .mat-expansion-panel.tb-settings { - & > .mat-expansion-panel-content > .mat-expansion-panel-body { + .mat-expansion-panel-content > .mat-expansion-panel-body { gap: 0; } } diff --git a/ui-ngx/src/app/modules/home/pages/notification/template/template-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/notification/template/template-table-config.resolver.ts index 0ee0b921f4..b1bc3eef3e 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/template/template-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/notification/template/template-table-config.resolver.ts @@ -47,7 +47,6 @@ export class TemplateTableConfigResolver { this.config.entityType = EntityType.NOTIFICATION_TEMPLATE; this.config.detailsPanelEnabled = false; - this.config.addAsTextButton = true; this.config.rowPointer = true; this.config.entityTranslations = entityTypeTranslations.get(EntityType.NOTIFICATION_TEMPLATE); diff --git a/ui-ngx/src/app/modules/login/pages/login/login.component.ts b/ui-ngx/src/app/modules/login/pages/login/login.component.ts index 1cf115f9dc..a10e213be5 100644 --- a/ui-ngx/src/app/modules/login/pages/login/login.component.ts +++ b/ui-ngx/src/app/modules/login/pages/login/login.component.ts @@ -76,7 +76,7 @@ export class LoginComponent extends PageComponent implements OnInit { getOAuth2Uri(oauth2Client: OAuth2ClientLoginInfo): string { let result = ""; if (this.authService.redirectUrl) { - result += "?prevUri=" + this.authService.redirectUrl; + result += "?prevUri=" + encodeURIComponent(this.authService.redirectUrl); } return oauth2Client.url + result; } diff --git a/ui-ngx/src/app/shared/components/color-picker/color-input.base.scss b/ui-ngx/src/app/shared/components/color-picker/color-input.base.scss new file mode 100644 index 0000000000..bae2e433aa --- /dev/null +++ b/ui-ngx/src/app/shared/components/color-picker/color-input.base.scss @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .color-input-container { + display: flex; + gap: 4px; + align-items: center; + } + .color-input { + max-width: 72px; + margin-bottom: 4px; + } +} + + diff --git a/ui-ngx/src/app/shared/components/color-picker/color-picker-panel.component.scss b/ui-ngx/src/app/shared/components/color-picker/color-picker-panel.component.scss index f9db566be8..5e867f9e61 100644 --- a/ui-ngx/src/app/shared/components/color-picker/color-picker-panel.component.scss +++ b/ui-ngx/src/app/shared/components/color-picker/color-picker-panel.component.scss @@ -16,7 +16,7 @@ @import "../scss/constants"; .tb-color-picker-panel { - width: 342px; + width: 370px; display: flex; flex-direction: column; max-height: calc(100vh - 24px); diff --git a/ui-ngx/src/app/shared/components/color-picker/color-picker.component.html b/ui-ngx/src/app/shared/components/color-picker/color-picker.component.html index 676dafbfef..fcb1cb067d 100644 --- a/ui-ngx/src/app/shared/components/color-picker/color-picker.component.html +++ b/ui-ngx/src/app/shared/components/color-picker/color-picker.component.html @@ -37,10 +37,8 @@ HSLA
- - + +
diff --git a/ui-ngx/src/app/shared/components/color-picker/color-picker.component.scss b/ui-ngx/src/app/shared/components/color-picker/color-picker.component.scss index ed161a5b9f..799bcdf00c 100644 --- a/ui-ngx/src/app/shared/components/color-picker/color-picker.component.scss +++ b/ui-ngx/src/app/shared/components/color-picker/color-picker.component.scss @@ -83,7 +83,7 @@ height: 56px; display: flex; align-items: center; - gap: 20px; + gap: 8px; .presentation-select { font-size: 14px; @@ -104,7 +104,7 @@ display: flex; flex-direction: row; flex-wrap: wrap; - justify-content: space-between; + justify-content: center; gap: 8px; @media #{$mat-xs} { flex-direction: column; diff --git a/ui-ngx/src/app/shared/components/color-picker/hex-input.component.scss b/ui-ngx/src/app/shared/components/color-picker/hex-input.component.scss index 2f7a404c0d..73a6aff9f6 100644 --- a/ui-ngx/src/app/shared/components/color-picker/hex-input.component.scss +++ b/ui-ngx/src/app/shared/components/color-picker/hex-input.component.scss @@ -19,11 +19,11 @@ gap: 8px; } .hex-input { - max-width: 190px; + max-width: 220px; } .alpha-input { - min-width: 60px; - max-width: 60px; + min-width: 72px; + max-width: 72px; } ::ng-deep { diff --git a/ui-ngx/src/app/shared/components/color-picker/hsla-input.component.html b/ui-ngx/src/app/shared/components/color-picker/hsla-input.component.html new file mode 100644 index 0000000000..cf8e25eb51 --- /dev/null +++ b/ui-ngx/src/app/shared/components/color-picker/hsla-input.component.html @@ -0,0 +1,54 @@ + +
+
+ + + + @if (labelVisible) { + H + } +
+
+ + + {{suffixValue}} + + @if (labelVisible) { + S + } +
+
+ + + {{suffixValue}} + + @if (labelVisible) { + L + } +
+
+ + + {{suffixValue}} + + @if (labelVisible) { + A + } +
+
diff --git a/ui-ngx/src/app/shared/components/color-picker/hsla-input.component.ts b/ui-ngx/src/app/shared/components/color-picker/hsla-input.component.ts new file mode 100644 index 0000000000..40ec056972 --- /dev/null +++ b/ui-ngx/src/app/shared/components/color-picker/hsla-input.component.ts @@ -0,0 +1,72 @@ +/// +/// Copyright © 2016-2026 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, EventEmitter, Input, Output } from '@angular/core'; +import { Color } from '@iplab/ngx-color-picker'; +import { coerceBoolean } from '@shared/decorators/coercion'; + +type Channel = 'H' | 'S' | 'L'; + +@Component({ + selector: 'tb-hsla-input', + templateUrl: './hsla-input.component.html', + styleUrl: './color-input.base.scss', + standalone: false +}) +export class HslaInputComponent { + + @Input() + public color: Color; + + @Output() + public colorChange = new EventEmitter(false); + + @Input() + @coerceBoolean() + public labelVisible = false; + + @Input() + public suffixValue = '%'; + + public get value() { + return this.color.getHsla(); + } + + public get alphaValue(): number { + return this.color ? Math.round(this.color.getHsla().getAlpha() * 100) : 0; + } + + public onAlphaInputChange(inputValue: number): void { + if (!this.color) return; + const hsla = this.color.getHsla(); + const alpha = +inputValue / 100; + if (hsla.alpha !== alpha) { + const newColor = new Color().setHsla(hsla.getHue(), hsla.getSaturation(), hsla.getLightness(), alpha); + this.colorChange.emit(newColor); + } + } + + public onInputChange(newValue: number, channel: Channel): void { + if (!this.color) return; + const hsla = this.value; + const hue = channel === 'H' ? +newValue : hsla.getHue(); + const saturation = channel === 'S' ? +newValue : hsla.getSaturation(); + const lightness = channel === 'L' ? +newValue : hsla.getLightness(); + if (hue === hsla.getHue() && saturation === hsla.getSaturation() && lightness === hsla.getLightness()) return; + const newColor = new Color().setHsla(hue, saturation, lightness, hsla.getAlpha()); + this.colorChange.emit(newColor); + } +} diff --git a/ui-ngx/src/app/shared/components/color-picker/input-change.directive.ts b/ui-ngx/src/app/shared/components/color-picker/input-change.directive.ts new file mode 100644 index 0000000000..55604df28e --- /dev/null +++ b/ui-ngx/src/app/shared/components/color-picker/input-change.directive.ts @@ -0,0 +1,46 @@ +/// +/// Copyright © 2016-2026 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Directive, EventEmitter, HostBinding, HostListener, Input, numberAttribute, Output } from '@angular/core'; + +@Directive({ + selector: '[inputChange]', + standalone: false +}) +export class InputChangeDirective { + + @Input({transform: numberAttribute}) + @HostBinding('attr.min') + min = 0; + + @Input({transform: numberAttribute}) + @HostBinding('attr.max') + max = 255; + + @Output() + public inputChange = new EventEmitter(); + + @HostListener('input', ['$event']) + public inputChanges(event: any): void { + const element = event.target as HTMLInputElement || event.srcElement as HTMLInputElement; + const value = element.value; + + const numeric = parseFloat(value); + if (!isNaN(numeric) && numeric >= this.min && numeric <= this.max) { + this.inputChange.emit(numeric); + } + } +} diff --git a/ui-ngx/src/app/shared/components/color-picker/rgba-input.component.html b/ui-ngx/src/app/shared/components/color-picker/rgba-input.component.html new file mode 100644 index 0000000000..3cb856f147 --- /dev/null +++ b/ui-ngx/src/app/shared/components/color-picker/rgba-input.component.html @@ -0,0 +1,52 @@ + +
+
+ + + + @if (labelVisible) { + R + } +
+
+ + + + @if (labelVisible) { + G + } +
+
+ + + + @if (labelVisible) { + B + } +
+
+ + + {{suffixValue}} + + @if (labelVisible) { + A + } +
+
diff --git a/ui-ngx/src/app/shared/components/color-picker/rgba-input.component.ts b/ui-ngx/src/app/shared/components/color-picker/rgba-input.component.ts new file mode 100644 index 0000000000..04f2dbc44b --- /dev/null +++ b/ui-ngx/src/app/shared/components/color-picker/rgba-input.component.ts @@ -0,0 +1,71 @@ +/// +/// Copyright © 2016-2026 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, EventEmitter, Input, Output } from '@angular/core'; +import { Color } from '@iplab/ngx-color-picker'; +import { coerceBoolean } from '@shared/decorators/coercion'; + +type Channel = 'R' | 'G' | 'B'; + +@Component({ + selector: 'tb-rgba-input', + templateUrl: './rgba-input.component.html', + styleUrl: './color-input.base.scss', + standalone: false +}) +export class RgbaInputComponent { + + @Input() + public color: Color; + + @Output() + public colorChange = new EventEmitter(false); + + @Input() + @coerceBoolean() + public labelVisible = false; + + @Input() + public suffixValue = '%'; + + public get value() { + return this.color.getRgba(); + } + + public get alphaValue(): string { + return this.color ? Math.round(this.color.getRgba().getAlpha() * 100).toString() : ''; + } + + public onAlphaInputChange(inputValue: number): void { + if (!this.color) return; + const color = this.color.getRgba(); + const alpha = +inputValue / 100; + if (color.getAlpha() !== alpha) { + const newColor = new Color().setRgba(color.getRed(), color.getGreen(), color.getBlue(), alpha).toRgbaString(); + this.colorChange.emit(new Color(newColor)); + } + } + + onInputChange(newValue: number, channel: Channel) { + if (!this.color) return; + const rgba = this.value; + const red = channel === 'R' ? newValue : rgba.getRed(); + const green = channel === 'G' ? newValue : rgba.getGreen(); + const blue = channel === 'B' ? newValue : rgba.getBlue(); + if (red === rgba.getRed() && green === rgba.getGreen() && blue === rgba.getBlue()) return; + this.colorChange.emit(new Color().setRgba(red, green, blue, rgba.alpha)); + } +} diff --git a/ui-ngx/src/app/shared/components/dialog/dynamic/dynamic-dialog.module.ts b/ui-ngx/src/app/shared/components/dialog/dynamic/dynamic-dialog.module.ts index c1defae012..078234dbda 100644 --- a/ui-ngx/src/app/shared/components/dialog/dynamic/dynamic-dialog.module.ts +++ b/ui-ngx/src/app/shared/components/dialog/dynamic/dynamic-dialog.module.ts @@ -14,9 +14,9 @@ /// limitations under the License. /// -import { OverlayModule } from '@angular/cdk/overlay'; +import { Overlay, OverlayContainer, OverlayModule } from '@angular/cdk/overlay'; import { NgModule } from '@angular/core'; -import { DEFAULT_DIALOG_CONFIG, DialogConfig, DialogModule } from '@angular/cdk/dialog'; +import { DEFAULT_DIALOG_CONFIG, Dialog, DialogConfig, DialogModule } from '@angular/cdk/dialog'; import { MatDialogModule } from '@angular/material/dialog'; import { DynamicDialog, DynamicMatDialog } from './dynamic-dialog'; import { DynamicOverlay } from './dynamic-overlay'; @@ -24,8 +24,11 @@ import { DynamicOverlayContainer } from './dynamic-overlay-container'; export const DYNAMIC_MAT_DIALOG_PROVIDERS = [ DynamicOverlayContainer, + { provide: OverlayContainer, useExisting: DynamicOverlayContainer }, DynamicOverlay, + { provide: Overlay, useExisting: DynamicOverlay }, DynamicDialog, + { provide: Dialog, useExisting: DynamicDialog }, DynamicMatDialog, { provide: DEFAULT_DIALOG_CONFIG, diff --git a/ui-ngx/src/app/shared/components/dialog/dynamic/dynamic-dialog.ts b/ui-ngx/src/app/shared/components/dialog/dynamic/dynamic-dialog.ts index 2fdc63f381..f717ccb0a7 100644 --- a/ui-ngx/src/app/shared/components/dialog/dynamic/dynamic-dialog.ts +++ b/ui-ngx/src/app/shared/components/dialog/dynamic/dynamic-dialog.ts @@ -14,18 +14,11 @@ /// limitations under the License. /// -import { Location } from "@angular/common"; -import { Inject, Injectable, Injector, Optional, SkipSelf, TemplateRef } from '@angular/core'; -import { - MAT_DIALOG_DEFAULT_OPTIONS, - MAT_DIALOG_SCROLL_STRATEGY, - MatDialog, - MatDialogConfig, MatDialogRef -} from '@angular/material/dialog'; -import { DynamicOverlay } from "./dynamic-overlay"; -import { DynamicOverlayContainer } from '@shared/components/dialog/dynamic/dynamic-overlay-container'; -import { ComponentType, ScrollStrategy } from '@angular/cdk/overlay'; -import { DEFAULT_DIALOG_CONFIG, Dialog, DialogConfig } from '@angular/cdk/dialog'; +import { inject, Injectable, TemplateRef } from '@angular/core'; +import { MatDialog, MatDialogConfig, MatDialogRef } from '@angular/material/dialog'; +import { DynamicOverlay } from './dynamic-overlay'; +import { ComponentType } from '@angular/cdk/overlay'; +import { Dialog } from '@angular/cdk/dialog'; export interface DynamicMatDialogConfig extends MatDialogConfig { containerElement?: HTMLElement; @@ -34,25 +27,12 @@ export interface DynamicMatDialogConfig extends MatDialogConfig { @Injectable() export class DynamicMatDialog extends MatDialog { - private _customOverlay: DynamicOverlay; + private _customOverlay = inject(DynamicOverlay); - constructor( _overlay: DynamicOverlay, - _injector: Injector, - @Optional() location: Location, - @Inject( MAT_DIALOG_DEFAULT_OPTIONS ) _defaultOptions: MatDialogConfig, - @Inject( MAT_DIALOG_SCROLL_STRATEGY ) _scrollStrategy: ScrollStrategy, - @Optional() @SkipSelf() _parentDialog:DynamicMatDialog, - _overlayContainer: DynamicOverlayContainer) { - - super( _overlay, _injector, location, _defaultOptions, _scrollStrategy, _parentDialog, _overlayContainer ); - this._dialog = _injector.get(DynamicDialog); - this._customOverlay = _overlay; - } - - public open(component: ComponentType | TemplateRef, config?: DynamicMatDialogConfig): MatDialogRef { + public override open(component: ComponentType | TemplateRef, config?: DynamicMatDialogConfig): MatDialogRef { if (config?.containerElement) { config.containerElement.style.transform = 'translateZ(0)'; - this._customOverlay.setContainerElement( config.containerElement ); + this._customOverlay.setContainerElement(config.containerElement); } const ref = super.open(component, config); if (config?.containerElement) { @@ -73,13 +53,4 @@ export class DynamicMatDialog extends MatDialog { @Injectable() export class DynamicDialog extends Dialog { - constructor( _overlay: DynamicOverlay, - _injector: Injector, - @Inject( DEFAULT_DIALOG_CONFIG ) _defaultOptions: DialogConfig, - @Inject( MAT_DIALOG_SCROLL_STRATEGY ) _scrollStrategy: ScrollStrategy, - @Optional() @SkipSelf() _parentDialog: DynamicDialog, - _overlayContainer: DynamicOverlayContainer) { - - super( _overlay, _injector, _defaultOptions, _parentDialog, _overlayContainer, _scrollStrategy ); - } } diff --git a/ui-ngx/src/app/shared/components/dialog/dynamic/dynamic-overlay.ts b/ui-ngx/src/app/shared/components/dialog/dynamic/dynamic-overlay.ts index 5ec5408be7..4d5a6836c9 100644 --- a/ui-ngx/src/app/shared/components/dialog/dynamic/dynamic-overlay.ts +++ b/ui-ngx/src/app/shared/components/dialog/dynamic/dynamic-overlay.ts @@ -14,49 +14,16 @@ /// limitations under the License. /// -import { - Overlay, - ScrollStrategyOptions, - OverlayKeyboardDispatcher, OverlayOutsideClickDispatcher, OverlayPositionBuilder -} from '@angular/cdk/overlay'; -import { ComponentFactoryResolver, Inject, Injectable, Injector, NgZone, DOCUMENT } from '@angular/core'; +import { Overlay } from '@angular/cdk/overlay'; +import { inject, Injectable } from '@angular/core'; import { DynamicOverlayContainer } from './dynamic-overlay-container'; -import { Location } from '@angular/common'; -import { Directionality } from '@angular/cdk/bidi'; @Injectable() export class DynamicOverlay extends Overlay { - private _dynamicOverlayContainer: DynamicOverlayContainer; + private _dynamicOverlayContainer = inject(DynamicOverlayContainer); - constructor( scrollStrategies: ScrollStrategyOptions, - _overlayContainer: DynamicOverlayContainer, - _componentFactoryResolver: ComponentFactoryResolver, - _positionBuilder: OverlayPositionBuilder, - _keyboardDispatcher: OverlayKeyboardDispatcher, - _injector: Injector, - _ngZone: NgZone, - @Inject(DOCUMENT) document: Document, - _directionality: Directionality, - _location: Location, - _outsideClickDispatcher: OverlayOutsideClickDispatcher) { - - super( scrollStrategies, - _overlayContainer, - _componentFactoryResolver, - _positionBuilder, - _keyboardDispatcher, - _injector, - _ngZone, - document, - _directionality, - _location, - _outsideClickDispatcher); - - this._dynamicOverlayContainer = _overlayContainer; - } - - public setContainerElement(containerElement:HTMLElement ): void { - this._dynamicOverlayContainer.setContainerElement( containerElement ); + public setContainerElement(containerElement: HTMLElement): void { + this._dynamicOverlayContainer.setContainerElement(containerElement); } } diff --git a/ui-ngx/src/app/shared/components/image/image-gallery.component.html b/ui-ngx/src/app/shared/components/image/image-gallery.component.html index f442556b18..b31dc16920 100644 --- a/ui-ngx/src/app/shared/components/image/image-gallery.component.html +++ b/ui-ngx/src/app/shared/components/image/image-gallery.component.html @@ -74,10 +74,10 @@ mdi:file-import
", + "markdownTextPattern": "
\n
\n
\n
\n {{ 'device.devices' | translate }}\n
\n
\n {{ 'widgets.devices.view-docs' | translate }}\n {{ 'device.add' | translate }}\n {{ 'device.add' | translate }}\n
\n
\n
\n \n
\n
widgets.devices.inactive
\n
\n
\n
${inactiveDevices:0}
\n
\n \n \n
\n
widgets.devices.active
\n
\n
\n
${activeDevices:0}
\n
\n \n \n
\n
widgets.devices.total
\n
\n
\n
${totalDevices:0}
\n
\n \n
\n
\n
", "applyDefaultMarkdownStyle": false, "markdownCss": ".tb-card-content {\n width: 100%;\n height: 100%;\n display: flex;\n flex-direction: row;\n}\n\n.tb-content-container {\n flex: 1;\n display: flex;\n flex-direction: column;\n justify-content: flex-start;\n gap: 12px;\n}\n\n.tb-card-header {\n height: 36px;\n display: flex;\n flex-direction: row;\n justify-content: space-between;\n}\n\n.tb-item-cards {\n flex: 1;\n display: flex;\n flex-direction: row;\n gap: 12px;\n overflow: hidden;\n}\n\na.tb-item-card {\n flex: 1;\n display: flex;\n flex-direction: column;\n padding: 8px 12px;\n border: 1px solid;\n border-radius: 10px;\n margin-bottom: 12px;\n overflow: hidden;\n justify-content: space-evenly;\n}\n\na.tb-item-card.tb-inactive {\n background: rgba(209, 39, 48, 0.04);\n border-color: rgba(209, 39, 48, 0.06);\n}\n\na.tb-item-card.tb-active {\n background: rgba(48, 86, 128, 0.04);\n border-color: rgba(48, 86, 128, 0.12);\n}\n\na.tb-item-card.tb-total {\n background: rgba(0, 0, 0, 0.01);\n border-color: rgba(0, 0, 0, 0.05);\n}\n\n.tb-item-title-container {\n display: grid;\n}\n\n.tb-item-title {\n font-weight: 400;\n font-size: 14px;\n line-height: 20px;\n letter-spacing: 0.2px;\n white-space: nowrap;\n overflow: hidden;\n text-overflow: ellipsis; \n color: rgba(0, 0, 0, 0.76);\n}\n\n.tb-item-title.tb-home-widget-link:after {\n position: absolute;\n right: 0;\n}\n\na.tb-item-card:hover .tb-item-title.tb-home-widget-link:after { \n color: rgba(0, 0, 0, 0.38);\n}\n\na.tb-item-card:hover {\n box-shadow: 0px 4px 10px rgba(23, 33, 90, 0.08);\n}\n\n.tb-count-container {\n flex: 1;\n display: flex;\n flex-direction: column;\n align-items: flex-start;\n justify-content: center;\n}\n\n.tb-count {\n font-style: normal;\n font-weight: 500;\n font-size: 24px;\n line-height: 36px;\n white-space: nowrap;\n color: rgba(0, 0, 0, 0.87);\n}\n\n@media screen and (max-width: 959px) {\n .tb-item-cards {\n flex-direction: column;\n }\n a.tb-item-card {\n margin-bottom: 0;\n }\n}\n\n@media screen and (max-width: 1279px) {\n a.tb-item-card {\n flex-direction: row;\n align-items: center;\n }\n .tb-item-title.tb-home-widget-link:after {\n position: relative;\n }\n .tb-count-container {\n align-items: flex-end;\n }\n}\n\n@media screen and (min-width: 960px) and (max-width: 1819px) {\n .tb-item-title {\n font-size: 11px;\n line-height: 16px;\n }\n .tb-count {\n font-size: 16px;\n line-height: 24px;\n }\n a.tb-item-card {\n padding: 4px 8px;\n margin-bottom: 6px;\n }\n a.tb-item-card:hover {\n box-shadow: 0px 2px 5px rgba(23, 33, 90, 0.08);\n }\n}\n" }, diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 116ff4b4c4..5c527c9698 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1102,6 +1102,7 @@ "expression": "Expression", "no-found": "No calculated fields found", "list": "{ count, plural, =1 {One calculated field} other {List of # calculated fields} }", + "add": "Add calculated field", "selected-fields": "{ count, plural, =1 {1 calculated field} other {# calculated fields} } selected", "type": { "simple": "Simple",