Browse Source

Save attributes strategies: merge with master

pull/12764/head
Dmytro Skarzhynets 1 year ago
parent
commit
e2cf33e6cb
No known key found for this signature in database GPG Key ID: 2B51652F224037DF
  1. 6
      .github/workflows/check-configuration-files.yml
  2. 4
      application/pom.xml
  3. 2
      application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json
  4. 17
      application/src/main/data/upgrade/basic/schema_update.sql
  5. 14
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  6. 39
      application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java
  7. 3
      application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java
  8. 10
      application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java
  9. 21
      application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java
  10. 3
      application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java
  11. 11
      application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java
  12. 1
      application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java
  13. 2
      application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java
  14. 33
      application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java
  15. 19
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java
  16. 2
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java
  17. 4
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java
  18. 15
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java
  19. 12
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java
  20. 2
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java
  21. 5
      application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java
  22. 12
      application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java
  23. 4
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeEntityProcessor.java
  24. 82
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/BaseRuleChainProcessor.java
  25. 98
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java
  26. 5
      application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java
  27. 117
      application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java
  28. 298
      application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java
  29. 61
      application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java
  30. 284
      application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java
  31. 42
      application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java
  32. 35
      application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java
  33. 10
      application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java
  34. 45
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
  35. 14
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java
  36. 2
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java
  37. 8
      application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java
  38. 2
      application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java
  39. 180
      application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateManager.java
  40. 108
      application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java
  41. 196
      application/src/main/java/org/thingsboard/server/service/state/DefaultRuleEngineDeviceStateManager.java
  42. 6
      application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java
  43. 36
      application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java
  44. 119
      application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java
  45. 3
      application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java
  46. 5
      application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java
  47. 1
      application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java
  48. 21
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java
  49. 6
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java
  50. 6
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java
  51. 57
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java
  52. 9
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java
  53. 6
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java
  54. 10
      application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
  55. 2
      application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java
  56. 5
      application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java
  57. 1
      application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java
  58. 2
      application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java
  59. 3
      application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java
  60. 1
      application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java
  61. 66
      application/src/main/resources/thingsboard.yml
  62. 2
      application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java
  63. 72
      application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java
  64. 198
      application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java
  65. 85
      application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java
  66. 49
      application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java
  67. 27
      application/src/test/java/org/thingsboard/server/edge/AssetEdgeTest.java
  68. 1
      application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java
  69. 24
      application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java
  70. 84
      application/src/test/java/org/thingsboard/server/edge/RuleChainEdgeTest.java
  71. 17
      application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java
  72. 4
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java
  73. 30
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java
  74. 2
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java
  75. 128
      application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java
  76. 799
      application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java
  77. 93
      application/src/test/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerServiceTest.java
  78. 11
      application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java
  79. 202
      application/src/test/java/org/thingsboard/server/service/script/TbelInvokeDocsIoTest.java
  80. 61
      application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateManagerTest.java
  81. 60
      application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java
  82. 80
      application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java
  83. 128
      application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java
  84. 26
      application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java
  85. 3
      application/src/test/resources/application-test.properties
  86. 6
      application/src/test/resources/update/330/device_profile_001_out.json
  87. 2
      common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java
  88. 10
      common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java
  89. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java
  90. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java
  91. 89
      common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java
  92. 1
      common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java
  93. 26
      common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java
  94. 1
      common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java
  95. 75
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java
  96. 40
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/DataPoint.java
  97. 34
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEvent.java
  98. 21
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEventType.java
  99. 32
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java
  100. 24
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsSyncRequest.java

6
.github/workflows/check-configuration-files.yml

@ -32,14 +32,14 @@ on:
jobs: jobs:
build: build:
name: Check thingsboard.yml file name: Check thingsboard.yml file
runs-on: ubuntu-20.04 runs-on: ubuntu-22.04
steps: steps:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v2 uses: actions/checkout@v2
- name: Set up Python 3.10 - name: Set up Python 3.13
uses: actions/setup-python@v3 uses: actions/setup-python@v3
with: with:
python-version: "3.10.2" python-version: "3.13.2"
architecture: "x64" architecture: "x64"
env: env:
AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache

4
application/pom.xml

@ -124,6 +124,10 @@
<groupId>org.thingsboard.common</groupId> <groupId>org.thingsboard.common</groupId>
<artifactId>edge-api</artifactId> <artifactId>edge-api</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.thingsboard.common</groupId>
<artifactId>edqs</artifactId>
</dependency>
<dependency> <dependency>
<groupId>org.thingsboard</groupId> <groupId>org.thingsboard</groupId>
<artifactId>dao</artifactId> <artifactId>dao</artifactId>

2
application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json

@ -122,7 +122,7 @@
"type": "org.thingsboard.rule.engine.edge.TbMsgPushToCloudNode", "type": "org.thingsboard.rule.engine.edge.TbMsgPushToCloudNode",
"name": "Push to cloud", "name": "Push to cloud",
"configuration": { "configuration": {
"scope": "SERVER_SCOPE" "scope": "CLIENT_SCOPE"
}, },
"externalId": null "externalId": null
}, },

17
application/src/main/data/upgrade/basic/schema_update.sql

@ -66,3 +66,20 @@ WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode'
-- UPDATE SAVE ATTRIBUTES NODES END -- UPDATE SAVE ATTRIBUTES NODES END
ALTER TABLE api_usage_state ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; ALTER TABLE api_usage_state ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1;
-- UPDATE TENANT PROFILE CALCULATED FIELD LIMITS START
UPDATE tenant_profile
SET profile_data = profile_data
|| jsonb_build_object(
'configuration', profile_data->'configuration' || jsonb_build_object(
'maxCalculatedFieldsPerEntity', COALESCE(profile_data->'configuration'->>'maxCalculatedFieldsPerEntity', '5')::bigint,
'maxArgumentsPerCF', COALESCE(profile_data->'configuration'->>'maxArgumentsPerCF', '10')::bigint,
'maxDataPointsPerRollingArg', COALESCE(profile_data->'configuration'->>'maxDataPointsPerRollingArg', '1000')::bigint,
'maxStateSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxStateSizeInKBytes', '32')::bigint,
'maxSingleValueArgumentSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxSingleValueArgumentSizeInKBytes', '2')::bigint
)
)
WHERE profile_data->'configuration'->>'maxCalculatedFieldsPerEntity' IS NULL;
-- UPDATE TENANT PROFILE CALCULATED FIELD LIMITS END

14
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@ -33,7 +33,7 @@ import org.springframework.stereotype.Component;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.MailService;
import org.thingsboard.rule.engine.api.NotificationCenter; import org.thingsboard.rule.engine.api.NotificationCenter;
import org.thingsboard.rule.engine.api.RuleEngineDeviceStateManager; import org.thingsboard.rule.engine.api.DeviceStateManager;
import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.rule.engine.api.SmsService;
import org.thingsboard.rule.engine.api.notification.SlackService; import org.thingsboard.rule.engine.api.notification.SlackService;
import org.thingsboard.rule.engine.api.sms.SmsSenderFactory; import org.thingsboard.rule.engine.api.sms.SmsSenderFactory;
@ -109,6 +109,7 @@ import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService;
import org.thingsboard.server.service.cf.CalculatedFieldQueueService;
import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.cf.CalculatedFieldStateService;
import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache;
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry;
@ -233,7 +234,7 @@ public class ActorSystemContext {
@Autowired(required = false) @Autowired(required = false)
@Getter @Getter
private RuleEngineDeviceStateManager deviceStateManager; private DeviceStateManager deviceStateManager;
@Autowired @Autowired
@Getter @Getter
@ -544,6 +545,11 @@ public class ActorSystemContext {
@Getter @Getter
private CalculatedFieldStateService calculatedFieldStateService; private CalculatedFieldStateService calculatedFieldStateService;
@Lazy
@Autowired(required = false)
@Getter
private CalculatedFieldQueueService calculatedFieldQueueService;
@Lazy @Lazy
@Autowired(required = false) @Autowired(required = false)
@Getter @Getter
@ -644,6 +650,10 @@ public class ActorSystemContext {
@Getter @Getter
private String deviceStateNodeRateLimitConfig; private String deviceStateNodeRateLimitConfig;
@Value("${actors.calculated_fields.calculation_timeout:5}")
@Getter
private long cfCalculationResultTimeout;
@Getter @Getter
@Setter @Setter
private TbActorSystem actorSystem; private TbActorSystem actorSystem;

39
application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java

@ -242,7 +242,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
justRestored = true; justRestored = true;
} }
if (state.isSizeOk()) { if (state.isSizeOk()) {
if (state.updateState(newArgValues) || justRestored) { if (state.updateState(ctx, newArgValues) || justRestored) {
cfIdList = new ArrayList<>(cfIdList); cfIdList = new ArrayList<>(cfIdList);
cfIdList.add(ctx.getCfId()); cfIdList.add(ctx.getCfId());
processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback);
@ -274,36 +274,34 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
private void processStateIfReady(CalculatedFieldCtx ctx, List<CalculatedFieldId> cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) throws CalculatedFieldException { private void processStateIfReady(CalculatedFieldCtx ctx, List<CalculatedFieldId> cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) throws CalculatedFieldException {
CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId);
boolean stateSizeOk; boolean stateSizeChecked = false;
if (ctx.isInitialized() && state.isReady()) { try {
try { if (ctx.isInitialized() && state.isReady()) {
CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(systemContext.getCfCalculationResultTimeout(), TimeUnit.SECONDS);
state.checkStateSize(ctxId, ctx.getMaxStateSize()); state.checkStateSize(ctxId, ctx.getMaxStateSize());
stateSizeOk = state.isSizeOk(); stateSizeChecked = true;
if (stateSizeOk) { if (state.isSizeOk()) {
cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback);
if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) {
systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null); systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null);
} }
} }
} catch (Exception e) {
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).msgId(tbMsgId).msgType(tbMsgType).arguments(state.getArguments()).cause(e).build();
} }
} else { } catch (Exception e) {
state.checkStateSize(ctxId, ctx.getMaxStateSize()); throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).msgId(tbMsgId).msgType(tbMsgType).arguments(state.getArguments()).cause(e).build();
stateSizeOk = state.isSizeOk(); } finally {
if (stateSizeOk) { if (!stateSizeChecked) {
callback.onSuccess(); // State was updated but no calculation performed; state.checkStateSize(ctxId, ctx.getMaxStateSize());
}
if (state.isSizeOk()) {
cfStateService.persistState(ctxId, state, callback);
} else {
removeStateAndRaiseSizeException(ctxId, CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(), callback);
} }
}
if (stateSizeOk) {
cfStateService.persistState(ctxId, state, callback);
} else {
removeStateAndRaiseSizeException(ctxId, CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(), callback);
} }
} }
private void removeStateAndRaiseSizeException(CalculatedFieldEntityCtxId ctxId, CalculatedFieldException ex, TbCallback callback) { private void removeStateAndRaiseSizeException(CalculatedFieldEntityCtxId ctxId, CalculatedFieldException ex, TbCallback callback) throws CalculatedFieldException {
// We remove the state, but remember that it is over-sized in a local map. // We remove the state, but remember that it is over-sized in a local map.
cfStateService.removeState(ctxId, new TbCallback() { cfStateService.removeState(ctxId, new TbCallback() {
@Override @Override
@ -316,6 +314,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
callback.onFailure(ex); callback.onFailure(ex);
} }
}); });
throw ex;
} }
private Map<String, ArgumentEntry> mapToArguments(CalculatedFieldCtx ctx, List<TsKvProto> data) { private Map<String, ArgumentEntry> mapToArguments(CalculatedFieldCtx ctx, List<TsKvProto> data) {

3
application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java

@ -16,7 +16,6 @@
package org.thingsboard.server.actors.calculatedField; package org.thingsboard.server.actors.calculatedField;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.DebugModeUtil;
import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorCtx;
import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.TbActorRef;
@ -51,8 +50,6 @@ import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CopyOnWriteArrayList;
import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto;

10
application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java

@ -28,8 +28,9 @@ import org.thingsboard.rule.engine.api.NotificationCenter;
import org.thingsboard.rule.engine.api.RuleEngineAlarmService; import org.thingsboard.rule.engine.api.RuleEngineAlarmService;
import org.thingsboard.rule.engine.api.RuleEngineApiUsageStateService; import org.thingsboard.rule.engine.api.RuleEngineApiUsageStateService;
import org.thingsboard.rule.engine.api.RuleEngineAssetProfileCache; import org.thingsboard.rule.engine.api.RuleEngineAssetProfileCache;
import org.thingsboard.rule.engine.api.RuleEngineCalculatedFieldQueueService;
import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache; import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache;
import org.thingsboard.rule.engine.api.RuleEngineDeviceStateManager; import org.thingsboard.rule.engine.api.DeviceStateManager;
import org.thingsboard.rule.engine.api.RuleEngineRpcService; import org.thingsboard.rule.engine.api.RuleEngineRpcService;
import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService;
import org.thingsboard.rule.engine.api.ScriptEngine; import org.thingsboard.rule.engine.api.ScriptEngine;
@ -725,7 +726,7 @@ public class DefaultTbContext implements TbContext {
} }
@Override @Override
public RuleEngineDeviceStateManager getDeviceStateManager() { public DeviceStateManager getDeviceStateManager() {
return mainCtx.getDeviceStateManager(); return mainCtx.getDeviceStateManager();
} }
@ -902,6 +903,11 @@ public class DefaultTbContext implements TbContext {
return mainCtx.getCalculatedFieldService(); return mainCtx.getCalculatedFieldService();
} }
@Override
public RuleEngineCalculatedFieldQueueService getCalculatedFieldQueueService() {
return mainCtx.getCalculatedFieldQueueService();
}
@Override @Override
public boolean isExternalNodeForceAck() { public boolean isExternalNodeForceAck() {
return mainCtx.isExternalNodeForceAck(); return mainCtx.isExternalNodeForceAck();

21
application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java

@ -20,6 +20,8 @@ import io.swagger.v3.oas.annotations.media.Schema;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestMethod;
@ -27,6 +29,7 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.request.async.DeferredResult; import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest;
import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.id.UserId;
@ -38,6 +41,8 @@ import org.thingsboard.server.common.data.query.EntityCountQuery;
import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataPageLink;
import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.msg.edqs.EdqsApiService;
import org.thingsboard.server.common.msg.edqs.EdqsService;
import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.config.annotations.ApiOperation;
import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.query.EntityQueryService; import org.thingsboard.server.service.query.EntityQueryService;
@ -55,6 +60,10 @@ public class EntityQueryController extends BaseController {
@Autowired @Autowired
private EntityQueryService entityQueryService; private EntityQueryService entityQueryService;
@Autowired
private EdqsService edqsService;
@Autowired
private EdqsApiService edqsApiService;
private static final int MAX_PAGE_SIZE = 100; private static final int MAX_PAGE_SIZE = 100;
@ -133,4 +142,16 @@ public class EntityQueryController extends BaseController {
return entityQueryService.getKeysByQuery(getCurrentUser(), tenantId, query, isTimeseries, isAttributes, scope); return entityQueryService.getKeysByQuery(getCurrentUser(), tenantId, query, isTimeseries, isAttributes, scope);
} }
@PreAuthorize("hasAnyAuthority('SYS_ADMIN')")
@PostMapping("/edqs/system/request")
public void processSystemEdqsRequest(@RequestBody ToCoreEdqsRequest request) {
edqsService.processSystemRequest(request);
}
@PreAuthorize("hasAnyAuthority('SYS_ADMIN')")
@GetMapping("/edqs/enabled")
public boolean isEdqsApiEnabled() {
return edqsApiService.isEnabled();
}
} }

3
application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java

@ -18,13 +18,14 @@ package org.thingsboard.server.service.cf;
import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.FutureCallback;
import org.thingsboard.rule.engine.api.AttributesDeleteRequest; import org.thingsboard.rule.engine.api.AttributesDeleteRequest;
import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.AttributesSaveRequest;
import org.thingsboard.rule.engine.api.RuleEngineCalculatedFieldQueueService;
import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest;
import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest;
import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult;
import java.util.List; import java.util.List;
public interface CalculatedFieldQueueService { public interface CalculatedFieldQueueService extends RuleEngineCalculatedFieldQueueService {
/** /**
* Filter CFs based on the request entity. Push to the queue if any matching CF exist; * Filter CFs based on the request entity. Push to the queue if any matching CF exist;

11
application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java

@ -15,22 +15,29 @@
*/ */
package org.thingsboard.server.service.cf; package org.thingsboard.server.service.cf;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy; import jakarta.annotation.PreDestroy;
import org.rocksdb.Options; import org.rocksdb.Options;
import org.rocksdb.WriteOptions; import org.rocksdb.WriteOptions;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.server.utils.TbRocksDb; import org.thingsboard.server.edqs.util.TbRocksDb;
@Component @Component
@ConditionalOnExpression("'${queue.type:null}'=='in-memory'") @ConditionalOnExpression("'${queue.type:null}'=='in-memory'")
public class CfRocksDb extends TbRocksDb { public class CfRocksDb extends TbRocksDb {
public CfRocksDb(@Value("${queue.calculated_fields.rocks_db_path:${user.home}/.rocksdb/cf_states}") String path) throws Exception { public CfRocksDb(@Value("${queue.calculated_fields.rocks_db_path:${user.home}/.rocksdb/cf_states}") String path) {
super(path, new Options().setCreateIfMissing(true), new WriteOptions().setSync(true)); super(path, new Options().setCreateIfMissing(true), new WriteOptions().setSync(true));
} }
@PostConstruct
@Override
public void init() {
super.init();
}
@PreDestroy @PreDestroy
@Override @Override
public void close() { public void close() {

1
application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java

@ -117,7 +117,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache {
CalculatedField calculatedField = getCalculatedField(calculatedFieldId); CalculatedField calculatedField = getCalculatedField(calculatedFieldId);
if (calculatedField != null) { if (calculatedField != null) {
ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService, apiLimitService); ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService, apiLimitService);
ctx.init();
calculatedFieldsCtx.put(calculatedFieldId, ctx); calculatedFieldsCtx.put(calculatedFieldId, ctx);
log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx); log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx);
} }

2
application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java

@ -125,7 +125,7 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP
} }
return Futures.whenAllComplete(argFutures.values()).call(() -> { return Futures.whenAllComplete(argFutures.values()).call(() -> {
var result = createStateByType(ctx); var result = createStateByType(ctx);
result.updateState(argFutures.entrySet().stream() result.updateState(ctx, argFutures.entrySet().stream()
.collect(Collectors.toMap( .collect(Collectors.toMap(
Entry::getKey, // Keep the key as is Entry::getKey, // Keep the key as is
entry -> { entry -> {

33
application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java

@ -47,6 +47,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbAssetProfileCache;
import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import java.util.Collections;
import java.util.EnumSet; import java.util.EnumSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
@ -89,6 +90,11 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS
() -> toCalculatedFieldTelemetryMsgProto(request, result), callback); () -> toCalculatedFieldTelemetryMsgProto(request, result), callback);
} }
@Override
public void pushRequestToQueue(TimeseriesSaveRequest request, FutureCallback<Void> callback) {
pushRequestToQueue(request, null, callback);
}
@Override @Override
public void pushRequestToQueue(AttributesSaveRequest request, List<Long> result, FutureCallback<Void> callback) { public void pushRequestToQueue(AttributesSaveRequest request, List<Long> result, FutureCallback<Void> callback) {
var tenantId = request.getTenantId(); var tenantId = request.getTenantId();
@ -97,6 +103,11 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS
() -> toCalculatedFieldTelemetryMsgProto(request, result), callback); () -> toCalculatedFieldTelemetryMsgProto(request, result), callback);
} }
@Override
public void pushRequestToQueue(AttributesSaveRequest request, FutureCallback<Void> callback) {
pushRequestToQueue(request, null, callback);
}
@Override @Override
public void pushRequestToQueue(AttributesDeleteRequest request, List<String> result, FutureCallback<Void> callback) { public void pushRequestToQueue(AttributesDeleteRequest request, List<String> result, FutureCallback<Void> callback) {
var tenantId = request.getTenantId(); var tenantId = request.getTenantId();
@ -156,15 +167,19 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS
ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder();
CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType());
List<TsKvEntry> entries = request.getEntries(); List<TsKvEntry> entries = request.getEntries();
List<Long> versions = result.getVersions(); List<Long> versions = result != null ? result.getVersions() : Collections.emptyList();
for (int i = 0; i < entries.size(); i++) { for (int i = 0; i < entries.size(); i++) {
long tsVersion = versions.get(i); TsKvProto.Builder tsProtoBuilder = toTsKvProto(entries.get(i)).toBuilder();
TsKvProto tsProto = toTsKvProto(entries.get(i)).toBuilder().setVersion(tsVersion).build(); if (result != null) {
telemetryMsg.addTsData(tsProto); tsProtoBuilder.setVersion(versions.get(i));
}
telemetryMsg.addTsData(tsProtoBuilder.build());
} }
msg.setTelemetryMsg(telemetryMsg.build());
msg.setTelemetryMsg(telemetryMsg.build());
return msg.build(); return msg.build();
} }
@ -175,9 +190,11 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS
telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name()));
List<AttributeKvEntry> entries = request.getEntries(); List<AttributeKvEntry> entries = request.getEntries();
for (int i = 0; i < entries.size(); i++) { for (int i = 0; i < entries.size(); i++) {
long attrVersion = versions.get(i); AttributeValueProto.Builder attrProtoBuilder = ProtoUtils.toProto(entries.get(i)).toBuilder();
AttributeValueProto attrProto = ProtoUtils.toProto(entries.get(i)).toBuilder().setVersion(attrVersion).build(); if (versions != null) {
telemetryMsg.addAttrData(attrProto); attrProtoBuilder.setVersion(versions.get(i));
}
telemetryMsg.addAttrData(attrProtoBuilder.build());
} }
msg.setTelemetryMsg(telemetryMsg.build()); msg.setTelemetryMsg(telemetryMsg.build());

19
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java

@ -25,6 +25,8 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArgumentProto;
@Data @Data
@AllArgsConstructor @AllArgsConstructor
public abstract class BaseCalculatedFieldState implements CalculatedFieldState { public abstract class BaseCalculatedFieldState implements CalculatedFieldState {
@ -43,7 +45,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState {
} }
@Override @Override
public boolean updateState(Map<String, ArgumentEntry> argumentValues) { public boolean updateState(CalculatedFieldCtx ctx, Map<String, ArgumentEntry> argumentValues) {
if (arguments == null) { if (arguments == null) {
arguments = new HashMap<>(); arguments = new HashMap<>();
} }
@ -53,6 +55,9 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState {
for (Map.Entry<String, ArgumentEntry> entry : argumentValues.entrySet()) { for (Map.Entry<String, ArgumentEntry> entry : argumentValues.entrySet()) {
String key = entry.getKey(); String key = entry.getKey();
ArgumentEntry newEntry = entry.getValue(); ArgumentEntry newEntry = entry.getValue();
checkArgumentSize(key, newEntry, ctx);
ArgumentEntry existingEntry = arguments.get(key); ArgumentEntry existingEntry = arguments.get(key);
if (existingEntry == null || newEntry.isForceResetPrevious()) { if (existingEntry == null || newEntry.isForceResetPrevious()) {
@ -81,6 +86,18 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState {
} }
} }
@Override
public void checkArgumentSize(String name, ArgumentEntry entry, CalculatedFieldCtx ctx) {
if (entry instanceof TsRollingArgumentEntry) {
return;
}
if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) {
if (ctx.getMaxSingleValueArgumentSize() > 0 && toSingleValueArgumentProto(name, singleValueArgumentEntry).getSerializedSize() > ctx.getMaxSingleValueArgumentSize()) {
throw new IllegalArgumentException("Single value size exceeds the maximum allowed limit. The argument will not be used for calculation.");
}
}
}
protected abstract void validateNewEntry(ArgumentEntry newEntry); protected abstract void validateNewEntry(ArgumentEntry newEntry);
} }

2
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java

@ -71,6 +71,7 @@ public class CalculatedFieldCtx {
private long maxDataPointsPerRollingArg; private long maxDataPointsPerRollingArg;
private long maxStateSize; private long maxStateSize;
private long maxSingleValueArgumentSize;
public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService, ApiLimitService apiLimitService) { public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService, ApiLimitService apiLimitService) {
this.calculatedField = calculatedField; this.calculatedField = calculatedField;
@ -104,6 +105,7 @@ public class CalculatedFieldCtx {
this.maxDataPointsPerRollingArg = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); this.maxDataPointsPerRollingArg = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg);
this.maxStateSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes) * 1024; this.maxStateSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes) * 1024;
this.maxSingleValueArgumentSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxSingleValueArgumentSizeInKBytes) * 1024;
} }
public void init() { public void init() {

4
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java

@ -44,7 +44,7 @@ public interface CalculatedFieldState {
void setRequiredArguments(List<String> requiredArguments); void setRequiredArguments(List<String> requiredArguments);
boolean updateState(Map<String, ArgumentEntry> argumentValues); boolean updateState(CalculatedFieldCtx ctx, Map<String, ArgumentEntry> argumentValues);
ListenableFuture<CalculatedFieldResult> performCalculation(CalculatedFieldCtx ctx); ListenableFuture<CalculatedFieldResult> performCalculation(CalculatedFieldCtx ctx);
@ -60,4 +60,6 @@ public interface CalculatedFieldState {
void checkStateSize(CalculatedFieldEntityCtxId ctxId, long maxStateSize); void checkStateSize(CalculatedFieldEntityCtxId ctxId, long maxStateSize);
void checkArgumentSize(String name, ArgumentEntry entry, CalculatedFieldCtx ctx);
} }

15
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java

@ -20,6 +20,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.script.api.tbel.TbUtils;
import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.Output;
import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.BasicKvEntry;
@ -64,7 +65,19 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState {
double expressionResult = expr.evaluate(); double expressionResult = expr.evaluate();
Output output = ctx.getOutput(); Output output = ctx.getOutput();
return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), JacksonUtil.valueToTree(Map.of(output.getName(), expressionResult)))); Object result;
Integer decimals = output.getDecimalsByDefault();
if (decimals != null) {
if (decimals.equals(0)) {
result = TbUtils.toInt(expressionResult);
} else {
result = TbUtils.toFixed(expressionResult, decimals);
}
} else {
result = expressionResult;
}
return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), JacksonUtil.valueToTree(Map.of(output.getName(), result))));
} }
} }

12
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java

@ -42,13 +42,17 @@ public class SingleValueArgumentEntry implements ArgumentEntry {
public SingleValueArgumentEntry(TsKvProto entry) { public SingleValueArgumentEntry(TsKvProto entry) {
this.ts = entry.getTs(); this.ts = entry.getTs();
this.version = entry.getVersion(); if (entry.hasVersion()) {
this.version = entry.getVersion();
}
this.kvEntryValue = ProtoUtils.fromProto(entry.getKv()); this.kvEntryValue = ProtoUtils.fromProto(entry.getKv());
} }
public SingleValueArgumentEntry(AttributeValueProto entry) { public SingleValueArgumentEntry(AttributeValueProto entry) {
this.ts = entry.getLastUpdateTs(); this.ts = entry.getLastUpdateTs();
this.version = entry.getVersion(); if (entry.hasVersion()) {
this.version = entry.getVersion();
}
this.kvEntryValue = ProtoUtils.basicKvEntryFromProto(entry); this.kvEntryValue = ProtoUtils.basicKvEntryFromProto(entry);
} }
@ -100,10 +104,6 @@ public class SingleValueArgumentEntry implements ArgumentEntry {
if (newVersion == null || this.version == null || newVersion > this.version) { if (newVersion == null || this.version == null || newVersion > this.version) {
this.ts = singleValueEntry.getTs(); this.ts = singleValueEntry.getTs();
this.version = newVersion; this.version = newVersion;
BasicKvEntry newValue = singleValueEntry.getKvEntryValue();
if (this.kvEntryValue != null && this.kvEntryValue.getValue().equals(newValue.getValue())) {
return false;
}
this.kvEntryValue = singleValueEntry.getKvEntryValue(); this.kvEntryValue = singleValueEntry.getKvEntryValue();
return true; return true;
} }

2
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java

@ -89,7 +89,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry {
for (var e : tsRecords.entrySet()) { for (var e : tsRecords.entrySet()) {
values.add(new TbelCfTsDoubleVal(e.getKey(), e.getValue())); values.add(new TbelCfTsDoubleVal(e.getKey(), e.getValue()));
} }
return new TbelCfTsRollingArg(limit, timeWindow, values); return new TbelCfTsRollingArg(timeWindow, values);
} }
@Override @Override

5
application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java

@ -82,6 +82,11 @@ public class EdgeEventSourcingListener {
@TransactionalEventListener(fallbackExecution = true) @TransactionalEventListener(fallbackExecution = true)
public void handleEvent(SaveEntityEvent<?> event) { public void handleEvent(SaveEntityEvent<?> event) {
if (Boolean.FALSE.equals(event.getBroadcastEvent())) {
log.trace("Ignoring event {}", event);
return;
}
try { try {
if (!isValidSaveEntityEventForEdgeProcessing(event)) { if (!isValidSaveEntityEventForEdgeProcessing(event)) {
return; return;

12
application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java

@ -74,6 +74,8 @@ import org.thingsboard.server.gen.edge.v1.RequestMsgType;
import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg;
import org.thingsboard.server.gen.edge.v1.ResponseMsg; import org.thingsboard.server.gen.edge.v1.ResponseMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataRequestMsg; import org.thingsboard.server.gen.edge.v1.RuleChainMetadataRequestMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg;
import org.thingsboard.server.gen.edge.v1.SyncCompletedMsg; import org.thingsboard.server.gen.edge.v1.SyncCompletedMsg;
import org.thingsboard.server.gen.edge.v1.UplinkMsg; import org.thingsboard.server.gen.edge.v1.UplinkMsg;
import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg;
@ -820,6 +822,16 @@ public abstract class EdgeGrpcSession implements Closeable {
result.add(ctx.getAssetProcessor().processAssetMsgFromEdge(edge.getTenantId(), edge, assetUpdateMsg)); result.add(ctx.getAssetProcessor().processAssetMsgFromEdge(edge.getTenantId(), edge, assetUpdateMsg));
} }
} }
if (uplinkMsg.getRuleChainUpdateMsgCount() > 0) {
for (RuleChainUpdateMsg ruleChainUpdateMsg : uplinkMsg.getRuleChainUpdateMsgList()) {
result.add(ctx.getRuleChainProcessor().processRuleChainMsgFromEdge(edge.getTenantId(), edge, ruleChainUpdateMsg));
}
}
if (uplinkMsg.getRuleChainMetadataUpdateMsgCount() > 0) {
for (RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg : uplinkMsg.getRuleChainMetadataUpdateMsgList()) {
result.add(ctx.getRuleChainProcessor().processRuleChainMetadataMsgFromEdge(edge.getTenantId(), edge, ruleChainMetadataUpdateMsg));
}
}
if (uplinkMsg.getEntityViewUpdateMsgCount() > 0) { if (uplinkMsg.getEntityViewUpdateMsgCount() > 0) {
for (EntityViewUpdateMsg entityViewUpdateMsg : uplinkMsg.getEntityViewUpdateMsgList()) { for (EntityViewUpdateMsg entityViewUpdateMsg : uplinkMsg.getEntityViewUpdateMsgList()) {
result.add(ctx.getEntityViewProcessor().processEntityViewMsgFromEdge(edge.getTenantId(), edge, entityViewUpdateMsg)); result.add(ctx.getEntityViewProcessor().processEntityViewMsgFromEdge(edge.getTenantId(), edge, entityViewUpdateMsg));

4
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeEntityProcessor.java

@ -49,8 +49,12 @@ public class EdgeEntityProcessor extends BaseEdgeProcessor {
@Override @Override
public ListenableFuture<Void> processEntityNotification(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto edgeNotificationMsg) { public ListenableFuture<Void> processEntityNotification(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto edgeNotificationMsg) {
try { try {
EdgeId originatorEdgeId = safeGetEdgeId(edgeNotificationMsg.getOriginatorEdgeIdMSB(), edgeNotificationMsg.getOriginatorEdgeIdLSB());
EdgeEventActionType actionType = EdgeEventActionType.valueOf(edgeNotificationMsg.getAction()); EdgeEventActionType actionType = EdgeEventActionType.valueOf(edgeNotificationMsg.getAction());
EdgeId edgeId = new EdgeId(new UUID(edgeNotificationMsg.getEntityIdMSB(), edgeNotificationMsg.getEntityIdLSB())); EdgeId edgeId = new EdgeId(new UUID(edgeNotificationMsg.getEntityIdMSB(), edgeNotificationMsg.getEntityIdLSB()));
if (edgeId.equals(originatorEdgeId)) {
return Futures.immediateFuture(null);
}
switch (actionType) { switch (actionType) {
case ASSIGNED_TO_CUSTOMER: { case ASSIGNED_TO_CUSTOMER: {
CustomerId customerId = JacksonUtil.fromString(edgeNotificationMsg.getBody(), CustomerId.class); CustomerId customerId = JacksonUtil.fromString(edgeNotificationMsg.getBody(), CustomerId.class);

82
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/BaseRuleChainProcessor.java

@ -0,0 +1,82 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.edge.rpc.processor.rule;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.util.Pair;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.dao.service.DataValidator;
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg;
import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor;
import java.util.function.Function;
@Slf4j
public class BaseRuleChainProcessor extends BaseEdgeProcessor {
@Autowired
private DataValidator<RuleChain> ruleChainValidator;
protected Pair<Boolean, Boolean> saveOrUpdateRuleChain(TenantId tenantId, RuleChainId ruleChainId, RuleChainUpdateMsg ruleChainUpdateMsg, RuleChainType ruleChainType) {
boolean created = false;
RuleChain ruleChainFromDb = edgeCtx.getRuleChainService().findRuleChainById(tenantId, ruleChainId);
if (ruleChainFromDb == null) {
created = true;
}
RuleChain ruleChain = JacksonUtil.fromString(ruleChainUpdateMsg.getEntity(), RuleChain.class, true);
if (ruleChain == null) {
throw new RuntimeException("[{" + tenantId + "}] ruleChainUpdateMsg {" + ruleChainUpdateMsg + "} cannot be converted to rule chain");
}
boolean isRoot = ruleChain.isRoot();
if (RuleChainType.CORE.equals(ruleChainType)) {
ruleChain.setRoot(false);
} else {
ruleChain.setRoot(ruleChainFromDb == null ? false : ruleChainFromDb.isRoot());
}
ruleChain.setType(ruleChainType);
ruleChainValidator.validate(ruleChain, RuleChain::getTenantId);
if (created) {
ruleChain.setId(ruleChainId);
}
edgeCtx.getRuleChainService().saveRuleChain(ruleChain, true, false);
return Pair.of(created, isRoot);
}
protected void saveOrUpdateRuleChainMetadata(TenantId tenantId, RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg) {
RuleChainMetaData ruleChainMetadata = JacksonUtil.fromString(ruleChainMetadataUpdateMsg.getEntity(), RuleChainMetaData.class, true);
if (ruleChainMetadata == null) {
throw new RuntimeException("[{" + tenantId + "}] ruleChainMetadataUpdateMsg {" + ruleChainMetadataUpdateMsg + "} cannot be converted to rule chain metadata");
}
if (!ruleChainMetadata.getNodes().isEmpty()) {
ruleChainMetadata.setVersion(null);
for (RuleNode ruleNode : ruleChainMetadata.getNodes()) {
ruleNode.setRuleChainId(null);
ruleNode.setId(null);
}
edgeCtx.getRuleChainService().saveRuleChainMetaData(tenantId, ruleChainMetadata, Function.identity(), true);
}
}
}

98
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java

@ -15,29 +15,123 @@
*/ */
package org.thingsboard.server.service.edge.rpc.processor.rule; package org.thingsboard.server.service.edge.rpc.processor.rule;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.data.util.Pair;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.EdgeUtils;
import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEvent;
import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg;
import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.edge.v1.UpdateMsgType;
import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils;
import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor;
import java.util.UUID;
import static org.thingsboard.server.dao.edge.EdgeServiceImpl.EDGE_IS_ROOT_BODY_KEY; import static org.thingsboard.server.dao.edge.EdgeServiceImpl.EDGE_IS_ROOT_BODY_KEY;
@Slf4j @Slf4j
@Component @Component
@TbCoreComponent @TbCoreComponent
public class RuleChainEdgeProcessor extends BaseEdgeProcessor { public class RuleChainEdgeProcessor extends BaseRuleChainProcessor {
public ListenableFuture<Void> processRuleChainMsgFromEdge(TenantId tenantId, Edge edge, RuleChainUpdateMsg ruleChainUpdateMsg) {
log.trace("[{}] executing processRuleChainMsgFromEdge [{}] from edge [{}]", tenantId, ruleChainUpdateMsg, edge.getName());
RuleChainId ruleChainId = new RuleChainId(new UUID(ruleChainUpdateMsg.getIdMSB(), ruleChainUpdateMsg.getIdLSB()));
try {
edgeSynchronizationManager.getEdgeId().set(edge.getId());
switch (ruleChainUpdateMsg.getMsgType()) {
case ENTITY_CREATED_RPC_MESSAGE:
case ENTITY_UPDATED_RPC_MESSAGE:
return saveOrUpdateRuleChain(tenantId, ruleChainId, ruleChainUpdateMsg, edge);
case ENTITY_DELETED_RPC_MESSAGE:
RuleChain ruleChainToDelete = edgeCtx.getRuleChainService().findRuleChainById(tenantId, ruleChainId);
if (ruleChainToDelete != null) {
edgeCtx.getRuleChainService().unassignRuleChainFromEdge(tenantId, ruleChainId, edge.getId(), false);
}
return Futures.immediateFuture(null);
case UNRECOGNIZED:
default:
return handleUnsupportedMsgType(ruleChainUpdateMsg.getMsgType());
}
} catch (DataValidationException e) {
if (e.getMessage().contains("limit reached")) {
log.warn("[{}] Number of allowed rule chains violated {}", tenantId, ruleChainUpdateMsg, e);
return Futures.immediateFuture(null);
} else {
return Futures.immediateFailedFuture(e);
}
} finally {
edgeSynchronizationManager.getEdgeId().remove();
}
}
private ListenableFuture<Void> saveOrUpdateRuleChain(TenantId tenantId, RuleChainId ruleChainId, RuleChainUpdateMsg ruleChainUpdateMsg, Edge edge) {
try {
Pair<Boolean, Boolean> resultPair = super.saveOrUpdateRuleChain(tenantId, ruleChainId, ruleChainUpdateMsg, RuleChainType.EDGE);
Boolean created = resultPair.getFirst();
if (created) {
createRelationFromEdge(tenantId, edge.getId(), ruleChainId);
pushRuleChainCreatedEventToRuleEngine(tenantId, edge, ruleChainId, ruleChainUpdateMsg.getEntity());
edgeCtx.getRuleChainService().assignRuleChainToEdge(tenantId, ruleChainId, edge.getId());
}
Boolean isRoot = resultPair.getSecond();
if (isRoot) {
edge = edgeCtx.getEdgeService().findEdgeById(tenantId, edge.getId());
edgeCtx.getEdgeService().setEdgeRootRuleChain(tenantId, edge, ruleChainId);
}
} catch (Exception e) {
log.error("Failed to save or update rule chain", e);
return Futures.immediateFailedFuture(e);
}
return Futures.immediateFuture(null);
}
private void pushRuleChainCreatedEventToRuleEngine(TenantId tenantId, Edge edge, RuleChainId ruleChainId, String ruleChainAsString) {
try {
TbMsgMetaData msgMetaData = getEdgeActionTbMsgMetaData(edge, null);
pushEntityEventToRuleEngine(tenantId, ruleChainId, null, TbMsgType.ENTITY_CREATED, ruleChainAsString, msgMetaData);
} catch (Exception e) {
log.warn("[{}][{}] Failed to push rule chain action to rule engine: {}", tenantId, ruleChainId, TbMsgType.ENTITY_CREATED.name(), e);
}
}
public ListenableFuture<Void> processRuleChainMetadataMsgFromEdge(TenantId tenantId, Edge edge, RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg) {
log.trace("[{}] executing processRuleChainMetadataMsgFromEdge [{}] from edge [{}]", tenantId, ruleChainMetadataUpdateMsg, edge.getName());
try {
edgeSynchronizationManager.getEdgeId().set(edge.getId());
switch (ruleChainMetadataUpdateMsg.getMsgType()) {
case ENTITY_CREATED_RPC_MESSAGE:
case ENTITY_UPDATED_RPC_MESSAGE:
saveOrUpdateRuleChainMetadata(tenantId, ruleChainMetadataUpdateMsg);
return Futures.immediateFuture(null);
case UNRECOGNIZED:
default:
return handleUnsupportedMsgType(ruleChainMetadataUpdateMsg.getMsgType());
}
} catch (Exception e) {
String errMsg = String.format("Can't process rule chain metadata update msg %s", ruleChainMetadataUpdateMsg);
log.error(errMsg, e);
return Futures.immediateFailedFuture(new RuntimeException(errMsg, e));
} finally {
edgeSynchronizationManager.getEdgeId().remove();
}
}
@Override @Override
public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) {

5
application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java

@ -152,8 +152,7 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService {
entityData = new HashMap<>(); entityData = new HashMap<>();
attributes = JacksonUtil.newObjectNode(); attributes = JacksonUtil.newObjectNode();
for (AttributeKvEntry attr : ssAttributes) { for (AttributeKvEntry attr : ssAttributes) {
if (DefaultDeviceStateService.PERSISTENT_ATTRIBUTES.contains(attr.getKey()) if (DefaultDeviceStateService.ACTIVITY_KEYS_WITHOUT_INACTIVITY_TIMEOUT.contains(attr.getKey())) {
&& !DefaultDeviceStateService.INACTIVITY_TIMEOUT.equals(attr.getKey())) {
continue; continue;
} }
if (attr.getDataType() == DataType.BOOLEAN && attr.getBooleanValue().isPresent()) { if (attr.getDataType() == DataType.BOOLEAN && attr.getBooleanValue().isPresent()) {
@ -200,7 +199,7 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService {
} }
Map<Long, Map<String, Object>> tsData = new HashMap<>(); Map<Long, Map<String, Object>> tsData = new HashMap<>();
for (TsKvEntry tsKvEntry : tsKvEntries) { for (TsKvEntry tsKvEntry : tsKvEntries) {
if (DefaultDeviceStateService.PERSISTENT_ATTRIBUTES.contains(tsKvEntry.getKey())) { if (DefaultDeviceStateService.ACTIVITY_KEYS_WITH_INACTIVITY_TIMEOUT.contains(tsKvEntry.getKey())) {
continue; continue;
} }
tsData.computeIfAbsent(tsKvEntry.getTs(), k -> new HashMap<>()).put(tsKvEntry.getKey(), tsKvEntry.getValue()); tsData.computeIfAbsent(tsKvEntry.getTs(), k -> new HashMap<>()).put(tsKvEntry.getKey(), tsKvEntry.getValue());

117
application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java

@ -0,0 +1,117 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.edqs;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.edqs.query.EdqsRequest;
import org.thingsboard.server.common.data.edqs.query.EdqsResponse;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.edqs.EdqsApiService;
import org.thingsboard.server.edqs.state.EdqsPartitionService;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg;
import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.provider.EdqsClientQueueFactory;
import java.util.UUID;
@Service
@Slf4j
@RequiredArgsConstructor
@ConditionalOnExpression("'${queue.edqs.api.supported:true}' == 'true' && ('${service.type:null}' == 'monolith' || '${service.type:null}' == 'tb-core')")
public class DefaultEdqsApiService implements EdqsApiService {
private final EdqsPartitionService edqsPartitionService;
private final EdqsClientQueueFactory queueFactory;
private TbQueueRequestTemplate<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> requestTemplate;
@Value("${queue.edqs.api.auto_enable:true}")
private boolean autoEnable;
private Boolean apiEnabled = null;
@PostConstruct
private void init() {
requestTemplate = queueFactory.createEdqsRequestTemplate();
requestTemplate.init();
}
@Override
public ListenableFuture<EdqsResponse> processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) {
var requestMsg = ToEdqsMsg.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setTs(System.currentTimeMillis())
.setRequestMsg(TransportProtos.EdqsRequestMsg.newBuilder()
.setValue(JacksonUtil.toString(request))
.build());
if (customerId != null && !customerId.isNullUid()) {
requestMsg.setCustomerIdMSB(customerId.getId().getMostSignificantBits());
requestMsg.setCustomerIdLSB(customerId.getId().getLeastSignificantBits());
}
Integer partition = edqsPartitionService.resolvePartition(tenantId);
ListenableFuture<TbProtoQueueMsg<FromEdqsMsg>> resultFuture = requestTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), requestMsg.build()), partition);
return Futures.transform(resultFuture, msg -> {
TransportProtos.EdqsResponseMsg responseMsg = msg.getValue().getResponseMsg();
return JacksonUtil.fromString(responseMsg.getValue(), EdqsResponse.class);
}, MoreExecutors.directExecutor());
}
@Override
public boolean isEnabled() {
return Boolean.TRUE.equals(apiEnabled);
}
@Override
public void setEnabled(boolean enabled) {
if (enabled) {
log.info("Enabling EDQS API");
} else {
log.info("Disabling EDQS API");
}
apiEnabled = enabled;
}
@Override
public boolean isSupported() {
return true;
}
@Override
public boolean isAutoEnable() {
return autoEnable;
}
@PreDestroy
private void stop() {
requestTemplate.stop();
}
}

298
application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java

@ -0,0 +1,298 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.edqs;
import com.google.protobuf.ByteString;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ObjectType;
import org.thingsboard.server.common.data.edqs.EdqsEventType;
import org.thingsboard.server.common.data.edqs.EdqsObject;
import org.thingsboard.server.common.data.edqs.EdqsSyncRequest;
import org.thingsboard.server.common.data.edqs.Entity;
import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg;
import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
import org.thingsboard.server.common.data.kv.JsonDataEntry;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.msg.edqs.EdqsApiService;
import org.thingsboard.server.common.msg.edqs.EdqsService;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.edqs.processor.EdqsProducer;
import org.thingsboard.server.edqs.state.EdqsPartitionService;
import org.thingsboard.server.edqs.util.EdqsConverter;
import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsCoreServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg;
import org.thingsboard.server.queue.discovery.HashPartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.discovery.TopicService;
import org.thingsboard.server.queue.edqs.EdqsQueue;
import org.thingsboard.server.queue.environment.DistributedLock;
import org.thingsboard.server.queue.environment.DistributedLockService;
import org.thingsboard.server.queue.provider.EdqsClientQueueFactory;
import org.thingsboard.server.queue.util.AfterStartUp;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
@Service
@RequiredArgsConstructor
@Slf4j
@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "true")
public class DefaultEdqsService implements EdqsService {
private final EdqsClientQueueFactory queueFactory;
private final EdqsConverter edqsConverter;
private final EdqsSyncService edqsSyncService;
private final EdqsApiService edqsApiService;
private final DistributedLockService distributedLockService;
private final AttributesService attributesService;
private final EdqsPartitionService edqsPartitionService;
private final TopicService topicService;
private final TbServiceInfoProvider serviceInfoProvider;
@Autowired @Lazy
private TbClusterService clusterService;
@Autowired @Lazy
private HashPartitionService hashPartitionService;
private EdqsProducer eventsProducer;
private ExecutorService executor;
private DistributedLock syncLock;
@PostConstruct
private void init() {
executor = ThingsBoardExecutors.newWorkStealingPool(12, getClass());
eventsProducer = EdqsProducer.builder()
.queue(EdqsQueue.EVENTS)
.partitionService(edqsPartitionService)
.topicService(topicService)
.producer(queueFactory.createEdqsMsgProducer(EdqsQueue.EVENTS))
.build();
syncLock = distributedLockService.getLock("edqs_sync");
}
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
public void onStartUp() {
if (!serviceInfoProvider.isService(ServiceType.TB_CORE)) {
return;
}
executor.submit(() -> {
try {
EdqsSyncState syncState = getSyncState();
if (edqsSyncService.isSyncNeeded() || syncState == null || syncState.getStatus() != EdqsSyncStatus.FINISHED) {
if (hashPartitionService.isSystemPartitionMine(ServiceType.TB_CORE)) {
processSystemRequest(ToCoreEdqsRequest.builder()
.syncRequest(new EdqsSyncRequest())
.build());
}
} else if (edqsApiService.isSupported() && edqsApiService.isAutoEnable()) {
// only if topic/RocksDB is not empty and sync is finished
edqsApiService.setEnabled(true);
}
} catch (Throwable e) {
log.error("Failed to start EDQS service", e);
}
});
}
@Override
public void processSystemRequest(ToCoreEdqsRequest request) {
log.info("Processing system request {}", request);
if (request.getSyncRequest() != null) {
saveSyncState(EdqsSyncStatus.REQUESTED);
}
broadcast(request.toInternalMsg());
}
@Override
public void processSystemMsg(ToCoreEdqsMsg msg) {
executor.submit(() -> {
log.info("Processing system msg {}", msg);
try {
if (msg.getApiEnabled() != null) {
edqsApiService.setEnabled(msg.getApiEnabled());
}
if (msg.getSyncRequest() != null) {
syncLock.lock();
try {
EdqsSyncState syncState = getSyncState();
if (syncState != null && syncState.getStatus() == EdqsSyncStatus.FINISHED) {
log.info("EDQS sync is already finished");
return;
}
saveSyncState(EdqsSyncStatus.STARTED);
edqsSyncService.sync();
saveSyncState(EdqsSyncStatus.FINISHED);
if (edqsApiService.isSupported())
if (edqsApiService.isAutoEnable()) {
log.info("EDQS sync is finished, auto-enabling API");
broadcast(ToCoreEdqsMsg.builder()
.apiEnabled(Boolean.TRUE)
.build());
} else {
log.info("EDQS sync is finished, but leaving API disabled");
}
} catch (Exception e) {
log.error("Failed to complete sync", e);
saveSyncState(EdqsSyncStatus.FAILED);
} finally {
syncLock.unlock();
}
}
} catch (Throwable e) {
log.error("Failed to process msg {}", msg, e);
}
});
}
@Override
public void onUpdate(TenantId tenantId, EntityId entityId, Object entity) {
EntityType entityType = entityId.getEntityType();
ObjectType objectType = ObjectType.fromEntityType(entityType);
if (!isEdqsType(tenantId, objectType)) {
log.trace("[{}][{}] Ignoring update event, type {} not supported", tenantId, entityId, entityType);
return;
}
onUpdate(tenantId, objectType, edqsConverter.toEntity(entityType, entity));
}
@Override
public void onUpdate(TenantId tenantId, ObjectType objectType, EdqsObject object) {
processEvent(tenantId, objectType, EdqsEventType.UPDATED, object);
}
@Override
public void onDelete(TenantId tenantId, EntityId entityId) {
EntityType entityType = entityId.getEntityType();
ObjectType objectType = ObjectType.fromEntityType(entityType);
if (!isEdqsType(tenantId, objectType)) {
log.trace("[{}][{}] Ignoring deletion event, type {} not supported", tenantId, entityId, entityType);
return;
}
onDelete(tenantId, objectType, new Entity(entityType, entityId.getId(), Long.MAX_VALUE));
}
@Override
public void onDelete(TenantId tenantId, ObjectType objectType, EdqsObject object) {
processEvent(tenantId, objectType, EdqsEventType.DELETED, object);
}
protected void processEvent(TenantId tenantId, ObjectType objectType, EdqsEventType eventType, EdqsObject object) {
executor.submit(() -> {
try {
String key = object.key();
Long version = object.version();
EdqsEventMsg.Builder eventMsg = EdqsEventMsg.newBuilder()
.setKey(key)
.setObjectType(objectType.name())
.setData(ByteString.copyFrom(edqsConverter.serialize(objectType, object)))
.setEventType(eventType.name());
if (version != null) {
eventMsg.setVersion(version);
}
eventsProducer.send(tenantId, objectType, key, ToEdqsMsg.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setTs(System.currentTimeMillis())
.setEventMsg(eventMsg)
.build());
} catch (Throwable e) {
log.error("[{}] Failed to push {} event for {} {}", tenantId, eventType, objectType, object, e);
}
});
}
private boolean isEdqsType(TenantId tenantId, ObjectType objectType) {
if (objectType == null) {
return false;
}
if (!tenantId.isSysTenantId()) {
return ObjectType.edqsTypes.contains(objectType);
} else {
return ObjectType.edqsSystemTypes.contains(objectType);
}
}
private void broadcast(ToCoreEdqsMsg msg) {
clusterService.broadcastToCore(ToCoreNotificationMsg.newBuilder()
.setToEdqsCoreServiceMsg(ToEdqsCoreServiceMsg.newBuilder()
.setValue(ByteString.copyFrom(JacksonUtil.writeValueAsBytes(msg))))
.build());
}
@SneakyThrows
private EdqsSyncState getSyncState() {
EdqsSyncState state = attributesService.find(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, AttributeScope.SERVER_SCOPE, "edqsSyncState").get(30, TimeUnit.SECONDS)
.flatMap(KvEntry::getJsonValue)
.map(value -> JacksonUtil.fromString(value, EdqsSyncState.class))
.orElse(null);
log.info("EDQS sync state: {}", state);
return state;
}
@SneakyThrows
private void saveSyncState(EdqsSyncStatus status) {
EdqsSyncState state = new EdqsSyncState(status);
log.info("New EDQS sync state: {}", state);
attributesService.save(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, AttributeScope.SERVER_SCOPE, new BaseAttributeKvEntry(
new JsonDataEntry("edqsSyncState", JacksonUtil.toString(state)),
System.currentTimeMillis())).get(30, TimeUnit.SECONDS);
}
@PreDestroy
private void stop() {
executor.shutdown();
eventsProducer.stop();
}
@Data
@AllArgsConstructor
@NoArgsConstructor
private static class EdqsSyncState {
private EdqsSyncStatus status;
}
private enum EdqsSyncStatus {
REQUESTED,
STARTED,
FINISHED,
FAILED
}
}

61
application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java

@ -0,0 +1,61 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.edqs;
import lombok.RequiredArgsConstructor;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import org.springframework.transaction.event.TransactionalEventListener;
import org.thingsboard.server.common.data.ObjectType;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.msg.edqs.EdqsService;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.RelationActionEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "true")
public class EdqsListener {
private final EdqsService edqsService;
@TransactionalEventListener(fallbackExecution = true)
public void onUpdate(SaveEntityEvent<?> event) {
if (event.getEntityId() == null || event.getEntity() == null) {
return;
}
edqsService.onUpdate(event.getTenantId(), event.getEntityId(), event.getEntity());
}
@TransactionalEventListener(fallbackExecution = true)
public void onDelete(DeleteEntityEvent<?> event) {
if (event.getEntityId() == null) {
return;
}
edqsService.onDelete(event.getTenantId(), event.getEntityId());
}
@TransactionalEventListener(fallbackExecution = true)
public void handleEvent(RelationActionEvent relationEvent) {
if (relationEvent.getActionType() == ActionType.RELATION_ADD_OR_UPDATE) {
edqsService.onUpdate(relationEvent.getTenantId(), ObjectType.RELATION, relationEvent.getRelation());
} else if (relationEvent.getActionType() == ActionType.RELATION_DELETED) {
edqsService.onDelete(relationEvent.getTenantId(), ObjectType.RELATION, relationEvent.getRelation());
}
}
}

284
application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java

@ -0,0 +1,284 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.edqs;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ObjectType;
import org.thingsboard.server.common.data.edqs.AttributeKv;
import org.thingsboard.server.common.data.edqs.EdqsEventType;
import org.thingsboard.server.common.data.edqs.EdqsObject;
import org.thingsboard.server.common.data.edqs.Entity;
import org.thingsboard.server.common.data.edqs.LatestTsKv;
import org.thingsboard.server.common.data.edqs.fields.EntityFields;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageDataIterable;
import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.dao.Dao;
import org.thingsboard.server.dao.attributes.AttributesDao;
import org.thingsboard.server.dao.dictionary.KeyDictionaryDao;
import org.thingsboard.server.dao.entity.EntityDaoRegistry;
import org.thingsboard.server.dao.model.sql.AttributeKvEntity;
import org.thingsboard.server.dao.model.sql.RelationEntity;
import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry;
import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity;
import org.thingsboard.server.dao.sql.relation.RelationRepository;
import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV;
import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV;
import static org.thingsboard.server.common.data.ObjectType.RELATION;
import static org.thingsboard.server.common.data.ObjectType.edqsTenantTypes;
@Slf4j
public abstract class EdqsSyncService {
@Value("${queue.edqs.sync.entity_batch_size:10000}")
private int entityBatchSize;
@Value("${queue.edqs.sync.ts_batch_size:10000}")
private int tsBatchSize;
@Autowired
private EntityDaoRegistry entityDaoRegistry;
@Autowired
private AttributesDao attributesDao;
@Autowired
private KeyDictionaryDao keyDictionaryDao;
@Autowired
private RelationRepository relationRepository;
@Autowired
private TsKvLatestRepository tsKvLatestRepository;
@Autowired
@Lazy
private DefaultEdqsService edqsService;
private final ConcurrentHashMap<UUID, EntityIdInfo> entityInfoMap = new ConcurrentHashMap<>();
private final ConcurrentHashMap<Integer, String> keys = new ConcurrentHashMap<>();
private final Map<ObjectType, AtomicInteger> counters = new ConcurrentHashMap<>();
public abstract boolean isSyncNeeded();
public void sync() {
log.info("Synchronizing data to EDQS");
long startTs = System.currentTimeMillis();
counters.clear();
syncTenantEntities();
syncRelations();
loadKeyDictionary();
syncAttributes();
syncLatestTimeseries();
counters.clear();
log.info("Finishing synchronizing data to EDQS in {} ms", (System.currentTimeMillis() - startTs));
}
private void process(TenantId tenantId, ObjectType type, EdqsObject object) {
AtomicInteger counter = counters.computeIfAbsent(type, t -> new AtomicInteger());
if (counter.incrementAndGet() % 10000 == 0) {
log.info("Processed {} {} objects", counter.get(), type);
}
edqsService.processEvent(tenantId, type, EdqsEventType.UPDATED, object);
}
private void syncTenantEntities() {
for (ObjectType type : edqsTenantTypes) {
log.info("Synchronizing {} entities to EDQS", type);
long ts = System.currentTimeMillis();
EntityType entityType = type.toEntityType();
Dao<?> dao = entityDaoRegistry.getDao(entityType);
UUID lastId = UUID.fromString("00000000-0000-0000-0000-000000000000");
while (true) {
var batch = dao.findNextBatch(lastId, entityBatchSize);
if (batch.isEmpty()) {
break;
}
for (EntityFields entityFields : batch) {
TenantId tenantId = TenantId.fromUUID(entityFields.getTenantId());
entityInfoMap.put(entityFields.getId(), new EntityIdInfo(entityType, tenantId));
process(tenantId, type, new Entity(entityType, entityFields));
}
EntityFields lastRecord = batch.get(batch.size() - 1);
lastId = lastRecord.getId();
}
log.info("Finished synchronizing {} entities to EDQS in {} ms", type, (System.currentTimeMillis() - ts));
}
}
private void syncRelations() {
log.info("Synchronizing relations to EDQS");
long ts = System.currentTimeMillis();
UUID lastFromEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000");
String lastFromEntityType = "";
String lastRelationTypeGroup = "";
String lastRelationType = "";
UUID lastToEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000");
String lastToEntityType = "";
while (true) {
List<RelationEntity> batch = relationRepository.findNextBatch(lastFromEntityId, lastFromEntityType, lastRelationTypeGroup,
lastRelationType, lastToEntityId, lastToEntityType, entityBatchSize);
if (batch.isEmpty()) {
break;
}
processRelationBatch(batch);
RelationEntity lastRecord = batch.get(batch.size() - 1);
lastFromEntityId = lastRecord.getFromId();
lastFromEntityType = lastRecord.getFromType();
lastRelationTypeGroup = lastRecord.getRelationTypeGroup();
lastRelationType = lastRecord.getRelationType();
lastToEntityId = lastRecord.getToId();
lastToEntityType = lastRecord.getToType();
}
log.info("Finished synchronizing relations to EDQS in {} ms", (System.currentTimeMillis() - ts));
}
private void processRelationBatch(List<RelationEntity> relations) {
for (RelationEntity relation : relations) {
if (RelationTypeGroup.COMMON.name().equals(relation.getRelationTypeGroup())) {
EntityIdInfo entityIdInfo = entityInfoMap.get(relation.getFromId());
if (entityIdInfo != null) {
process(entityIdInfo.tenantId(), RELATION, relation.toData());
} else {
log.info("Relation from id not found: {} ", relation);
}
}
}
}
private void loadKeyDictionary() {
log.info("Loading key dictionary");
long ts = System.currentTimeMillis();
var keyDictionaryEntries = new PageDataIterable<>(keyDictionaryDao::findAll, 10000);
for (KeyDictionaryEntry keyDictionaryEntry : keyDictionaryEntries) {
keys.put(keyDictionaryEntry.getKeyId(), keyDictionaryEntry.getKey());
}
log.info("Finished loading key dictionary in {} ms", (System.currentTimeMillis() - ts));
}
private void syncAttributes() {
log.info("Synchronizing attributes to EDQS");
long ts = System.currentTimeMillis();
UUID lastEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000");
int lastAttributeType = Integer.MIN_VALUE;
int lastAttributeKey = Integer.MIN_VALUE;
while (true) {
List<AttributeKvEntity> batch = attributesDao.findNextBatch(lastEntityId, lastAttributeType, lastAttributeKey, tsBatchSize);
if (batch.isEmpty()) {
break;
}
processAttributeBatch(batch);
AttributeKvEntity lastRecord = batch.get(batch.size() - 1);
lastEntityId = lastRecord.getId().getEntityId();
lastAttributeType = lastRecord.getId().getAttributeType();
lastAttributeKey = lastRecord.getId().getAttributeKey();
}
log.info("Finished synchronizing attributes to EDQS in {} ms", (System.currentTimeMillis() - ts));
}
private void processAttributeBatch(List<AttributeKvEntity> batch) {
for (AttributeKvEntity attribute : batch) {
attribute.setStrKey(getStrKeyOrFetchFromDb(attribute.getId().getAttributeKey()));
UUID entityId = attribute.getId().getEntityId();
EntityIdInfo entityIdInfo = entityInfoMap.get(entityId);
if (entityIdInfo == null) {
log.debug("Skipping attribute with entity UUID {} as it is not found in entityInfoMap", entityId);
continue;
}
AttributeKv attributeKv = new AttributeKv(
EntityIdFactory.getByTypeAndUuid(entityIdInfo.entityType(), entityId),
AttributeScope.valueOf(attribute.getId().getAttributeType()),
attribute.toData(),
attribute.getVersion());
process(entityIdInfo.tenantId(), ATTRIBUTE_KV, attributeKv);
}
}
private void syncLatestTimeseries() {
log.info("Synchronizing latest timeseries to EDQS");
long ts = System.currentTimeMillis();
UUID lastEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000");
int lastKey = Integer.MIN_VALUE;
while (true) {
List<TsKvLatestEntity> batch = tsKvLatestRepository.findNextBatch(lastEntityId, lastKey, tsBatchSize);
if (batch.isEmpty()) {
break;
}
processTsKvLatestBatch(batch);
TsKvLatestEntity lastRecord = batch.get(batch.size() - 1);
lastEntityId = lastRecord.getEntityId();
lastKey = lastRecord.getKey();
}
log.info("Finished synchronizing latest timeseries to EDQS in {} ms", (System.currentTimeMillis() - ts));
}
private void processTsKvLatestBatch(List<TsKvLatestEntity> tsKvLatestEntities) {
for (TsKvLatestEntity tsKvLatestEntity : tsKvLatestEntities) {
try {
String strKey = getStrKeyOrFetchFromDb(tsKvLatestEntity.getKey());
if (strKey == null) {
log.debug("Skipping latest timeseries with key {} as it is not found in key dictionary", tsKvLatestEntity.getKey());
continue;
}
tsKvLatestEntity.setStrKey(strKey);
UUID entityUuid = tsKvLatestEntity.getEntityId();
EntityIdInfo entityIdInfo = entityInfoMap.get(entityUuid);
if (entityIdInfo != null) {
EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityIdInfo.entityType(), entityUuid);
LatestTsKv latestTsKv = new LatestTsKv(entityId, tsKvLatestEntity.toData(), tsKvLatestEntity.getVersion());
process(entityIdInfo.tenantId(), LATEST_TS_KV, latestTsKv);
}
} catch (Exception e) {
log.error("Failed to sync latest timeseries: {}", tsKvLatestEntity, e);
}
}
}
private String getStrKeyOrFetchFromDb(int key) {
String strKey = keys.get(key);
if (strKey != null) {
return strKey;
} else {
strKey = keyDictionaryDao.getKey(key);
if (strKey != null) {
keys.put(key, strKey);
}
}
return strKey;
}
public record EntityIdInfo(EntityType entityType, TenantId tenantId) {
}
}

42
application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java

@ -0,0 +1,42 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.edqs;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Service;
import org.thingsboard.server.queue.edqs.EdqsQueue;
import org.thingsboard.server.queue.kafka.TbKafkaAdmin;
import org.thingsboard.server.queue.kafka.TbKafkaSettings;
import java.util.Collections;
@Service
@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'kafka'")
public class KafkaEdqsSyncService extends EdqsSyncService {
private final boolean syncNeeded;
public KafkaEdqsSyncService(TbKafkaSettings kafkaSettings) {
TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, Collections.emptyMap());
this.syncNeeded = kafkaAdmin.isTopicEmpty(EdqsQueue.EVENTS.getTopic());
}
@Override
public boolean isSyncNeeded() {
return syncNeeded;
}
}

35
application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java

@ -0,0 +1,35 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.edqs;
import lombok.RequiredArgsConstructor;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Service;
import org.thingsboard.server.edqs.util.EdqsRocksDb;
@Service
@RequiredArgsConstructor
@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'in-memory'")
public class LocalEdqsSyncService extends EdqsSyncService {
private final EdqsRocksDb db;
@Override
public boolean isSyncNeeded() {
return db.isNew();
}
}

10
application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java

@ -52,6 +52,7 @@ import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg;
import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNotificationMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNotificationMsg;
import org.thingsboard.server.dao.edge.EdgeSynchronizationManager;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
@ -67,6 +68,7 @@ public class EntityStateSourcingListener {
private final TenantService tenantService; private final TenantService tenantService;
private final TbClusterService tbClusterService; private final TbClusterService tbClusterService;
private final EdgeSynchronizationManager edgeSynchronizationManager;
@PostConstruct @PostConstruct
public void init() { public void init() {
@ -75,6 +77,11 @@ public class EntityStateSourcingListener {
@TransactionalEventListener(fallbackExecution = true) @TransactionalEventListener(fallbackExecution = true)
public void handleEvent(SaveEntityEvent<?> event) { public void handleEvent(SaveEntityEvent<?> event) {
if (Boolean.FALSE.equals(event.getBroadcastEvent())) {
log.trace("Ignoring event {}", event);
return;
}
TenantId tenantId = event.getTenantId(); TenantId tenantId = event.getTenantId();
EntityId entityId = event.getEntityId(); EntityId entityId = event.getEntityId();
if (entityId == null) { if (entityId == null) {
@ -270,6 +277,9 @@ public class EntityStateSourcingListener {
private void onEdgeEvent(TenantId tenantId, EntityId entityId, Object entity, ComponentLifecycleEvent lifecycleEvent) { private void onEdgeEvent(TenantId tenantId, EntityId entityId, Object entity, ComponentLifecycleEvent lifecycleEvent) {
if (entity instanceof Edge) { if (entity instanceof Edge) {
if (entityId.equals(edgeSynchronizationManager.getEdgeId().get())) {
return;
}
tbClusterService.onEdgeStateChangeEvent(new ComponentLifecycleMsg(tenantId, entityId, lifecycleEvent)); tbClusterService.onEdgeStateChangeEvent(new ComponentLifecycleMsg(tenantId, entityId, lifecycleEvent));
} else if (entity instanceof EdgeEvent edgeEvent) { } else if (entity instanceof EdgeEvent edgeEvent) {
tbClusterService.onEdgeEventUpdate(new EdgeEventUpdateMsg(tenantId, edgeEvent.getEdgeId())); tbClusterService.onEdgeEventUpdate(new EdgeEventUpdateMsg(tenantId, edgeEvent.getEdgeId()));

45
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@ -20,7 +20,6 @@ import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.MoreExecutors;
import jakarta.annotation.PostConstruct; import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy; import jakarta.annotation.PreDestroy;
import lombok.Data;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
@ -33,6 +32,7 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.JavaSerDesUtil; import org.thingsboard.server.common.data.JavaSerDesUtil;
import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmInfo;
import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg;
import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.ErrorEvent;
import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.Event;
import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.event.LifecycleEvent;
@ -45,6 +45,7 @@ import org.thingsboard.server.common.data.queue.QueueConfig;
import org.thingsboard.server.common.data.rpc.RpcError; import org.thingsboard.server.common.data.rpc.RpcError;
import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.MsgType;
import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.edqs.EdqsService;
import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbCallback;
@ -76,6 +77,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceM
import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager;
import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; import org.thingsboard.server.queue.common.consumer.QueueConsumerManager;
import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.QueueKey;
@ -88,7 +90,6 @@ import org.thingsboard.server.service.notification.NotificationSchedulerService;
import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.ota.OtaPackageStateService;
import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbAssetProfileCache;
import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager;
import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.queue.processing.AbstractConsumerService;
import org.thingsboard.server.service.queue.processing.IdMsgPair; import org.thingsboard.server.service.queue.processing.IdMsgPair;
import org.thingsboard.server.service.resource.TbImageService; import org.thingsboard.server.service.resource.TbImageService;
@ -146,9 +147,10 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
private final TbCoreQueueFactory queueFactory; private final TbCoreQueueFactory queueFactory;
private final TbImageService imageService; private final TbImageService imageService;
private final RuleEngineCallService ruleEngineCallService; private final RuleEngineCallService ruleEngineCallService;
private final EdqsService edqsService;
private final TbCoreConsumerStats stats; private final TbCoreConsumerStats stats;
private MainQueueConsumerManager<TbProtoQueueMsg<ToCoreMsg>, CoreQueueConfig> mainConsumer; private MainQueueConsumerManager<TbProtoQueueMsg<ToCoreMsg>, QueueConfig> mainConsumer;
private QueueConsumerManager<TbProtoQueueMsg<ToUsageStatsServiceMsg>> usageStatsConsumer; private QueueConsumerManager<TbProtoQueueMsg<ToUsageStatsServiceMsg>> usageStatsConsumer;
private QueueConsumerManager<TbProtoQueueMsg<ToOtaPackageStateServiceMsg>> firmwareStatesConsumer; private QueueConsumerManager<TbProtoQueueMsg<ToOtaPackageStateServiceMsg>> firmwareStatesConsumer;
@ -175,7 +177,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
NotificationRuleProcessor notificationRuleProcessor, NotificationRuleProcessor notificationRuleProcessor,
TbImageService imageService, TbImageService imageService,
RuleEngineCallService ruleEngineCallService, RuleEngineCallService ruleEngineCallService,
CalculatedFieldCache calculatedFieldCache) { CalculatedFieldCache calculatedFieldCache,
EdqsService edqsService) {
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService,
eventPublisher, jwtSettingsService); eventPublisher, jwtSettingsService);
this.stateService = stateService; this.stateService = stateService;
@ -191,6 +194,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
this.imageService = imageService; this.imageService = imageService;
this.ruleEngineCallService = ruleEngineCallService; this.ruleEngineCallService = ruleEngineCallService;
this.queueFactory = tbCoreQueueFactory; this.queueFactory = tbCoreQueueFactory;
this.edqsService = edqsService;
} }
@PostConstruct @PostConstruct
@ -198,9 +202,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
super.init("tb-core"); super.init("tb-core");
this.deviceActivityEventsExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-device-activity-events-executor"))); this.deviceActivityEventsExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-device-activity-events-executor")));
this.mainConsumer = MainQueueConsumerManager.<TbProtoQueueMsg<ToCoreMsg>, CoreQueueConfig>builder() this.mainConsumer = MainQueueConsumerManager.<TbProtoQueueMsg<ToCoreMsg>, QueueConfig>builder()
.queueKey(new QueueKey(ServiceType.TB_CORE)) .queueKey(new QueueKey(ServiceType.TB_CORE))
.config(CoreQueueConfig.of(consumerPerPartition, (int) pollInterval)) .config(QueueConfig.of(consumerPerPartition, pollInterval))
.msgPackProcessor(this::processMsgs) .msgPackProcessor(this::processMsgs)
.consumerCreator((config, partitionId) -> queueFactory.createToCoreMsgConsumer()) .consumerCreator((config, partitionId) -> queueFactory.createToCoreMsgConsumer())
.consumerExecutor(consumersExecutor) .consumerExecutor(consumersExecutor)
@ -251,7 +255,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
.collect(Collectors.toSet())); .collect(Collectors.toSet()));
} }
private void processMsgs(List<TbProtoQueueMsg<ToCoreMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> consumer, CoreQueueConfig config) throws Exception { private void processMsgs(List<TbProtoQueueMsg<ToCoreMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> consumer, QueueConfig config) throws Exception {
List<IdMsgPair<ToCoreMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); List<IdMsgPair<ToCoreMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList();
ConcurrentMap<UUID, TbProtoQueueMsg<ToCoreMsg>> pendingMap = orderedMsgList.stream().collect( ConcurrentMap<UUID, TbProtoQueueMsg<ToCoreMsg>> pendingMap = orderedMsgList.stream().collect(
Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg));
@ -289,6 +293,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
} else if (toCoreMsg.hasDeviceInactivityMsg()) { } else if (toCoreMsg.hasDeviceInactivityMsg()) {
log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceInactivityMsg()); log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceInactivityMsg());
forwardToStateService(toCoreMsg.getDeviceInactivityMsg(), callback); forwardToStateService(toCoreMsg.getDeviceInactivityMsg(), callback);
} else if (toCoreMsg.hasDeviceInactivityTimeoutUpdateMsg()) {
log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceInactivityTimeoutUpdateMsg());
forwardToStateService(toCoreMsg.getDeviceInactivityTimeoutUpdateMsg(), callback);
} else if (toCoreMsg.hasToDeviceActorNotification()) { } else if (toCoreMsg.hasToDeviceActorNotification()) {
TbActorMsg actorMsg = ProtoUtils.fromProto(toCoreMsg.getToDeviceActorNotification()); TbActorMsg actorMsg = ProtoUtils.fromProto(toCoreMsg.getToDeviceActorNotification());
if (actorMsg != null) { if (actorMsg != null) {
@ -389,6 +396,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
callback.onSuccess(); callback.onSuccess();
} else if (toCoreNotification.hasResourceCacheInvalidateMsg()) { } else if (toCoreNotification.hasResourceCacheInvalidateMsg()) {
forwardToResourceService(toCoreNotification.getResourceCacheInvalidateMsg(), callback); forwardToResourceService(toCoreNotification.getResourceCacheInvalidateMsg(), callback);
} else if (toCoreNotification.hasToEdqsCoreServiceMsg()) {
edqsService.processSystemMsg(JacksonUtil.fromBytes(toCoreNotification.getToEdqsCoreServiceMsg().getValue().toByteArray(), ToCoreEdqsMsg.class));
callback.onSuccess();
} }
if (statsEnabled) { if (statsEnabled) {
stats.log(toCoreNotification); stats.log(toCoreNotification);
@ -660,6 +670,21 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
}); });
} }
void forwardToStateService(TransportProtos.DeviceInactivityTimeoutUpdateProto deviceInactivityTimeoutUpdateMsg, TbCallback callback) {
if (statsEnabled) {
stats.log(deviceInactivityTimeoutUpdateMsg);
}
var tenantId = toTenantId(deviceInactivityTimeoutUpdateMsg.getTenantIdMSB(), deviceInactivityTimeoutUpdateMsg.getTenantIdLSB());
var deviceId = new DeviceId(new UUID(deviceInactivityTimeoutUpdateMsg.getDeviceIdMSB(), deviceInactivityTimeoutUpdateMsg.getDeviceIdLSB()));
ListenableFuture<?> future = deviceActivityEventsExecutor.submit(() -> stateService.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, deviceInactivityTimeoutUpdateMsg.getInactivityTimeout()));
DonAsynchron.withCallback(future,
__ -> callback.onSuccess(),
t -> {
log.warn("[{}] Failed to process device inactivity timeout update message for device [{}]", tenantId.getId(), deviceId.getId(), t);
callback.onFailure(t);
});
}
private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) {
TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB());
NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB()));
@ -732,10 +757,4 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
firmwareStatesConsumer.stop(); firmwareStatesConsumer.stop();
} }
@Data(staticConstructor = "of")
public static class CoreQueueConfig implements QueueConfig {
private final boolean consumerPerPartition;
private final int pollInterval;
}
} }

14
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java

@ -85,7 +85,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService<ToEdge
private final EdgeContextComponent edgeCtx; private final EdgeContextComponent edgeCtx;
private final EdgeConsumerStats stats; private final EdgeConsumerStats stats;
private MainQueueConsumerManager<TbProtoQueueMsg<ToEdgeMsg>, EdgeQueueConfig> mainConsumer; private MainQueueConsumerManager<TbProtoQueueMsg<ToEdgeMsg>, QueueConfig> mainConsumer;
public DefaultTbEdgeConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, public DefaultTbEdgeConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext,
StatsFactory statsFactory, EdgeContextComponent edgeCtx) { StatsFactory statsFactory, EdgeContextComponent edgeCtx) {
@ -100,9 +100,9 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService<ToEdge
public void init() { public void init() {
super.init("tb-edge"); super.init("tb-edge");
this.mainConsumer = MainQueueConsumerManager.<TbProtoQueueMsg<ToEdgeMsg>, EdgeQueueConfig>builder() this.mainConsumer = MainQueueConsumerManager.<TbProtoQueueMsg<ToEdgeMsg>, QueueConfig>builder()
.queueKey(new QueueKey(ServiceType.TB_CORE).withQueueName(DataConstants.EDGE_QUEUE_NAME)) .queueKey(new QueueKey(ServiceType.TB_CORE).withQueueName(DataConstants.EDGE_QUEUE_NAME))
.config(EdgeQueueConfig.of(consumerPerPartition, pollInterval)) .config(QueueConfig.of(consumerPerPartition, pollInterval))
.msgPackProcessor(this::processMsgs) .msgPackProcessor(this::processMsgs)
.consumerCreator((config, partitionId) -> queueFactory.createEdgeMsgConsumer()) .consumerCreator((config, partitionId) -> queueFactory.createEdgeMsgConsumer())
.consumerExecutor(consumersExecutor) .consumerExecutor(consumersExecutor)
@ -128,7 +128,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService<ToEdge
mainConsumer.update(partitions); mainConsumer.update(partitions);
} }
private void processMsgs(List<TbProtoQueueMsg<ToEdgeMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToEdgeMsg>> consumer, EdgeQueueConfig edgeQueueConfig) throws InterruptedException { private void processMsgs(List<TbProtoQueueMsg<ToEdgeMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToEdgeMsg>> consumer, QueueConfig edgeQueueConfig) throws InterruptedException {
List<IdMsgPair<ToEdgeMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); List<IdMsgPair<ToEdgeMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList();
ConcurrentMap<UUID, TbProtoQueueMsg<ToEdgeMsg>> pendingMap = orderedMsgList.stream().collect( ConcurrentMap<UUID, TbProtoQueueMsg<ToEdgeMsg>> pendingMap = orderedMsgList.stream().collect(
Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg));
@ -285,10 +285,4 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService<ToEdge
mainConsumer.awaitStop(); mainConsumer.awaitStop();
} }
@Data(staticConstructor = "of")
public static class EdgeQueueConfig implements QueueConfig {
private final boolean consumerPerPartition;
private final int pollInterval;
}
} }

2
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java

@ -107,7 +107,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
@Override @Override
protected void onTbApplicationEvent(PartitionChangeEvent event) { protected void onTbApplicationEvent(PartitionChangeEvent event) {
event.getPartitionsMap().forEach((queueKey, partitions) -> { event.getNewPartitions().forEach((queueKey, partitions) -> {
if (CollectionsUtil.isOneOf(queueKey, QueueKey.CF, QueueKey.CF_STATES)) { if (CollectionsUtil.isOneOf(queueKey, QueueKey.CF, QueueKey.CF_STATES)) {
return; return;
} }

8
application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java

@ -40,6 +40,7 @@ public class TbCoreConsumerStats {
public static final String DEVICE_ACTIVITIES = "deviceActivity"; public static final String DEVICE_ACTIVITIES = "deviceActivity";
public static final String DEVICE_DISCONNECTS = "deviceDisconnect"; public static final String DEVICE_DISCONNECTS = "deviceDisconnect";
public static final String DEVICE_INACTIVITIES = "deviceInactivity"; public static final String DEVICE_INACTIVITIES = "deviceInactivity";
public static final String DEVICE_INACTIVITY_TIMEOUT_UPDATES = "deviceInactivityTimeoutUpdate";
public static final String TO_CORE_NF_OTHER = "coreNfOther"; // normally, there is no messages when codebase is fine public static final String TO_CORE_NF_OTHER = "coreNfOther"; // normally, there is no messages when codebase is fine
public static final String TO_CORE_NF_COMPONENT_LIFECYCLE = "coreNfCompLfcl"; public static final String TO_CORE_NF_COMPONENT_LIFECYCLE = "coreNfCompLfcl";
@ -65,6 +66,7 @@ public class TbCoreConsumerStats {
private final StatsCounter deviceActivitiesCounter; private final StatsCounter deviceActivitiesCounter;
private final StatsCounter deviceDisconnectsCounter; private final StatsCounter deviceDisconnectsCounter;
private final StatsCounter deviceInactivitiesCounter; private final StatsCounter deviceInactivitiesCounter;
private final StatsCounter deviceInactivityTimeoutUpdatesCounter;
private final StatsCounter toCoreNfOtherCounter; private final StatsCounter toCoreNfOtherCounter;
private final StatsCounter toCoreNfComponentLifecycleCounter; private final StatsCounter toCoreNfComponentLifecycleCounter;
@ -95,6 +97,7 @@ public class TbCoreConsumerStats {
this.deviceActivitiesCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_ACTIVITIES)); this.deviceActivitiesCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_ACTIVITIES));
this.deviceDisconnectsCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_DISCONNECTS)); this.deviceDisconnectsCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_DISCONNECTS));
this.deviceInactivitiesCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_INACTIVITIES)); this.deviceInactivitiesCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_INACTIVITIES));
this.deviceInactivityTimeoutUpdatesCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_INACTIVITY_TIMEOUT_UPDATES));
// Core notification counters // Core notification counters
this.toCoreNfOtherCounter = register(statsFactory.createStatsCounter(statsKey, TO_CORE_NF_OTHER)); this.toCoreNfOtherCounter = register(statsFactory.createStatsCounter(statsKey, TO_CORE_NF_OTHER));
@ -163,6 +166,11 @@ public class TbCoreConsumerStats {
deviceInactivitiesCounter.increment(); deviceInactivitiesCounter.increment();
} }
public void log(TransportProtos.DeviceInactivityTimeoutUpdateProto msg) {
totalCounter.increment();
deviceInactivityTimeoutUpdatesCounter.increment();
}
public void log(TransportProtos.SubscriptionMgrMsgProto msg) { public void log(TransportProtos.SubscriptionMgrMsgProto msg) {
totalCounter.increment(); totalCounter.increment();
subscriptionMsgCounter.increment(); subscriptionMsgCounter.increment();

2
application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java

@ -72,7 +72,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager<T
ExecutorService consumerExecutor, ExecutorService consumerExecutor,
ScheduledExecutorService scheduler, ScheduledExecutorService scheduler,
ExecutorService taskExecutor) { ExecutorService taskExecutor) {
super(queueKey, null, null, ctx.getQueueFactory()::createToRuleEngineMsgConsumer, consumerExecutor, scheduler, taskExecutor); super(queueKey, null, null, ctx.getQueueFactory()::createToRuleEngineMsgConsumer, consumerExecutor, scheduler, taskExecutor, null);
this.ctx = ctx; this.ctx = ctx;
this.stats = new TbRuleEngineConsumerStats(queueKey, ctx.getStatsFactory()); this.stats = new TbRuleEngineConsumerStats(queueKey, ctx.getStatsFactory());
} }

180
application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateManager.java

@ -0,0 +1,180 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.state;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.thingsboard.rule.engine.api.DeviceStateManager;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.common.SimpleTbQueueCallback;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import java.util.Optional;
import java.util.function.Consumer;
import java.util.function.Supplier;
@Slf4j
@Service
@RequiredArgsConstructor
public class DefaultDeviceStateManager implements DeviceStateManager {
private final TbServiceInfoProvider serviceInfoProvider;
private final PartitionService partitionService;
private final Optional<DeviceStateService> deviceStateService;
private final TbClusterService clusterService;
@Override
public void onDeviceConnect(TenantId tenantId, DeviceId deviceId, long connectTime, TbCallback callback) {
forwardToDeviceStateService(tenantId, deviceId,
deviceStateService -> {
log.debug("[{}][{}] Forwarding device connect event to local service. Connect time: [{}].", tenantId.getId(), deviceId.getId(), connectTime);
deviceStateService.onDeviceConnect(tenantId, deviceId, connectTime);
},
() -> {
log.debug("[{}][{}] Sending device connect message to core. Connect time: [{}].", tenantId.getId(), deviceId.getId(), connectTime);
var deviceConnectMsg = TransportProtos.DeviceConnectProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setLastConnectTime(connectTime)
.build();
return TransportProtos.ToCoreMsg.newBuilder()
.setDeviceConnectMsg(deviceConnectMsg)
.build();
}, callback);
}
@Override
public void onDeviceActivity(TenantId tenantId, DeviceId deviceId, long activityTime, TbCallback callback) {
forwardToDeviceStateService(tenantId, deviceId,
deviceStateService -> {
log.debug("[{}][{}] Forwarding device activity event to local service. Activity time: [{}].", tenantId.getId(), deviceId.getId(), activityTime);
deviceStateService.onDeviceActivity(tenantId, deviceId, activityTime);
},
() -> {
log.debug("[{}][{}] Sending device activity message to core. Activity time: [{}].", tenantId.getId(), deviceId.getId(), activityTime);
var deviceActivityMsg = TransportProtos.DeviceActivityProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setLastActivityTime(activityTime)
.build();
return TransportProtos.ToCoreMsg.newBuilder()
.setDeviceActivityMsg(deviceActivityMsg)
.build();
}, callback);
}
@Override
public void onDeviceDisconnect(TenantId tenantId, DeviceId deviceId, long disconnectTime, TbCallback callback) {
forwardToDeviceStateService(tenantId, deviceId,
deviceStateService -> {
log.debug("[{}][{}] Forwarding device disconnect event to local service. Disconnect time: [{}].", tenantId.getId(), deviceId.getId(), disconnectTime);
deviceStateService.onDeviceDisconnect(tenantId, deviceId, disconnectTime);
},
() -> {
log.debug("[{}][{}] Sending device disconnect message to core. Disconnect time: [{}].", tenantId.getId(), deviceId.getId(), disconnectTime);
var deviceDisconnectMsg = TransportProtos.DeviceDisconnectProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setLastDisconnectTime(disconnectTime)
.build();
return TransportProtos.ToCoreMsg.newBuilder()
.setDeviceDisconnectMsg(deviceDisconnectMsg)
.build();
}, callback);
}
@Override
public void onDeviceInactivity(TenantId tenantId, DeviceId deviceId, long inactivityTime, TbCallback callback) {
forwardToDeviceStateService(tenantId, deviceId,
deviceStateService -> {
log.debug("[{}][{}] Forwarding device inactivity event to local service. Inactivity time: [{}].", tenantId.getId(), deviceId.getId(), inactivityTime);
deviceStateService.onDeviceInactivity(tenantId, deviceId, inactivityTime);
},
() -> {
log.debug("[{}][{}] Sending device inactivity message to core. Inactivity time: [{}].", tenantId.getId(), deviceId.getId(), inactivityTime);
var deviceInactivityMsg = TransportProtos.DeviceInactivityProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setLastInactivityTime(inactivityTime)
.build();
return TransportProtos.ToCoreMsg.newBuilder()
.setDeviceInactivityMsg(deviceInactivityMsg)
.build();
}, callback);
}
@Override
public void onDeviceInactivityTimeoutUpdate(TenantId tenantId, DeviceId deviceId, long inactivityTimeout, TbCallback callback) {
forwardToDeviceStateService(tenantId, deviceId,
deviceStateService -> {
log.debug("[{}][{}] Forwarding device inactivity timeout update to local service. Updated inactivity timeout: [{}].", tenantId.getId(), deviceId.getId(), inactivityTimeout);
deviceStateService.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, inactivityTimeout);
},
() -> {
log.debug("[{}][{}] Sending device inactivity timeout update message to core. Updated inactivity timeout: [{}].", tenantId.getId(), deviceId.getId(), inactivityTimeout);
var deviceInactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setInactivityTimeout(inactivityTimeout)
.build();
return TransportProtos.ToCoreMsg.newBuilder()
.setDeviceInactivityTimeoutUpdateMsg(deviceInactivityTimeoutUpdateMsg)
.build();
}, callback);
}
private void forwardToDeviceStateService(
TenantId tenantId, DeviceId deviceId,
Consumer<DeviceStateService> toDeviceStateService,
Supplier<TransportProtos.ToCoreMsg> toCore,
TbCallback callback
) {
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId);
if (serviceInfoProvider.isService(ServiceType.TB_CORE) && tpi.isMyPartition() && deviceStateService.isPresent()) {
try {
toDeviceStateService.accept(deviceStateService.get());
} catch (Exception e) {
log.error("[{}][{}] Failed to process device connectivity event.", tenantId.getId(), deviceId.getId(), e);
callback.onFailure(e);
return;
}
callback.onSuccess();
} else {
TransportProtos.ToCoreMsg toCoreMsg = toCore.get();
clusterService.pushMsgToCore(tpi, deviceId.getId(), toCoreMsg, new SimpleTbQueueCallback(__ -> callback.onSuccess(), callback::onFailure));
}
}
}

108
application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java

@ -96,6 +96,7 @@ import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Optional;
import java.util.Random; import java.util.Random;
import java.util.Set; import java.util.Set;
import java.util.UUID; import java.util.UUID;
@ -129,11 +130,10 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
private static final List<EntityKey> PERSISTENT_TELEMETRY_KEYS = Arrays.asList( private static final List<EntityKey> PERSISTENT_TELEMETRY_KEYS = Arrays.asList(
new EntityKey(EntityKeyType.TIME_SERIES, LAST_ACTIVITY_TIME), new EntityKey(EntityKeyType.TIME_SERIES, LAST_ACTIVITY_TIME),
new EntityKey(EntityKeyType.TIME_SERIES, INACTIVITY_ALARM_TIME), new EntityKey(EntityKeyType.TIME_SERIES, INACTIVITY_ALARM_TIME),
new EntityKey(EntityKeyType.TIME_SERIES, INACTIVITY_TIMEOUT),
new EntityKey(EntityKeyType.TIME_SERIES, ACTIVITY_STATE), new EntityKey(EntityKeyType.TIME_SERIES, ACTIVITY_STATE),
new EntityKey(EntityKeyType.TIME_SERIES, LAST_CONNECT_TIME), new EntityKey(EntityKeyType.TIME_SERIES, LAST_CONNECT_TIME),
new EntityKey(EntityKeyType.TIME_SERIES, LAST_DISCONNECT_TIME), new EntityKey(EntityKeyType.TIME_SERIES, LAST_DISCONNECT_TIME),
new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, INACTIVITY_TIMEOUT)); new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, INACTIVITY_TIMEOUT)); // inactivity timeout is always a server attribute, even when activity data is stored as time series
private static final List<EntityKey> PERSISTENT_ATTRIBUTE_KEYS = Arrays.asList( private static final List<EntityKey> PERSISTENT_ATTRIBUTE_KEYS = Arrays.asList(
new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, LAST_ACTIVITY_TIME), new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, LAST_ACTIVITY_TIME),
@ -143,8 +143,14 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, LAST_CONNECT_TIME), new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, LAST_CONNECT_TIME),
new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, LAST_DISCONNECT_TIME)); new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, LAST_DISCONNECT_TIME));
public static final List<String> PERSISTENT_ATTRIBUTES = Arrays.asList(ACTIVITY_STATE, LAST_CONNECT_TIME, public static final Set<String> ACTIVITY_KEYS_WITHOUT_INACTIVITY_TIMEOUT = Set.of(
LAST_DISCONNECT_TIME, LAST_ACTIVITY_TIME, INACTIVITY_ALARM_TIME, INACTIVITY_TIMEOUT); ACTIVITY_STATE, LAST_CONNECT_TIME, LAST_DISCONNECT_TIME, LAST_ACTIVITY_TIME, INACTIVITY_ALARM_TIME
);
public static final Set<String> ACTIVITY_KEYS_WITH_INACTIVITY_TIMEOUT = Set.of(
ACTIVITY_STATE, LAST_CONNECT_TIME, LAST_DISCONNECT_TIME, LAST_ACTIVITY_TIME, INACTIVITY_ALARM_TIME, INACTIVITY_TIMEOUT
);
private static final List<EntityKey> PERSISTENT_ENTITY_FIELDS = Arrays.asList( private static final List<EntityKey> PERSISTENT_ENTITY_FIELDS = Arrays.asList(
new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "name"),
new EntityKey(EntityKeyType.ENTITY_FIELD, "type"), new EntityKey(EntityKeyType.ENTITY_FIELD, "type"),
@ -251,7 +257,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
} }
log.trace("[{}][{}] On device connect: processing connect event with ts [{}].", tenantId.getId(), deviceId.getId(), lastConnectTime); log.trace("[{}][{}] On device connect: processing connect event with ts [{}].", tenantId.getId(), deviceId.getId(), lastConnectTime);
stateData.getState().setLastConnectTime(lastConnectTime); stateData.getState().setLastConnectTime(lastConnectTime);
save(deviceId, LAST_CONNECT_TIME, lastConnectTime); save(tenantId, deviceId, LAST_CONNECT_TIME, lastConnectTime);
pushRuleEngineMessage(stateData, TbMsgType.CONNECT_EVENT); pushRuleEngineMessage(stateData, TbMsgType.CONNECT_EVENT);
checkAndUpdateState(deviceId, stateData); checkAndUpdateState(deviceId, stateData);
} }
@ -271,14 +277,14 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
void updateActivityState(DeviceId deviceId, DeviceStateData stateData, long lastReportedActivity) { void updateActivityState(DeviceId deviceId, DeviceStateData stateData, long lastReportedActivity) {
log.trace("updateActivityState - fetched state {} for device {}, lastReportedActivity {}", stateData, deviceId, lastReportedActivity); log.trace("updateActivityState - fetched state {} for device {}, lastReportedActivity {}", stateData, deviceId, lastReportedActivity);
if (stateData != null) { if (stateData != null) {
save(deviceId, LAST_ACTIVITY_TIME, lastReportedActivity); save(stateData.getTenantId(), deviceId, LAST_ACTIVITY_TIME, lastReportedActivity);
DeviceState state = stateData.getState(); DeviceState state = stateData.getState();
state.setLastActivityTime(lastReportedActivity); state.setLastActivityTime(lastReportedActivity);
if (!state.isActive()) { if (!state.isActive()) {
state.setActive(true); state.setActive(true);
if (lastReportedActivity <= state.getLastInactivityAlarmTime()) { if (lastReportedActivity <= state.getLastInactivityAlarmTime()) {
state.setLastInactivityAlarmTime(0); state.setLastInactivityAlarmTime(0);
save(deviceId, INACTIVITY_ALARM_TIME, 0); save(stateData.getTenantId(), deviceId, INACTIVITY_ALARM_TIME, 0);
} }
onDeviceActivityStatusChange(deviceId, true, stateData); onDeviceActivityStatusChange(deviceId, true, stateData);
} }
@ -307,7 +313,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
} }
log.trace("[{}][{}] On device disconnect: processing disconnect event with ts [{}].", tenantId.getId(), deviceId.getId(), lastDisconnectTime); log.trace("[{}][{}] On device disconnect: processing disconnect event with ts [{}].", tenantId.getId(), deviceId.getId(), lastDisconnectTime);
stateData.getState().setLastDisconnectTime(lastDisconnectTime); stateData.getState().setLastDisconnectTime(lastDisconnectTime);
save(deviceId, LAST_DISCONNECT_TIME, lastDisconnectTime); save(tenantId, deviceId, LAST_DISCONNECT_TIME, lastDisconnectTime);
pushRuleEngineMessage(stateData, TbMsgType.DISCONNECT_EVENT); pushRuleEngineMessage(stateData, TbMsgType.DISCONNECT_EVENT);
} }
@ -418,7 +424,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
private void initializeActivityState(DeviceId deviceId, DeviceStateData fetchedState) { private void initializeActivityState(DeviceId deviceId, DeviceStateData fetchedState) {
DeviceStateData cachedState = deviceStates.putIfAbsent(fetchedState.getDeviceId(), fetchedState); DeviceStateData cachedState = deviceStates.putIfAbsent(fetchedState.getDeviceId(), fetchedState);
boolean activityState = Objects.requireNonNullElse(cachedState, fetchedState).getState().isActive(); boolean activityState = Objects.requireNonNullElse(cachedState, fetchedState).getState().isActive();
save(deviceId, ACTIVITY_STATE, activityState); save(fetchedState.getTenantId(), deviceId, ACTIVITY_STATE, activityState);
} }
@Override @Override
@ -496,7 +502,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
updateActivityState(deviceId, state, deviceState.getLastActivityTime()); updateActivityState(deviceId, state, deviceState.getLastActivityTime());
if (deviceState.getLastInactivityAlarmTime() != 0L && deviceState.getLastInactivityAlarmTime() >= deviceState.getLastActivityTime()) { if (deviceState.getLastInactivityAlarmTime() != 0L && deviceState.getLastInactivityAlarmTime() >= deviceState.getLastActivityTime()) {
deviceState.setLastInactivityAlarmTime(0L); deviceState.setLastInactivityAlarmTime(0L);
save(deviceId, INACTIVITY_ALARM_TIME, 0L); save(state.getTenantId(), deviceId, INACTIVITY_ALARM_TIME, 0L);
} }
} }
} }
@ -583,7 +589,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
DeviceState state = stateData.getState(); DeviceState state = stateData.getState();
state.setActive(false); state.setActive(false);
state.setLastInactivityAlarmTime(ts); state.setLastInactivityAlarmTime(ts);
save(deviceId, INACTIVITY_ALARM_TIME, ts); save(stateData.getTenantId(), deviceId, INACTIVITY_ALARM_TIME, ts);
onDeviceActivityStatusChange(deviceId, false, stateData); onDeviceActivityStatusChange(deviceId, false, stateData);
} }
@ -611,7 +617,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
} }
private void onDeviceActivityStatusChange(DeviceId deviceId, boolean active, DeviceStateData stateData) { private void onDeviceActivityStatusChange(DeviceId deviceId, boolean active, DeviceStateData stateData) {
save(deviceId, ACTIVITY_STATE, active); save(stateData.getTenantId(), deviceId, ACTIVITY_STATE, active);
pushRuleEngineMessage(stateData, active ? TbMsgType.ACTIVITY_EVENT : TbMsgType.INACTIVITY_EVENT); pushRuleEngineMessage(stateData, active ? TbMsgType.ACTIVITY_EVENT : TbMsgType.INACTIVITY_EVENT);
TbMsgMetaData metaData = stateData.getMetaData(); TbMsgMetaData metaData = stateData.getMetaData();
notificationRuleProcessor.process(DeviceActivityTrigger.builder() notificationRuleProcessor.process(DeviceActivityTrigger.builder()
@ -643,41 +649,45 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
deviceStates.remove(deviceId); deviceStates.remove(deviceId);
} }
private ListenableFuture<DeviceStateData> fetchDeviceState(Device device) { private ListenableFuture<DeviceStateData> fetchDeviceState(Device device) {
ListenableFuture<DeviceStateData> future; ListenableFuture<DeviceStateData> future;
if (persistToTelemetry) { if (persistToTelemetry) {
ListenableFuture<List<TsKvEntry>> tsData = tsService.findLatest(TenantId.SYS_TENANT_ID, device.getId(), PERSISTENT_ATTRIBUTES); ListenableFuture<List<TsKvEntry>> timeseriesActivityDataFuture = tsService.findLatest(TenantId.SYS_TENANT_ID, device.getId(), ACTIVITY_KEYS_WITHOUT_INACTIVITY_TIMEOUT);
future = Futures.transform(tsData, extractDeviceStateData(device), MoreExecutors.directExecutor()); ListenableFuture<Optional<AttributeKvEntry>> inactivityTimeoutAttributeFuture = attributesService.find(
TenantId.SYS_TENANT_ID, device.getId(), AttributeScope.SERVER_SCOPE, INACTIVITY_TIMEOUT
);
ListenableFuture<List<? extends KvEntry>> fullActivityDataFuture = Futures.whenAllSucceed(timeseriesActivityDataFuture, inactivityTimeoutAttributeFuture).call(() -> {
List<TsKvEntry> activityTimeseries = Futures.getDone(timeseriesActivityDataFuture);
Optional<AttributeKvEntry> inactivityTimeoutAttribute = Futures.getDone(inactivityTimeoutAttributeFuture);
List<KvEntry> result;
if (inactivityTimeoutAttribute.isPresent()) {
result = new ArrayList<>(activityTimeseries.size() + 1);
result.addAll(activityTimeseries);
inactivityTimeoutAttribute.ifPresent(result::add);
} else {
return activityTimeseries;
}
return result;
}, deviceStateCallbackExecutor);
future = Futures.transform(fullActivityDataFuture, extractDeviceStateData(device), MoreExecutors.directExecutor());
} else { } else {
ListenableFuture<List<AttributeKvEntry>> attrData = attributesService.find(TenantId.SYS_TENANT_ID, device.getId(), AttributeScope.SERVER_SCOPE, PERSISTENT_ATTRIBUTES); ListenableFuture<List<AttributeKvEntry>> attributesActivityDataFuture = attributesService.find(
future = Futures.transform(attrData, extractDeviceStateData(device), MoreExecutors.directExecutor()); TenantId.SYS_TENANT_ID, device.getId(), AttributeScope.SERVER_SCOPE, ACTIVITY_KEYS_WITH_INACTIVITY_TIMEOUT
);
future = Futures.transform(attributesActivityDataFuture, extractDeviceStateData(device), MoreExecutors.directExecutor());
} }
return transformInactivityTimeout(future); return future;
} }
private ListenableFuture<DeviceStateData> transformInactivityTimeout(ListenableFuture<DeviceStateData> future) { private Function<List<? extends KvEntry>, DeviceStateData> extractDeviceStateData(Device device) {
return Futures.transformAsync(future, deviceStateData -> {
if (!persistToTelemetry || deviceStateData.getState().getInactivityTimeout() != defaultInactivityTimeoutMs) {
return future; //fail fast
}
var attributesFuture = attributesService.find(TenantId.SYS_TENANT_ID, deviceStateData.getDeviceId(), AttributeScope.SERVER_SCOPE, INACTIVITY_TIMEOUT);
return Futures.transform(attributesFuture, attributes -> {
attributes.flatMap(KvEntry::getLongValue).ifPresent((inactivityTimeout) -> {
if (inactivityTimeout > 0) {
deviceStateData.getState().setInactivityTimeout(inactivityTimeout);
}
});
return deviceStateData;
}, MoreExecutors.directExecutor());
}, deviceStateCallbackExecutor);
}
private <T extends KvEntry> Function<List<T>, DeviceStateData> extractDeviceStateData(Device device) {
return new Function<>() { return new Function<>() {
@Nonnull @Nonnull
@Override @Override
public DeviceStateData apply(@Nullable List<T> data) { public DeviceStateData apply(@Nullable List<? extends KvEntry> data) {
try { try {
long lastActivityTime = getEntryValue(data, LAST_ACTIVITY_TIME, 0L); long lastActivityTime = getEntryValue(data, LAST_ACTIVITY_TIME, 0L);
long inactivityAlarmTime = getEntryValue(data, INACTIVITY_ALARM_TIME, 0L); long inactivityAlarmTime = getEntryValue(data, INACTIVITY_ALARM_TIME, 0L);
@ -690,7 +700,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
.lastDisconnectTime(getEntryValue(data, LAST_DISCONNECT_TIME, 0L)) .lastDisconnectTime(getEntryValue(data, LAST_DISCONNECT_TIME, 0L))
.lastActivityTime(lastActivityTime) .lastActivityTime(lastActivityTime)
.lastInactivityAlarmTime(inactivityAlarmTime) .lastInactivityAlarmTime(inactivityAlarmTime)
.inactivityTimeout(inactivityTimeout) .inactivityTimeout(inactivityTimeout > 0 ? inactivityTimeout : defaultInactivityTimeoutMs)
.build(); .build();
TbMsgMetaData md = new TbMsgMetaData(); TbMsgMetaData md = new TbMsgMetaData();
md.putValue("deviceName", device.getName()); md.putValue("deviceName", device.getName());
@ -761,12 +771,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
DeviceStateData toDeviceStateData(EntityData ed, DeviceIdInfo deviceIdInfo) { DeviceStateData toDeviceStateData(EntityData ed, DeviceIdInfo deviceIdInfo) {
long lastActivityTime = getEntryValue(ed, getKeyType(), LAST_ACTIVITY_TIME, 0L); long lastActivityTime = getEntryValue(ed, getKeyType(), LAST_ACTIVITY_TIME, 0L);
long inactivityAlarmTime = getEntryValue(ed, getKeyType(), INACTIVITY_ALARM_TIME, 0L); long inactivityAlarmTime = getEntryValue(ed, getKeyType(), INACTIVITY_ALARM_TIME, 0L);
long inactivityTimeout = getEntryValue(ed, getKeyType(), INACTIVITY_TIMEOUT, defaultInactivityTimeoutMs); long inactivityTimeout = getEntryValue(ed, EntityKeyType.SERVER_ATTRIBUTE, INACTIVITY_TIMEOUT, defaultInactivityTimeoutMs);
if (persistToTelemetry && inactivityTimeout == defaultInactivityTimeoutMs) {
log.trace("[{}] default value for inactivity timeout fetched {}, going to fetch inactivity timeout from attributes",
deviceIdInfo.getDeviceId(), inactivityTimeout);
inactivityTimeout = getEntryValue(ed, EntityKeyType.SERVER_ATTRIBUTE, INACTIVITY_TIMEOUT, defaultInactivityTimeoutMs);
}
// Actual active state by wall-clock will be updated outside this method. This method is only for fetching persistent state // Actual active state by wall-clock will be updated outside this method. This method is only for fetching persistent state
final boolean active = getEntryValue(ed, getKeyType(), ACTIVITY_STATE, false); final boolean active = getEntryValue(ed, getKeyType(), ACTIVITY_STATE, false);
DeviceState deviceState = DeviceState.builder() DeviceState deviceState = DeviceState.builder()
@ -874,18 +879,18 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
} }
} }
private void save(DeviceId deviceId, String key, long value) { private void save(TenantId tenantId, DeviceId deviceId, String key, long value) {
save(deviceId, new LongDataEntry(key, value), getCurrentTimeMillis()); save(tenantId, deviceId, new LongDataEntry(key, value), getCurrentTimeMillis());
} }
private void save(DeviceId deviceId, String key, boolean value) { private void save(TenantId tenantId, DeviceId deviceId, String key, boolean value) {
save(deviceId, new BooleanDataEntry(key, value), getCurrentTimeMillis()); save(tenantId, deviceId, new BooleanDataEntry(key, value), getCurrentTimeMillis());
} }
private void save(DeviceId deviceId, KvEntry kvEntry, long ts) { private void save(TenantId tenantId, DeviceId deviceId, KvEntry kvEntry, long ts) {
if (persistToTelemetry) { if (persistToTelemetry) {
tsSubService.saveTimeseriesInternal(TimeseriesSaveRequest.builder() tsSubService.saveTimeseriesInternal(TimeseriesSaveRequest.builder()
.tenantId(TenantId.SYS_TENANT_ID) .tenantId(tenantId)
.entityId(deviceId) .entityId(deviceId)
.entry(new BasicTsKvEntry(ts, kvEntry)) .entry(new BasicTsKvEntry(ts, kvEntry))
.ttl(telemetryTtl) .ttl(telemetryTtl)
@ -893,7 +898,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
.build()); .build());
} else { } else {
tsSubService.saveAttributes(AttributesSaveRequest.builder() tsSubService.saveAttributes(AttributesSaveRequest.builder()
.tenantId(TenantId.SYS_TENANT_ID) .tenantId(tenantId)
.entityId(deviceId) .entityId(deviceId)
.scope(AttributeScope.SERVER_SCOPE) .scope(AttributeScope.SERVER_SCOPE)
.entry(new BaseAttributeKvEntry(ts, kvEntry)) .entry(new BaseAttributeKvEntry(ts, kvEntry))
@ -925,4 +930,5 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
log.warn("[{}] Failed to update entry {}", deviceId, kvEntry, t); log.warn("[{}] Failed to update entry {}", deviceId, kvEntry, t);
} }
} }
} }

196
application/src/main/java/org/thingsboard/server/service/state/DefaultRuleEngineDeviceStateManager.java

@ -1,196 +0,0 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.state;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.thingsboard.rule.engine.api.RuleEngineDeviceStateManager;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.common.SimpleTbQueueCallback;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.util.TbRuleEngineComponent;
import java.util.Optional;
import java.util.UUID;
@Slf4j
@Service
@TbRuleEngineComponent
public class DefaultRuleEngineDeviceStateManager implements RuleEngineDeviceStateManager {
private final TbServiceInfoProvider serviceInfoProvider;
private final PartitionService partitionService;
private final Optional<DeviceStateService> deviceStateService;
private final TbClusterService clusterService;
public DefaultRuleEngineDeviceStateManager(
TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService,
Optional<DeviceStateService> deviceStateServiceOptional, TbClusterService clusterService
) {
this.serviceInfoProvider = serviceInfoProvider;
this.partitionService = partitionService;
this.deviceStateService = deviceStateServiceOptional;
this.clusterService = clusterService;
}
@Getter
private abstract static class ConnectivityEventInfo {
private final TenantId tenantId;
private final DeviceId deviceId;
private final long eventTime;
private ConnectivityEventInfo(TenantId tenantId, DeviceId deviceId, long eventTime) {
this.tenantId = tenantId;
this.deviceId = deviceId;
this.eventTime = eventTime;
}
abstract void forwardToLocalService();
abstract TransportProtos.ToCoreMsg toQueueMsg();
}
@Override
public void onDeviceConnect(TenantId tenantId, DeviceId deviceId, long connectTime, TbCallback callback) {
routeEvent(new ConnectivityEventInfo(tenantId, deviceId, connectTime) {
@Override
void forwardToLocalService() {
deviceStateService.ifPresent(service -> service.onDeviceConnect(tenantId, deviceId, connectTime));
}
@Override
TransportProtos.ToCoreMsg toQueueMsg() {
var deviceConnectMsg = TransportProtos.DeviceConnectProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setLastConnectTime(connectTime)
.build();
return TransportProtos.ToCoreMsg.newBuilder()
.setDeviceConnectMsg(deviceConnectMsg)
.build();
}
}, callback);
}
@Override
public void onDeviceActivity(TenantId tenantId, DeviceId deviceId, long activityTime, TbCallback callback) {
routeEvent(new ConnectivityEventInfo(tenantId, deviceId, activityTime) {
@Override
void forwardToLocalService() {
deviceStateService.ifPresent(service -> service.onDeviceActivity(tenantId, deviceId, activityTime));
}
@Override
TransportProtos.ToCoreMsg toQueueMsg() {
var deviceActivityMsg = TransportProtos.DeviceActivityProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setLastActivityTime(activityTime)
.build();
return TransportProtos.ToCoreMsg.newBuilder()
.setDeviceActivityMsg(deviceActivityMsg)
.build();
}
}, callback);
}
@Override
public void onDeviceDisconnect(TenantId tenantId, DeviceId deviceId, long disconnectTime, TbCallback callback) {
routeEvent(new ConnectivityEventInfo(tenantId, deviceId, disconnectTime) {
@Override
void forwardToLocalService() {
deviceStateService.ifPresent(service -> service.onDeviceDisconnect(tenantId, deviceId, disconnectTime));
}
@Override
TransportProtos.ToCoreMsg toQueueMsg() {
var deviceDisconnectMsg = TransportProtos.DeviceDisconnectProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setLastDisconnectTime(disconnectTime)
.build();
return TransportProtos.ToCoreMsg.newBuilder()
.setDeviceDisconnectMsg(deviceDisconnectMsg)
.build();
}
}, callback);
}
@Override
public void onDeviceInactivity(TenantId tenantId, DeviceId deviceId, long inactivityTime, TbCallback callback) {
routeEvent(new ConnectivityEventInfo(tenantId, deviceId, inactivityTime) {
@Override
void forwardToLocalService() {
deviceStateService.ifPresent(service -> service.onDeviceInactivity(tenantId, deviceId, inactivityTime));
}
@Override
TransportProtos.ToCoreMsg toQueueMsg() {
var deviceInactivityMsg = TransportProtos.DeviceInactivityProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setLastInactivityTime(inactivityTime)
.build();
return TransportProtos.ToCoreMsg.newBuilder()
.setDeviceInactivityMsg(deviceInactivityMsg)
.build();
}
}, callback);
}
private void routeEvent(ConnectivityEventInfo eventInfo, TbCallback callback) {
var tenantId = eventInfo.getTenantId();
var deviceId = eventInfo.getDeviceId();
long eventTime = eventInfo.getEventTime();
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId);
if (serviceInfoProvider.isService(ServiceType.TB_CORE) && tpi.isMyPartition() && deviceStateService.isPresent()) {
log.debug("[{}][{}] Forwarding device connectivity event to local service. Event time: [{}].", tenantId.getId(), deviceId.getId(), eventTime);
try {
eventInfo.forwardToLocalService();
} catch (Exception e) {
log.error("[{}][{}] Failed to process device connectivity event. Event time: [{}].", tenantId.getId(), deviceId.getId(), eventTime, e);
callback.onFailure(e);
return;
}
callback.onSuccess();
} else {
TransportProtos.ToCoreMsg msg = eventInfo.toQueueMsg();
log.debug("[{}][{}] Sending device connectivity message to core. Event time: [{}].", tenantId.getId(), deviceId.getId(), eventTime);
clusterService.pushMsgToCore(tpi, UUID.randomUUID(), msg, new SimpleTbQueueCallback(__ -> callback.onSuccess(), callback::onFailure));
}
}
}

6
application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java

@ -160,9 +160,6 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene
@Override @Override
public void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, TbCallback callback) { public void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, TbCallback callback) {
onTimeSeriesUpdate(entityId, ts); onTimeSeriesUpdate(entityId, ts);
if (entityId.getEntityType() == EntityType.DEVICE) {
updateDeviceInactivityTimeout(tenantId, entityId, ts);
}
callback.onSuccess(); callback.onSuccess();
} }
@ -170,9 +167,6 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene
public void onTimeSeriesDelete(TenantId tenantId, EntityId entityId, List<String> keys, TbCallback callback) { public void onTimeSeriesDelete(TenantId tenantId, EntityId entityId, List<String> keys, TbCallback callback) {
onTimeSeriesUpdate(entityId, onTimeSeriesUpdate(entityId,
keys.stream().map(key -> new BasicTsKvEntry(0, new StringDataEntry(key, ""))).collect(Collectors.toList())); keys.stream().map(key -> new BasicTsKvEntry(0, new StringDataEntry(key, ""))).collect(Collectors.toList()));
if (entityId.getEntityType() == EntityType.DEVICE) {
deleteDeviceInactivityTimeout(tenantId, entityId, keys);
}
callback.onSuccess(); callback.onSuccess();
} }

36
application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java

@ -33,7 +33,7 @@ import org.springframework.stereotype.Service;
import org.springframework.web.socket.CloseStatus; import org.springframework.web.socket.CloseStatus;
import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult;
@ -41,7 +41,6 @@ import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.query.AlarmDataQuery; import org.thingsboard.server.common.data.query.AlarmDataQuery;
import org.thingsboard.server.common.data.query.ComparisonTsValue; import org.thingsboard.server.common.data.query.ComparisonTsValue;
import org.thingsboard.server.common.data.query.OriginatorAlarmFilter;
import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKey;
@ -55,17 +54,16 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.executors.DbCallbackExecutorService;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.ws.WebSocketService; import org.thingsboard.server.service.ws.WebSocketService;
import org.thingsboard.server.service.ws.WebSocketSessionRef; import org.thingsboard.server.service.ws.WebSocketSessionRef;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.AggHistoryCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AggHistoryCmd;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.AggKey; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AggKey;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.AggTimeSeriesCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AggTimeSeriesCmd;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmCountCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmCountCmd;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmCountUpdate;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataCmd;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataUpdate;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmStatusCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmStatusCmd;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.CmdUpdate;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityCountCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityCountCmd;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataCmd;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataUpdate;
@ -74,7 +72,6 @@ import org.thingsboard.server.service.ws.telemetry.cmd.v2.GetTsCmd;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.LatestValueCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.LatestValueCmd;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.TimeSeriesCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.TimeSeriesCmd;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.UnsubscribeCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.UnsubscribeCmd;
import org.thingsboard.server.service.ws.telemetry.sub.AlarmSubscriptionUpdate;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
@ -83,7 +80,6 @@ import java.util.LinkedHashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
@ -430,13 +426,25 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
ctx.fetchData(); ctx.fetchData();
long end = System.currentTimeMillis(); long end = System.currentTimeMillis();
stats.getAlarmQueryInvocationCnt().incrementAndGet(); stats.getRegularQueryInvocationCnt().incrementAndGet();
stats.getAlarmQueryTimeSpent().addAndGet(end - start); stats.getRegularQueryTimeSpent().addAndGet(end - start);
TbAlarmCountSubCtx finalCtx = ctx; Set<EntityId> entitiesIds = ctx.getEntitiesIds();
ScheduledFuture<?> task = scheduler.scheduleWithFixedDelay( ctx.cancelTasks();
() -> refreshDynamicQuery(finalCtx), ctx.clearAlarmSubscriptions();
dynamicPageLinkRefreshInterval, dynamicPageLinkRefreshInterval, TimeUnit.SECONDS); if (entitiesIds != null && entitiesIds.isEmpty()) {
finalCtx.setRefreshTask(task); AlarmCountUpdate update = new AlarmCountUpdate(cmd.getCmdId(), 0);
ctx.sendWsMsg(update);
} else {
ctx.doFetchAlarmCount();
if (entitiesIds != null) {
ctx.createAlarmSubscriptions();
}
TbAlarmCountSubCtx finalCtx = ctx;
ScheduledFuture<?> task = scheduler.scheduleWithFixedDelay(
() -> refreshDynamicQuery(finalCtx),
dynamicPageLinkRefreshInterval, dynamicPageLinkRefreshInterval, TimeUnit.SECONDS);
finalCtx.setRefreshTask(task);
}
} else { } else {
log.debug("[{}][{}] Received duplicate command: {}", session.getSessionId(), cmd.getCmdId(), cmd); log.debug("[{}][{}] Received duplicate command: {}", session.getSessionId(), cmd.getCmdId(), cmd);
} }
@ -555,7 +563,7 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc
private TbAlarmCountSubCtx createSubCtx(WebSocketSessionRef sessionRef, AlarmCountCmd cmd) { private TbAlarmCountSubCtx createSubCtx(WebSocketSessionRef sessionRef, AlarmCountCmd cmd) {
Map<Integer, TbAbstractSubCtx> sessionSubs = subscriptionsBySessionId.computeIfAbsent(sessionRef.getSessionId(), k -> new ConcurrentHashMap<>()); Map<Integer, TbAbstractSubCtx> sessionSubs = subscriptionsBySessionId.computeIfAbsent(sessionRef.getSessionId(), k -> new ConcurrentHashMap<>());
TbAlarmCountSubCtx ctx = new TbAlarmCountSubCtx(serviceId, wsService, entityService, localSubscriptionService, TbAlarmCountSubCtx ctx = new TbAlarmCountSubCtx(serviceId, wsService, entityService, localSubscriptionService,
attributesService, stats, alarmService, sessionRef, cmd.getCmdId()); attributesService, stats, alarmService, sessionRef, cmd.getCmdId(), maxEntitiesPerAlarmSubscription, maxAlarmQueriesPerRefreshInterval);
if (cmd.getQuery() != null) { if (cmd.getQuery() != null) {
ctx.setAndResolveQuery(cmd.getQuery()); ctx.setAndResolveQuery(cmd.getQuery());
} }

119
application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java

@ -19,49 +19,152 @@ import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import lombok.ToString; import lombok.ToString;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmCountQuery;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataPageLink;
import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityDataSortOrder;
import org.thingsboard.server.common.data.query.EntityKey;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.alarm.AlarmService;
import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.service.ws.WebSocketService; import org.thingsboard.server.service.ws.WebSocketService;
import org.thingsboard.server.service.ws.WebSocketSessionRef; import org.thingsboard.server.service.ws.WebSocketSessionRef;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmCountUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmCountUpdate;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@Slf4j @Slf4j
@ToString(callSuper = true) @ToString(callSuper = true)
public class TbAlarmCountSubCtx extends TbAbstractEntityQuerySubCtx<AlarmCountQuery> { public class TbAlarmCountSubCtx extends TbAbstractEntityQuerySubCtx<AlarmCountQuery> {
private final AlarmService alarmService; private final AlarmService alarmService;
protected final Map<Integer, EntityId> subToEntityIdMap;
@Getter
private LinkedHashSet<EntityId> entitiesIds;
private final int maxEntitiesPerAlarmSubscription;
private final int maxAlarmQueriesPerRefreshInterval;
@Getter @Getter
@Setter @Setter
private volatile int result; private volatile int result;
@Getter
@Setter
private boolean tooManyEntities;
private int alarmCountInvocationAttempts;
public TbAlarmCountSubCtx(String serviceId, WebSocketService wsService, public TbAlarmCountSubCtx(String serviceId, WebSocketService wsService,
EntityService entityService, TbLocalSubscriptionService localSubscriptionService, EntityService entityService, TbLocalSubscriptionService localSubscriptionService,
AttributesService attributesService, SubscriptionServiceStatistics stats, AlarmService alarmService, AttributesService attributesService, SubscriptionServiceStatistics stats, AlarmService alarmService,
WebSocketSessionRef sessionRef, int cmdId) { WebSocketSessionRef sessionRef, int cmdId, int maxEntitiesPerAlarmSubscription, int maxAlarmQueriesPerRefreshInterval) {
super(serviceId, wsService, entityService, localSubscriptionService, attributesService, stats, sessionRef, cmdId); super(serviceId, wsService, entityService, localSubscriptionService, attributesService, stats, sessionRef, cmdId);
this.alarmService = alarmService; this.alarmService = alarmService;
this.subToEntityIdMap = new ConcurrentHashMap<>();
this.maxEntitiesPerAlarmSubscription = maxEntitiesPerAlarmSubscription;
this.maxAlarmQueriesPerRefreshInterval = maxAlarmQueriesPerRefreshInterval;
this.entitiesIds = null;
}
@Override
public void clearSubscriptions() {
clearAlarmSubscriptions();
} }
@Override @Override
public void fetchData() { public void fetchData() {
result = (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query); resetInvocationCounter();
sendWsMsg(new AlarmCountUpdate(cmdId, result)); if (query.getEntityFilter() != null) {
entitiesIds = new LinkedHashSet<>();
log.trace("[{}] Fetching data: {}", cmdId, alarmCountInvocationAttempts);
PageData<EntityData> data = entityService.findEntityDataByQuery(getTenantId(), getCustomerId(), buildEntityDataQuery());
entitiesIds.clear();
tooManyEntities = data.hasNext();
for (EntityData entityData : data.getData()) {
entitiesIds.add(entityData.getEntityId());
}
}
} }
@Override @Override
protected void update() { protected void update() {
int newCount = (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query); resetInvocationCounter();
if (newCount != result) { fetchAlarmCount();
result = newCount;
sendWsMsg(new AlarmCountUpdate(cmdId, result));
}
} }
@Override @Override
public boolean isDynamic() { public boolean isDynamic() {
return true; return true;
} }
public void fetchAlarmCount() {
alarmCountInvocationAttempts++;
log.trace("[{}] Fetching alarms: {}", cmdId, alarmCountInvocationAttempts);
if (alarmCountInvocationAttempts <= maxAlarmQueriesPerRefreshInterval) {
int newCount = (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query, entitiesIds);
if (newCount != result) {
result = newCount;
sendWsMsg(new AlarmCountUpdate(cmdId, result));
}
} else {
log.trace("[{}] Ignore alarm count fetch due to rate limit: [{}] of maximum [{}]", cmdId, alarmCountInvocationAttempts, maxAlarmQueriesPerRefreshInterval);
}
}
public void doFetchAlarmCount() {
result = (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query, entitiesIds);
sendWsMsg(new AlarmCountUpdate(cmdId, result));
}
private EntityDataQuery buildEntityDataQuery() {
EntityDataPageLink edpl = new EntityDataPageLink(maxEntitiesPerAlarmSubscription, 0, null,
new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, ModelConstants.CREATED_TIME_PROPERTY)));
return new EntityDataQuery(query.getEntityFilter(), edpl, null, null, query.getKeyFilters());
}
private void resetInvocationCounter() {
alarmCountInvocationAttempts = 0;
}
public void createAlarmSubscriptions() {
for (EntityId entityId : entitiesIds) {
createAlarmSubscriptionForEntity(entityId);
}
}
private void createAlarmSubscriptionForEntity(EntityId entityId) {
int subIdx = sessionRef.getSessionSubIdSeq().incrementAndGet();
subToEntityIdMap.put(subIdx, entityId);
log.trace("[{}][{}][{}] Creating alarms subscription for [{}] ", serviceId, cmdId, subIdx, entityId);
TbAlarmsSubscription subscription = TbAlarmsSubscription.builder()
.serviceId(serviceId)
.sessionId(sessionRef.getSessionId())
.subscriptionId(subIdx)
.tenantId(sessionRef.getSecurityCtx().getTenantId())
.entityId(entityId)
.updateProcessor((sub, update) -> fetchAlarmCount())
.build();
localSubscriptionService.addSubscription(subscription, sessionRef);
}
public void clearAlarmSubscriptions() {
if (subToEntityIdMap != null) {
for (Integer subId : subToEntityIdMap.keySet()) {
localSubscriptionService.cancelSubscription(getTenantId(), getSessionId(), subId);
}
subToEntityIdMap.clear();
}
}
} }

3
application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java

@ -39,6 +39,7 @@ import org.thingsboard.server.dao.alarm.AlarmService;
import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.sql.query.EntityKeyMapping;
import org.thingsboard.server.service.ws.WebSocketService; import org.thingsboard.server.service.ws.WebSocketService;
import org.thingsboard.server.service.ws.WebSocketSessionRef; import org.thingsboard.server.service.ws.WebSocketSessionRef;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataUpdate;
@ -359,7 +360,7 @@ public class TbAlarmDataSubCtx extends TbAbstractDataSubCtx<AlarmDataQuery> {
EntityDataSortOrder sortOrder = query.getPageLink().getSortOrder(); EntityDataSortOrder sortOrder = query.getPageLink().getSortOrder();
EntityDataSortOrder entitiesSortOrder; EntityDataSortOrder entitiesSortOrder;
if (sortOrder == null || sortOrder.getKey().getType().equals(EntityKeyType.ALARM_FIELD)) { if (sortOrder == null || sortOrder.getKey().getType().equals(EntityKeyType.ALARM_FIELD)) {
entitiesSortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, ModelConstants.CREATED_TIME_PROPERTY)); entitiesSortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, EntityKeyMapping.CREATED_TIME));
} else { } else {
entitiesSortOrder = sortOrder; entitiesSortOrder = sortOrder;
} }

5
application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java

@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.common.data.util.ThrowingRunnable; import org.thingsboard.server.common.data.util.ThrowingRunnable;
import org.thingsboard.server.dao.cf.CalculatedFieldService;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.queue.util.TbCoreComponent;
@ -61,6 +62,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS
private final Map<EntityType, EntityImportService<?, ?, ?>> importServices = new HashMap<>(); private final Map<EntityType, EntityImportService<?, ?, ?>> importServices = new HashMap<>();
private final RelationService relationService; private final RelationService relationService;
private final CalculatedFieldService calculatedFieldService;
private final RateLimitService rateLimitService; private final RateLimitService rateLimitService;
private final TbLogEntityActionService logEntityActionService; private final TbLogEntityActionService logEntityActionService;
@ -72,7 +74,6 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS
EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_RULE EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_RULE
); );
@Override @Override
public <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> exportEntity(EntitiesExportCtx<?> ctx, I entityId) throws ThingsboardException { public <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> exportEntity(EntitiesExportCtx<?> ctx, I entityId) throws ThingsboardException {
if (!rateLimitService.checkRateLimit(LimitedApi.ENTITY_EXPORT, ctx.getTenantId())) { if (!rateLimitService.checkRateLimit(LimitedApi.ENTITY_EXPORT, ctx.getTenantId())) {
@ -129,13 +130,11 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS
} }
} }
@Override @Override
public Comparator<EntityType> getEntityTypeComparatorForImport() { public Comparator<EntityType> getEntityTypeComparatorForImport() {
return Comparator.comparing(SUPPORTED_ENTITY_TYPES::indexOf); return Comparator.comparing(SUPPORTED_ENTITY_TYPES::indexOf);
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
private <I extends EntityId, E extends ExportableEntity<I>, D extends EntityExportData<E>> EntityExportService<I, E, D> getExportService(EntityType entityType) { private <I extends EntityId, E extends ExportableEntity<I>, D extends EntityExportData<E>> EntityExportService<I, E, D> getExportService(EntityType entityType) {
EntityExportService<?, ?, ?> exportService = exportServices.get(entityType); EntityExportService<?, ?, ?> exportService = exportServices.get(entityType);

1
application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java

@ -32,7 +32,6 @@ public interface EntitiesExportImportService {
<E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> importEntity(EntitiesImportCtx ctx, EntityExportData<E> exportData) throws ThingsboardException; <E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> importEntity(EntitiesImportCtx ctx, EntityExportData<E> exportData) throws ThingsboardException;
void saveReferencesAndRelations(EntitiesImportCtx ctx) throws ThingsboardException; void saveReferencesAndRelations(EntitiesImportCtx ctx) throws ThingsboardException;
Comparator<EntityType> getEntityTypeComparatorForImport(); Comparator<EntityType> getEntityTypeComparatorForImport();

21
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java

@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.HasVersion; import org.thingsboard.server.common.data.HasVersion;
import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EntityIdFactory;
@ -31,6 +32,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.common.data.sync.ie.AttributeExportData; import org.thingsboard.server.common.data.sync.ie.AttributeExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.cf.CalculatedFieldService;
import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.dao.relation.RelationDao;
import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService;
@ -59,6 +61,8 @@ public class DefaultEntityExportService<I extends EntityId, E extends Exportable
private RelationDao relationDao; private RelationDao relationDao;
@Autowired @Autowired
private AttributesService attributesService; private AttributesService attributesService;
@Autowired
private CalculatedFieldService calculatedFieldService;
@Override @Override
public final D getExportData(EntitiesExportCtx<?> ctx, I entityId) throws ThingsboardException { public final D getExportData(EntitiesExportCtx<?> ctx, I entityId) throws ThingsboardException {
@ -98,6 +102,10 @@ public class DefaultEntityExportService<I extends EntityId, E extends Exportable
Map<String, List<AttributeExportData>> attributes = exportAttributes(ctx, entity); Map<String, List<AttributeExportData>> attributes = exportAttributes(ctx, entity);
exportData.setAttributes(attributes); exportData.setAttributes(attributes);
} }
if (ctx.getSettings().isExportCalculatedFields()) {
List<CalculatedField> calculatedFields = exportCalculatedFields(ctx, entity.getId());
exportData.setCalculatedFields(calculatedFields);
}
} }
private List<EntityRelation> exportRelations(EntitiesExportCtx<?> ctx, E entity) throws ThingsboardException { private List<EntityRelation> exportRelations(EntitiesExportCtx<?> ctx, E entity) throws ThingsboardException {
@ -141,6 +149,19 @@ public class DefaultEntityExportService<I extends EntityId, E extends Exportable
return attributes; return attributes;
} }
private List<CalculatedField> exportCalculatedFields(EntitiesExportCtx<?> ctx, EntityId entityId) {
List<CalculatedField> calculatedFields = calculatedFieldService.findCalculatedFieldsByEntityId(ctx.getTenantId(), entityId);
calculatedFields.forEach(calculatedField -> {
calculatedField.setEntityId(getExternalIdOrElseInternal(ctx, entityId));
calculatedField.getConfiguration().getArguments().values().forEach(argument -> {
if (argument.getRefEntityId() != null) {
argument.setRefEntityId(getExternalIdOrElseInternal(ctx, argument.getRefEntityId()));
}
});
});
return calculatedFields;
}
protected <ID extends EntityId> ID getExternalIdOrElseInternal(EntitiesExportCtx<?> ctx, ID internalId) { protected <ID extends EntityId> ID getExternalIdOrElseInternal(EntitiesExportCtx<?> ctx, ID internalId) {
if (internalId == null || internalId.isNullUid()) return internalId; if (internalId == null || internalId.isNullUid()) return internalId;
var result = ctx.getExternalId(internalId); var result = ctx.getExternalId(internalId);

6
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java

@ -47,7 +47,11 @@ public class AssetImportService extends BaseEntityImportService<AssetId, Asset,
@Override @Override
protected Asset saveOrUpdate(EntitiesImportCtx ctx, Asset asset, EntityExportData<Asset> exportData, IdProvider idProvider) { protected Asset saveOrUpdate(EntitiesImportCtx ctx, Asset asset, EntityExportData<Asset> exportData, IdProvider idProvider) {
return assetService.saveAsset(asset); Asset savedAsset = assetService.saveAsset(asset);
if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) {
importCalculatedFields(ctx, savedAsset, exportData, idProvider);
}
return savedAsset;
} }
@Override @Override

6
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java

@ -50,7 +50,11 @@ public class AssetProfileImportService extends BaseEntityImportService<AssetProf
@Override @Override
protected AssetProfile saveOrUpdate(EntitiesImportCtx ctx, AssetProfile assetProfile, EntityExportData<AssetProfile> exportData, IdProvider idProvider) { protected AssetProfile saveOrUpdate(EntitiesImportCtx ctx, AssetProfile assetProfile, EntityExportData<AssetProfile> exportData, IdProvider idProvider) {
return assetProfileService.saveAssetProfile(assetProfile); AssetProfile saved = assetProfileService.saveAssetProfile(assetProfile);
if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) {
importCalculatedFields(ctx, saved, exportData, idProvider);
}
return saved;
} }
@Override @Override

57
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java

@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.HasDefaultOption;
import org.thingsboard.server.common.data.HasVersion; import org.thingsboard.server.common.data.HasVersion;
import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EntityIdFactory;
@ -50,6 +51,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.common.data.sync.ie.AttributeExportData; import org.thingsboard.server.common.data.sync.ie.AttributeExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.dao.cf.CalculatedFieldService;
import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.dao.relation.RelationDao;
import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.action.EntityActionService;
@ -78,6 +80,8 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
@Lazy @Lazy
private ExportableEntitiesService entitiesService; private ExportableEntitiesService entitiesService;
@Autowired @Autowired
private CalculatedFieldService calculatedFieldService;
@Autowired
private RelationService relationService; private RelationService relationService;
@Autowired @Autowired
private RelationDao relationDao; private RelationDao relationDao;
@ -134,7 +138,7 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
} }
protected boolean updateRelatedEntitiesIfUnmodified(EntitiesImportCtx ctx, E prepared, D exportData, IdProvider idProvider) { protected boolean updateRelatedEntitiesIfUnmodified(EntitiesImportCtx ctx, E prepared, D exportData, IdProvider idProvider) {
return false; return importCalculatedFields(ctx, prepared, exportData, idProvider);
} }
@Override @Override
@ -278,12 +282,59 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
}); });
} }
protected boolean importCalculatedFields(EntitiesImportCtx ctx, E savedEntity, D exportData, IdProvider idProvider) {
if (exportData.getCalculatedFields() == null || !ctx.isSaveCalculatedFields()) {
return false;
}
boolean updated = false;
List<CalculatedField> existing = calculatedFieldService.findCalculatedFieldsByEntityId(ctx.getTenantId(), savedEntity.getId());
List<CalculatedField> fieldsToSave = exportData.getCalculatedFields().stream()
.peek(calculatedField -> {
calculatedField.setTenantId(ctx.getTenantId());
calculatedField.setEntityId(savedEntity.getId());
calculatedField.getConfiguration().getArguments().values().forEach(argument -> {
if (argument.getRefEntityId() != null) {
argument.setRefEntityId(idProvider.getInternalId(argument.getRefEntityId(), ctx.isFinalImportAttempt()));
}
});
}).toList();
for (CalculatedField existingField : existing) {
boolean found = fieldsToSave.stream().anyMatch(importedField -> compareCalculatedFields(existingField, importedField));
if (!found) {
calculatedFieldService.deleteCalculatedField(ctx.getTenantId(), existingField.getId());
updated = true;
}
}
for (CalculatedField calculatedField : fieldsToSave) {
boolean found = existing.stream().anyMatch(existingField -> compareCalculatedFields(existingField, calculatedField));
if (!found) {
calculatedFieldService.save(calculatedField);
updated = true;
}
}
return updated;
}
private boolean compareCalculatedFields(CalculatedField existingField, CalculatedField newField) {
CalculatedField oldCopy = new CalculatedField(existingField);
CalculatedField newCopy = new CalculatedField(newField);
oldCopy.setId(null);
newCopy.setId(null);
oldCopy.setVersion(null);
newCopy.setVersion(null);
oldCopy.setCreatedTime(0);
newCopy.setCreatedTime(0);
return oldCopy.equals(newCopy);
}
protected void onEntitySaved(User user, E savedEntity, E oldEntity) throws ThingsboardException { protected void onEntitySaved(User user, E savedEntity, E oldEntity) throws ThingsboardException {
logEntityActionService.logEntityAction(user.getTenantId(), savedEntity.getId(), savedEntity, null, logEntityActionService.logEntityAction(user.getTenantId(), savedEntity.getId(), savedEntity, null,
oldEntity == null ? ActionType.ADDED : ActionType.UPDATED, user); oldEntity == null ? ActionType.ADDED : ActionType.UPDATED, user);
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
protected E findExistingEntity(EntitiesImportCtx ctx, E entity, IdProvider idProvider) { protected E findExistingEntity(EntitiesImportCtx ctx, E entity, IdProvider idProvider) {
return (E) Optional.ofNullable(entitiesService.findEntityByTenantIdAndExternalId(ctx.getTenantId(), entity.getId())) return (E) Optional.ofNullable(entitiesService.findEntityByTenantIdAndExternalId(ctx.getTenantId(), entity.getId()))
@ -313,10 +364,10 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
.orElseThrow(() -> new MissingEntityException(externalId)); .orElseThrow(() -> new MissingEntityException(externalId));
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@RequiredArgsConstructor @RequiredArgsConstructor
protected class IdProvider { protected class IdProvider {
private final EntitiesImportCtx ctx; private final EntitiesImportCtx ctx;
private final EntityImportResult<E> importResult; private final EntityImportResult<E> importResult;

9
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java

@ -64,13 +64,18 @@ public class DeviceImportService extends BaseEntityImportService<DeviceId, Devic
@Override @Override
protected Device saveOrUpdate(EntitiesImportCtx ctx, Device device, DeviceExportData exportData, IdProvider idProvider) { protected Device saveOrUpdate(EntitiesImportCtx ctx, Device device, DeviceExportData exportData, IdProvider idProvider) {
Device savedDevice;
if (exportData.getCredentials() != null && ctx.isSaveCredentials()) { if (exportData.getCredentials() != null && ctx.isSaveCredentials()) {
exportData.getCredentials().setId(null); exportData.getCredentials().setId(null);
exportData.getCredentials().setDeviceId(null); exportData.getCredentials().setDeviceId(null);
return deviceService.saveDeviceWithCredentials(device, exportData.getCredentials()); savedDevice = deviceService.saveDeviceWithCredentials(device, exportData.getCredentials());
} else { } else {
return deviceService.saveDevice(device); savedDevice = deviceService.saveDevice(device);
} }
if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) {
importCalculatedFields(ctx, savedDevice, exportData, idProvider);
}
return savedDevice;
} }
@Override @Override

6
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java

@ -52,7 +52,11 @@ public class DeviceProfileImportService extends BaseEntityImportService<DevicePr
@Override @Override
protected DeviceProfile saveOrUpdate(EntitiesImportCtx ctx, DeviceProfile deviceProfile, EntityExportData<DeviceProfile> exportData, IdProvider idProvider) { protected DeviceProfile saveOrUpdate(EntitiesImportCtx ctx, DeviceProfile deviceProfile, EntityExportData<DeviceProfile> exportData, IdProvider idProvider) {
return deviceProfileService.saveDeviceProfile(deviceProfile); DeviceProfile saved = deviceProfileService.saveDeviceProfile(deviceProfile);
if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) {
importCalculatedFields(ctx, saved, exportData, idProvider);
}
return saved;
} }
@Override @Override

10
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java

@ -94,7 +94,6 @@ import java.util.Set;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.function.Function; import java.util.function.Function;
import java.util.stream.Collectors;
import static com.google.common.util.concurrent.Futures.transform; import static com.google.common.util.concurrent.Futures.transform;
import static org.thingsboard.server.common.data.sync.vc.VcUtils.checkBranchName; import static org.thingsboard.server.common.data.sync.vc.VcUtils.checkBranchName;
@ -304,6 +303,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
.updateRelations(config.isLoadRelations()) .updateRelations(config.isLoadRelations())
.saveAttributes(config.isLoadAttributes()) .saveAttributes(config.isLoadAttributes())
.saveCredentials(config.isLoadCredentials()) .saveCredentials(config.isLoadCredentials())
.saveCalculatedFields(config.isLoadCalculatedFields())
.findExistingByName(false) .findExistingByName(false)
.build()); .build());
ctx.setFinalImportAttempt(true); ctx.setFinalImportAttempt(true);
@ -327,7 +327,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
var sw = TbStopWatch.create("before"); var sw = TbStopWatch.create("before");
List<EntityType> entityTypes = request.getEntityTypes().keySet().stream() List<EntityType> entityTypes = request.getEntityTypes().keySet().stream()
.sorted(exportImportService.getEntityTypeComparatorForImport()).collect(Collectors.toList()); .sorted(exportImportService.getEntityTypeComparatorForImport()).toList();
for (EntityType entityType : entityTypes) { for (EntityType entityType : entityTypes) {
log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType); log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType);
sw.startNew("Entities " + entityType.name()); sw.startNew("Entities " + entityType.name());
@ -362,6 +362,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
.updateRelations(config.isLoadRelations()) .updateRelations(config.isLoadRelations())
.saveAttributes(config.isLoadAttributes()) .saveAttributes(config.isLoadAttributes())
.saveCredentials(config.isLoadCredentials()) .saveCredentials(config.isLoadCredentials())
.saveCalculatedFields(config.isLoadCalculatedFields())
.findExistingByName(config.isFindExistingEntityByName()) .findExistingByName(config.isFindExistingEntityByName())
.build(); .build();
} }
@ -471,7 +472,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
} }
@Override @Override
public ListenableFuture<EntityDataDiff> compareEntityDataToVersion(User user, EntityId entityId, String versionId) throws Exception { public ListenableFuture<EntityDataDiff> compareEntityDataToVersion(User user, EntityId entityId, String versionId) {
HasId<EntityId> entity = exportableEntitiesService.findEntityByTenantIdAndId(user.getTenantId(), entityId); HasId<EntityId> entity = exportableEntitiesService.findEntityByTenantIdAndId(user.getTenantId(), entityId);
if (!(entity instanceof ExportableEntity)) throw new IllegalArgumentException("Unsupported entity type"); if (!(entity instanceof ExportableEntity)) throw new IllegalArgumentException("Unsupported entity type");
@ -484,6 +485,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
.exportRelations(otherVersion.hasRelations()) .exportRelations(otherVersion.hasRelations())
.exportAttributes(otherVersion.hasAttributes()) .exportAttributes(otherVersion.hasAttributes())
.exportCredentials(otherVersion.hasCredentials()) .exportCredentials(otherVersion.hasCredentials())
.exportCalculatedFields(otherVersion.hasCalculatedFields())
.build()); .build());
EntityExportData<?> currentVersion; EntityExportData<?> currentVersion;
try { try {
@ -503,7 +505,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
@Override @Override
public ListenableFuture<List<BranchInfo>> listBranches(TenantId tenantId) throws Exception { public ListenableFuture<List<BranchInfo>> listBranches(TenantId tenantId) {
return gitServiceQueue.listBranches(tenantId); return gitServiceQueue.listBranches(tenantId);
} }

2
application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java

@ -69,6 +69,7 @@ public abstract class EntitiesExportCtx<R extends VersionCreateRequest> {
.exportRelations(config.isSaveRelations()) .exportRelations(config.isSaveRelations())
.exportAttributes(config.isSaveAttributes()) .exportAttributes(config.isSaveAttributes())
.exportCredentials(config.isSaveCredentials()) .exportCredentials(config.isSaveCredentials())
.exportCalculatedFields(config.isSaveCalculatedFields())
.build(); .build();
} }
@ -85,4 +86,5 @@ public abstract class EntitiesExportCtx<R extends VersionCreateRequest> {
log.debug("[{}][{}] Local cache put: {}", internalId.getEntityType(), internalId.getId(), externalId); log.debug("[{}][{}] Local cache put: {}", internalId.getEntityType(), internalId.getId(), externalId);
externalIdMap.put(internalId, externalId != null ? externalId : internalId); externalIdMap.put(internalId, externalId != null ? externalId : internalId);
} }
} }

5
application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java

@ -91,6 +91,10 @@ public class EntitiesImportCtx {
return getSettings().isSaveCredentials(); return getSettings().isSaveCredentials();
} }
public boolean isSaveCalculatedFields() {
return getSettings().isSaveCalculatedFields();
}
public EntityId getInternalId(EntityId externalId) { public EntityId getInternalId(EntityId externalId) {
var result = externalToInternalIdMap.get(externalId); var result = externalToInternalIdMap.get(externalId);
log.debug("[{}][{}] Local cache {} for id", externalId.getEntityType(), externalId.getId(), result != null ? "hit" : "miss"); log.debug("[{}][{}] Local cache {} for id", externalId.getEntityType(), externalId.getId(), result != null ? "hit" : "miss");
@ -140,5 +144,4 @@ public class EntitiesImportCtx {
return notFoundIds.contains(externalId); return notFoundIds.contains(externalId);
} }
} }

1
application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java

@ -39,6 +39,7 @@ public class EntityTypeExportCtx extends EntitiesExportCtx<VersionCreateRequest>
.exportRelations(config.isSaveRelations()) .exportRelations(config.isSaveRelations())
.exportAttributes(config.isSaveAttributes()) .exportAttributes(config.isSaveAttributes())
.exportCredentials(config.isSaveCredentials()) .exportCredentials(config.isSaveCredentials())
.exportCalculatedFields(config.isSaveCalculatedFields())
.build(); .build();
this.overwrite = ObjectUtils.defaultIfNull(config.getSyncStrategy(), defaultSyncStrategy) == SyncStrategy.OVERWRITE; this.overwrite = ObjectUtils.defaultIfNull(config.getSyncStrategy(), defaultSyncStrategy) == SyncStrategy.OVERWRITE;
} }

2
application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java

@ -370,7 +370,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
} }
private <S> void addMainCallback(ListenableFuture<S> saveFuture, Consumer<S> onSuccess) { private <S> void addMainCallback(ListenableFuture<S> saveFuture, Consumer<S> onSuccess) {
DonAsynchron.withCallback(saveFuture, onSuccess, null, tsCallBackExecutor); addMainCallback(saveFuture, onSuccess, null);
} }
private <S> void addMainCallback(ListenableFuture<S> saveFuture, Consumer<S> onSuccess, Consumer<Throwable> onFailure) { private <S> void addMainCallback(ListenableFuture<S> saveFuture, Consumer<S> onSuccess, Consumer<Throwable> onFailure) {

3
application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java

@ -93,7 +93,8 @@ public class TbCoreTransportApiService {
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
log.info("Received application ready event. Starting polling for events."); log.info("Received application ready event. Starting polling for events.");
transportApiTemplate.init(transportApiService); transportApiTemplate.subscribe();
transportApiTemplate.launch(transportApiService);
} }
@PreDestroy @PreDestroy

1
application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java

@ -229,6 +229,7 @@ public class DefaultWebSocketService implements WebSocketService {
} catch (TbRateLimitsException e) { } catch (TbRateLimitsException e) {
log.debug("{} Failed to handle WS cmd: {}", sessionRef, cmd, e); log.debug("{} Failed to handle WS cmd: {}", sessionRef, cmd, e);
} catch (Exception e) { } catch (Exception e) {
sendError(sessionRef, cmd.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR, e.getMessage());
log.error("{} Failed to handle WS cmd: {}", sessionRef, cmd, e); log.error("{} Failed to handle WS cmd: {}", sessionRef, cmd, e);
} }
} }

66
application/src/main/resources/thingsboard.yml

@ -512,6 +512,8 @@ actors:
enabled: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_ENABLED:true}" enabled: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_ENABLED:true}"
# The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour # The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour
configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}"
# Time in seconds to receive calculation result.
calculation_timeout: "${ACTORS_CALCULATION_TIMEOUT_SEC:5}"
debug: debug:
settings: settings:
@ -1597,6 +1599,16 @@ queue:
- key: max.poll.records - key: max.poll.records
# Amount of records to be returned in a single poll. For Housekeeper reprocessing topic, we should consume messages (tasks) one by one # Amount of records to be returned in a single poll. For Housekeeper reprocessing topic, we should consume messages (tasks) one by one
value: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_MAX_POLL_RECORDS:1}" value: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_MAX_POLL_RECORDS:1}"
edqs.events:
# Key-value properties for Kafka consumer for edqs.events topic
- key: max.poll.records
# Max poll records for edqs.events topic
value: "${TB_QUEUE_KAFKA_EDQS_EVENTS_MAX_POLL_RECORDS:512}"
edqs.state:
# Key-value properties for Kafka consumer for edqs.state topic
- key: max.poll.records
# Max poll records for edqs.state topic
value: "${TB_QUEUE_KAFKA_EDQS_STATE_MAX_POLL_RECORDS:512}"
other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000"
other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside
# - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms
@ -1630,6 +1642,12 @@ queue:
calculated-field: "${TB_QUEUE_KAFKA_CF_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" calculated-field: "${TB_QUEUE_KAFKA_CF_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}"
# Kafka properties for Calculated Field State topics # Kafka properties for Calculated Field State topics
calculated-field-state: "${TB_QUEUE_KAFKA_CF_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:104857600000;partitions:1;min.insync.replicas:1;cleanup.policy:compact}" calculated-field-state: "${TB_QUEUE_KAFKA_CF_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:104857600000;partitions:1;min.insync.replicas:1;cleanup.policy:compact}"
# Kafka properties for EDQS events topics. Partitions number must be the same as queue.edqs.partitions
edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1}"
# Kafka properties for EDQS requests topic (default: 3 minutes retention). Partitions number must be the same as queue.edqs.partitions
edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}"
# Kafka properties for EDQS state topic (infinite retention, compaction). Partitions number must be the same as queue.edqs.partitions
edqs-state: "${TB_QUEUE_KAFKA_EDQS_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1;cleanup.policy:compact}"
consumer-stats: consumer-stats:
# Prints lag between consumer group offset and last messages offset in Kafka topics # Prints lag between consumer group offset and last messages offset in Kafka topics
enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}"
@ -1659,6 +1677,8 @@ queue:
core: core:
# Default topic name # Default topic name
topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" topic: "${TB_QUEUE_CORE_TOPIC:tb_core}"
# For high-priority notifications that require minimum latency and processing time
notifications_topic: "${TB_QUEUE_CORE_NOTIFICATIONS_TOPIC:tb_core.notifications}"
# Interval in milliseconds to poll messages by Core microservices # Interval in milliseconds to poll messages by Core microservices
poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}"
# Amount of partitions used by Core microservices # Amount of partitions used by Core microservices
@ -1703,6 +1723,44 @@ queue:
enabled: "${TB_HOUSEKEEPER_STATS_ENABLED:true}" enabled: "${TB_HOUSEKEEPER_STATS_ENABLED:true}"
# Statistics printing interval for Housekeeper # Statistics printing interval for Housekeeper
print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}" print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}"
edqs:
sync:
# Enable/disable EDQS synchronization
enabled: "${TB_EDQS_SYNC_ENABLED:false}"
# Batch size of entities being synced with EDQS
entity_batch_size: "${TB_EDQS_SYNC_ENTITY_BATCH_SIZE:10000}"
# Batch size of timeseries data being synced with EDQS
ts_batch_size: "${TB_EDQS_SYNC_TS_BATCH_SIZE:10000}"
api:
# Whether to forward entity data query requests to EDQS (otherwise use PostgreSQL implementation)
supported: "${TB_EDQS_API_SUPPORTED:false}"
# Whether to auto-enable EDQS API (if queue.edqs.api.supported is true) when sync of data to Kafka is finished
auto_enable: "${TB_EDQS_API_AUTO_ENABLE:true}"
# Mode of EDQS: local (for monolith) or remote (with separate EDQS microservices)
mode: "${TB_EDQS_MODE:local}"
local:
# Path to RocksDB for EDQS backup when running in local mode
rocksdb_path: "${TB_EDQS_ROCKSDB_PATH:${user.home}/.rocksdb/edqs}"
# Number of partitions for EDQS topics
partitions: "${TB_EDQS_PARTITIONS:12}"
# EDQS partitioning strategy: tenant (partition is resolved by tenant id) or none (no specific strategy, resolving by message key)
partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}"
# EDQS requests topic
requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}"
# EDQS responses topic
responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}"
# Poll interval for EDQS topics
poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}"
# Maximum amount of pending requests to EDQS
max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}"
# Maximum timeout for requests to EDQS
max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:20000}"
stats:
# Enable/disable statistics for EDQS
enabled: "${TB_EDQS_STATS_ENABLED:true}"
# Statistics printing interval for EDQS
print-interval-ms: "${TB_EDQS_STATS_PRINT_INTERVAL_MS:300000}"
vc: vc:
# Default topic name # Default topic name
topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}"
@ -1734,6 +1792,8 @@ queue:
rule-engine: rule-engine:
# Deprecated. It will be removed in the nearest releases # Deprecated. It will be removed in the nearest releases
topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}"
# For high-priority notifications that require minimum latency and processing time
notifications_topic: "${TB_QUEUE_RULE_ENGINE_NOTIFICATIONS_TOPIC:tb_rule_engine.notifications}"
# Interval in milliseconds to poll messages by Rule Engine # Interval in milliseconds to poll messages by Rule Engine
poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}"
# Timeout for processing a message pack of Rule Engine # Timeout for processing a message pack of Rule Engine
@ -1754,6 +1814,8 @@ queue:
event_topic: "${TB_QUEUE_CF_EVENT_TOPIC:tb_cf_event}" event_topic: "${TB_QUEUE_CF_EVENT_TOPIC:tb_cf_event}"
# Topic name for Calculated Field (CF) compacted states # Topic name for Calculated Field (CF) compacted states
state_topic: "${TB_QUEUE_CF_STATE_TOPIC:tb_cf_state}" state_topic: "${TB_QUEUE_CF_STATE_TOPIC:tb_cf_state}"
# For high-priority notifications that require minimum latency and processing time
notifications_topic: "${TB_QUEUE_CF_NOTIFICATIONS_TOPIC:calculated_field.notifications}"
# Interval in milliseconds to poll messages by CF (Rule Engine) microservices # Interval in milliseconds to poll messages by CF (Rule Engine) microservices
poll_interval: "${TB_QUEUE_CF_POLL_INTERVAL_MS:25}" poll_interval: "${TB_QUEUE_CF_POLL_INTERVAL_MS:25}"
# Amount of partitions used by CF microservices # Amount of partitions used by CF microservices
@ -1772,6 +1834,10 @@ queue:
edge: edge:
# Default topic name # Default topic name
topic: "${TB_QUEUE_EDGE_TOPIC:tb_edge}" topic: "${TB_QUEUE_EDGE_TOPIC:tb_edge}"
# For high-priority notifications that require minimum latency and processing time
notifications_topic: "${TB_QUEUE_EDGE_NOTIFICATIONS_TOPIC:tb_edge.notifications}"
# For edge events messages
event_notifications_topic: "${TB_QUEUE_EDGE_EVENT_NOTIFICATIONS_TOPIC:tb_edge_event.notifications}"
# Amount of partitions used by Edge services # Amount of partitions used by Edge services
partitions: "${TB_QUEUE_EDGE_PARTITIONS:10}" partitions: "${TB_QUEUE_EDGE_PARTITIONS:10}"
# Poll interval for topics related to Edge services # Poll interval for topics related to Edge services

2
application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java

@ -402,7 +402,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
Awaitility.await("all tasks processed").atMost(60, TimeUnit.SECONDS).during(300, TimeUnit.MILLISECONDS) Awaitility.await("all tasks processed").atMost(90, TimeUnit.SECONDS).during(300, TimeUnit.MILLISECONDS)
.until(() -> storage.getLag("tb_housekeeper") == 0); .until(() -> storage.getLag("tb_housekeeper") == 0);
} }

72
application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java

@ -0,0 +1,72 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.controller;
import org.junit.Before;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.TestPropertySource;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.query.EntityCountQuery;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.msg.edqs.EdqsApiService;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.edqs.state.EdqsStateService;
import org.thingsboard.server.edqs.util.EdqsRocksDb;
import java.util.concurrent.TimeUnit;
import static org.awaitility.Awaitility.await;
@DaoSqlTest
@TestPropertySource(properties = {
// "queue.type=kafka", // uncomment to use Kafka
// "queue.kafka.bootstrap.servers=10.7.1.254:9092",
"queue.edqs.sync.enabled=true",
"queue.edqs.api.supported=true",
"queue.edqs.api.auto_enable=true",
"queue.edqs.mode=local"
})
public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest {
@Autowired
private EdqsApiService edqsApiService;
@Autowired
private EdqsStateService edqsStateService;
@MockBean // so that we don't do backup for tests
private EdqsRocksDb edqsRocksDb;
@Before
public void before() {
await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsApiService.isEnabled() && edqsStateService.isReady());
}
@Override
protected PageData<EntityData> findByQueryAndCheck(EntityDataQuery query, int expectedResultSize) {
return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findByQuery(query),
result -> result.getTotalElements() == expectedResultSize);
}
@Override
protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult) {
return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(query),
result -> result == expectedResult);
}
}

198
application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java

@ -18,12 +18,14 @@ package org.thingsboard.server.controller;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import org.awaitility.Awaitility; import org.awaitility.Awaitility;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.After; import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.web.servlet.ResultActions; import org.springframework.test.web.servlet.ResultActions;
import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Dashboard;
@ -49,6 +51,7 @@ import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityDataSortOrder;
import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKey;
import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.EntityKeyValueType;
import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.query.EntityListFilter;
import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.common.data.query.EntityTypeFilter;
import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.query.FilterPredicateValue;
@ -70,9 +73,11 @@ import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.function.BiPredicate;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@DaoSqlTest @DaoSqlTest
@ -130,36 +135,25 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
filter.setDeviceNameFilter(""); filter.setDeviceNameFilter("");
EntityCountQuery countQuery = new EntityCountQuery(filter); EntityCountQuery countQuery = new EntityCountQuery(filter);
countByQueryAndCheck(countQuery, 97);
Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
Assert.assertEquals(97, count.longValue());
filter.setDeviceTypes(List.of("unknown")); filter.setDeviceTypes(List.of("unknown"));
count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); countByQueryAndCheck(countQuery, 0);
Assert.assertEquals(0, count.longValue());
filter.setDeviceTypes(List.of("default")); filter.setDeviceTypes(List.of("default"));
filter.setDeviceNameFilter("Device1"); filter.setDeviceNameFilter("Device1");
countByQueryAndCheck(countQuery, 11);
count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
Assert.assertEquals(11, count.longValue());
EntityListFilter entityListFilter = new EntityListFilter(); EntityListFilter entityListFilter = new EntityListFilter();
entityListFilter.setEntityType(EntityType.DEVICE); entityListFilter.setEntityType(EntityType.DEVICE);
entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList())); entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList()));
countQuery = new EntityCountQuery(entityListFilter); countQuery = new EntityCountQuery(entityListFilter);
countByQueryAndCheck(countQuery, 97);
count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
Assert.assertEquals(97, count.longValue());
EntityTypeFilter filter2 = new EntityTypeFilter(); EntityTypeFilter filter2 = new EntityTypeFilter();
filter2.setEntityType(EntityType.DEVICE); filter2.setEntityType(EntityType.DEVICE);
countQuery = new EntityCountQuery(filter2);
EntityCountQuery countQuery2 = new EntityCountQuery(filter2); countByQueryAndCheck(countQuery, 97);
Long count2 = doPostWithResponse("/api/entitiesQuery/count", countQuery2, Long.class);
Assert.assertEquals(97, count2.longValue());
} }
@Test @Test
@ -169,51 +163,44 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
EntityTypeFilter allDeviceFilter = new EntityTypeFilter(); EntityTypeFilter allDeviceFilter = new EntityTypeFilter();
allDeviceFilter.setEntityType(EntityType.DEVICE); allDeviceFilter.setEntityType(EntityType.DEVICE);
EntityCountQuery query = new EntityCountQuery(allDeviceFilter); EntityCountQuery query = new EntityCountQuery(allDeviceFilter);
Long initialCount = doPostWithResponse("/api/entitiesQuery/count", query, Long.class); countByQueryAndCheck(query, 0);
loginTenantAdmin(); loginTenantAdmin();
List<Device> devices = new ArrayList<>(); List<Device> devices = new ArrayList<>();
String devicePrefix = "Device" + RandomStringUtils.randomAlphabetic(5);
for (int i = 0; i < 97; i++) { for (int i = 0; i < 97; i++) {
Device device = new Device(); Device device = new Device();
device.setName("Device" + i); device.setName(devicePrefix + i);
device.setType("default"); device.setType("default");
device.setLabel("testLabel" + (int) (Math.random() * 1000)); device.setLabel("testLabel" + (int) (Math.random() * 1000));
devices.add(doPost("/api/device", device, Device.class)); devices.add(doPost("/api/device", device, Device.class));
Thread.sleep(1); Thread.sleep(1);
} }
DeviceTypeFilter filter = new DeviceTypeFilter(); DeviceTypeFilter filter = new DeviceTypeFilter();
filter.setDeviceType("default"); filter.setDeviceTypes(List.of("default"));
filter.setDeviceNameFilter(""); filter.setDeviceNameFilter("");
loginSysAdmin(); loginSysAdmin();
EntityCountQuery countQuery = new EntityCountQuery(filter); EntityCountQuery countQuery = new EntityCountQuery(filter);
countByQueryAndCheck(countQuery, 97);
Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); filter.setDeviceTypes(List.of("unknown"));
Assert.assertEquals(97, count.longValue()); countByQueryAndCheck(countQuery, 0);
filter.setDeviceType("unknown");
count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
Assert.assertEquals(0, count.longValue());
filter.setDeviceType("default");
filter.setDeviceNameFilter("Device1");
count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); filter.setDeviceTypes(List.of("default"));
Assert.assertEquals(11, count.longValue()); filter.setDeviceNameFilter(devicePrefix + "1");
countByQueryAndCheck(countQuery, 11);
EntityListFilter entityListFilter = new EntityListFilter(); EntityListFilter entityListFilter = new EntityListFilter();
entityListFilter.setEntityType(EntityType.DEVICE); entityListFilter.setEntityType(EntityType.DEVICE);
entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList())); entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList()));
countQuery = new EntityCountQuery(entityListFilter); countQuery = new EntityCountQuery(entityListFilter);
countByQueryAndCheck(countQuery, 97);
count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); countByQueryAndCheck(countQuery, 97);
Assert.assertEquals(97, count.longValue());
Long count2 = doPostWithResponse("/api/entitiesQuery/count", query, Long.class);
Assert.assertEquals(initialCount + 97, count2.longValue());
} }
@Test @Test
@ -371,11 +358,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
PageData<EntityData> data = PageData<EntityData> data = findByQueryAndCheck(query, 97);
doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
});
Assert.assertEquals(97, data.getTotalElements());
Assert.assertEquals(10, data.getTotalPages()); Assert.assertEquals(10, data.getTotalPages());
Assert.assertTrue(data.hasNext()); Assert.assertTrue(data.hasNext());
Assert.assertEquals(10, data.getData().size()); Assert.assertEquals(10, data.getData().size());
@ -383,8 +366,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
List<EntityData> loadedEntities = new ArrayList<>(data.getData()); List<EntityData> loadedEntities = new ArrayList<>(data.getData());
while (data.hasNext()) { while (data.hasNext()) {
query = query.next(); query = query.next();
data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() { data = findByQuery(query);
});
loadedEntities.addAll(data.getData()); loadedEntities.addAll(data.getData());
} }
Assert.assertEquals(97, loadedEntities.size()); Assert.assertEquals(97, loadedEntities.size());
@ -406,8 +388,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
pageLink = new EntityDataPageLink(10, 0, "device1", sortOrder); pageLink = new EntityDataPageLink(10, 0, "device1", sortOrder);
query = new EntityDataQuery(filter, pageLink, entityFields, null, null); query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() { data = findByQuery(query);
});
Assert.assertEquals(11, data.getTotalElements()); Assert.assertEquals(11, data.getTotalElements());
Assert.assertEquals("Device19", data.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); Assert.assertEquals("Device19", data.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue());
@ -423,9 +404,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
EntityDataQuery query2 = new EntityDataQuery(filter2, pageLink2, entityFields2, null, null); EntityDataQuery query2 = new EntityDataQuery(filter2, pageLink2, entityFields2, null, null);
PageData<EntityData> data2 = PageData<EntityData> data2 = findByQuery(query2);
doPostWithTypedResponse("/api/entitiesQuery/find", query2, new TypeReference<PageData<EntityData>>() {
});
Assert.assertEquals(97, data2.getTotalElements()); Assert.assertEquals(97, data2.getTotalElements());
Assert.assertEquals(10, data2.getTotalPages()); Assert.assertEquals(10, data2.getTotalPages());
@ -473,20 +452,15 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
List<EntityKey> entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); List<EntityKey> entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"));
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
findByQueryAndCheck(query, 87);
PageData<EntityData> data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {});
Assert.assertEquals(87, data.getTotalElements());
filter.setFilters(List.of(new RelationEntityTypeFilter("NOT_CONTAINS", List.of(EntityType.DEVICE), false))); filter.setFilters(List.of(new RelationEntityTypeFilter("NOT_CONTAINS", List.of(EntityType.DEVICE), false)));
query = new EntityDataQuery(filter, pageLink, entityFields, null, null); query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {}); findByQueryAndCheck(query, 10);
Assert.assertEquals(10, data.getTotalElements());
filter.setFilters(List.of(new RelationEntityTypeFilter("NOT_CONTAINS", List.of(EntityType.DEVICE), true))); filter.setFilters(List.of(new RelationEntityTypeFilter("NOT_CONTAINS", List.of(EntityType.DEVICE), true)));
query = new EntityDataQuery(filter, pageLink, entityFields, null, null); query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {}); findByQueryAndCheck(query, 87);
Assert.assertEquals(87, data.getTotalElements());
} }
private EntityRelation createFromRelation(Device mainDevice, Device device, String relationType) { private EntityRelation createFromRelation(Device mainDevice, Device device, String relationType) {
@ -531,14 +505,12 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
List<EntityKey> latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); List<EntityKey> latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature"));
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null);
PageData<EntityData> data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() { PageData<EntityData> data = findByQueryAndCheck(query, 67);
});
List<EntityData> loadedEntities = new ArrayList<>(data.getData()); List<EntityData> loadedEntities = new ArrayList<>(data.getData());
while (data.hasNext()) { while (data.hasNext()) {
query = query.next(); query = query.next();
data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() { data = findByQuery(query);
});
loadedEntities.addAll(data.getData()); loadedEntities.addAll(data.getData());
} }
Assert.assertEquals(67, loadedEntities.size()); Assert.assertEquals(67, loadedEntities.size());
@ -551,6 +523,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
pageLink = new EntityDataPageLink(10, 0, null, sortOrder); pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
KeyFilter highTemperatureFilter = new KeyFilter(); KeyFilter highTemperatureFilter = new KeyFilter();
highTemperatureFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); highTemperatureFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature"));
highTemperatureFilter.setValueType(EntityKeyValueType.NUMERIC);
NumericFilterPredicate predicate = new NumericFilterPredicate(); NumericFilterPredicate predicate = new NumericFilterPredicate();
predicate.setValue(FilterPredicateValue.fromDouble(45)); predicate.setValue(FilterPredicateValue.fromDouble(45));
predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER);
@ -559,13 +532,11 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters);
data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() { data = findByQuery(query);
});
loadedEntities = new ArrayList<>(data.getData()); loadedEntities = new ArrayList<>(data.getData());
while (data.hasNext()) { while (data.hasNext()) {
query = query.next(); query = query.next();
data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() { data = findByQuery(query);
});
loadedEntities.addAll(data.getData()); loadedEntities.addAll(data.getData());
} }
Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); Assert.assertEquals(highTemperatures.size(), loadedEntities.size());
@ -604,6 +575,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
KeyFilter highTemperatureFilter = new KeyFilter(); KeyFilter highTemperatureFilter = new KeyFilter();
highTemperatureFilter.setKey(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, "alarmActiveTime")); highTemperatureFilter.setKey(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, "alarmActiveTime"));
highTemperatureFilter.setValueType(EntityKeyValueType.NUMERIC);
NumericFilterPredicate predicate = new NumericFilterPredicate(); NumericFilterPredicate predicate = new NumericFilterPredicate();
DynamicValue<Double> dynamicValue = DynamicValue<Double> dynamicValue =
@ -627,16 +599,16 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters);
Awaitility.await() await()
.alias("data by query") .alias("data by query")
.atMost(TIMEOUT, TimeUnit.SECONDS) .atMost(TIMEOUT, TimeUnit.SECONDS)
.until(() -> { .until(() -> {
var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {}); var data = findByQuery(query);
var loadedEntities = new ArrayList<>(data.getData()); var loadedEntities = new ArrayList<>(data.getData());
return loadedEntities.size() == numOfDevices; return loadedEntities.size() == numOfDevices;
}); });
var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {}); var data = findByQuery(query);
var loadedEntities = new ArrayList<>(data.getData()); var loadedEntities = new ArrayList<>(data.getData());
Assert.assertEquals(numOfDevices, loadedEntities.size()); Assert.assertEquals(numOfDevices, loadedEntities.size());
@ -694,11 +666,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
EntityDataQuery query = new EntityDataQuery(entityTypeFilter, pageLink, entityFields, null, null); EntityDataQuery query = new EntityDataQuery(entityTypeFilter, pageLink, entityFields, null, null);
PageData<EntityData> data = PageData<EntityData> data = findByQueryAndCheck(query, 97);
doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
});
Assert.assertEquals(97, data.getTotalElements());
Assert.assertEquals(10, data.getTotalPages()); Assert.assertEquals(10, data.getTotalPages());
Assert.assertTrue(data.hasNext()); Assert.assertTrue(data.hasNext());
Assert.assertEquals(10, data.getData().size()); Assert.assertEquals(10, data.getData().size());
@ -712,9 +680,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
}); });
EntityCountQuery countQuery = new EntityCountQuery(entityTypeFilter); EntityCountQuery countQuery = new EntityCountQuery(entityTypeFilter);
countByQueryAndCheck(countQuery, 97);
Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
Assert.assertEquals(97, count.longValue());
} }
@Test @Test
@ -742,28 +708,29 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
KeyFilter activeAlarmTimeToLongFilter = getServerAttributeNumericGreaterThanKeyFilter("alarmActiveTime", 30); KeyFilter activeAlarmTimeToLongFilter = getServerAttributeNumericGreaterThanKeyFilter("alarmActiveTime", 30);
KeyFilter tenantOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", TEST_TENANT_NAME); KeyFilter tenantOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", TEST_TENANT_NAME);
KeyFilter wrongOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", "wrongName"); KeyFilter wrongOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", "wrongName");
KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT"); KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT");
KeyFilter customerOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "CUSTOMER"); KeyFilter customerOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "CUSTOMER");
// all devices with ownerName = TEST TENANT // all devices with ownerName = TEST TENANT
EntityCountQuery query = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerNameFilter)); EntityCountQuery query = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerNameFilter));
checkEntitiesCount(query, numOfDevices); await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(query),
result -> result == numOfDevices);
// all devices with ownerName = TEST TENANT // all devices with ownerName = TEST TENANT
EntityCountQuery activeAlarmTimeToLongQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeToLongFilter, tenantOwnerNameFilter)); EntityCountQuery activeAlarmTimeToLongQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeToLongFilter, tenantOwnerNameFilter));
checkEntitiesCount(activeAlarmTimeToLongQuery, 0); countByQueryAndCheck(activeAlarmTimeToLongQuery, 0);
// all devices with wrong ownerName // all devices with wrong ownerName
EntityCountQuery wrongTenantNameQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, wrongOwnerNameFilter)); EntityCountQuery wrongTenantNameQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, wrongOwnerNameFilter));
checkEntitiesCount(wrongTenantNameQuery, 0); countByQueryAndCheck(wrongTenantNameQuery, 0);
// all devices with owner type = TENANT // all devices with owner type = TENANT
EntityCountQuery tenantEntitiesQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerTypeFilter)); EntityCountQuery tenantEntitiesQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerTypeFilter));
checkEntitiesCount(tenantEntitiesQuery, numOfDevices); countByQueryAndCheck(tenantEntitiesQuery, numOfDevices);
// all devices with owner type = CUSTOMER // all devices with owner type = CUSTOMER
EntityCountQuery customerEntitiesQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, customerOwnerTypeFilter)); EntityCountQuery customerEntitiesQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, customerOwnerTypeFilter));
checkEntitiesCount(customerEntitiesQuery, 0); countByQueryAndCheck(customerEntitiesQuery, 0);
} }
@Test @Test
@ -790,7 +757,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
KeyFilter activeAlarmTimeFilter = getServerAttributeNumericGreaterThanKeyFilter("alarmActiveTime", 5); KeyFilter activeAlarmTimeFilter = getServerAttributeNumericGreaterThanKeyFilter("alarmActiveTime", 5);
KeyFilter tenantOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", TEST_TENANT_NAME); KeyFilter tenantOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", TEST_TENANT_NAME);
KeyFilter wrongOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", "wrongName"); KeyFilter wrongOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", "wrongName");
KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT"); KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT");
KeyFilter customerOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "CUSTOMER"); KeyFilter customerOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "CUSTOMER");
EntityDataSortOrder sortOrder = new EntityDataSortOrder( EntityDataSortOrder sortOrder = new EntityDataSortOrder(
@ -851,41 +818,29 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
PageData<EntityData> data = findByQueryAndCheck(query, 1);
doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
});
Assert.assertEquals(1, data.getTotalElements());
Assert.assertEquals(1, data.getTotalPages());
Assert.assertEquals(1, data.getData().size());
// unnassign dashboard // unnassign dashboard
login(TENANT_EMAIL, TENANT_PASSWORD); login(TENANT_EMAIL, TENANT_PASSWORD);
doDelete("/api/customer/" + savedCustomer.getId().getId().toString() + "/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class); doDelete("/api/customer/" + savedCustomer.getId().getId().toString() + "/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class);
login(CUSTOMER_USER_EMAIL, CUSTOMER_USER_PASSWORD); login(CUSTOMER_USER_EMAIL, CUSTOMER_USER_PASSWORD);
PageData<EntityData> dataAfterUnassign = findByQueryAndCheck(query, 0);
doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
});
Assert.assertEquals(0, dataAfterUnassign.getTotalElements());
Assert.assertEquals(0, dataAfterUnassign.getTotalPages());
Assert.assertEquals(0, dataAfterUnassign.getData().size());
} }
private void checkEntitiesByQuery(EntityDataQuery query, int expectedNumOfDevices, String expectedOwnerName, String expectedOwnerType) throws Exception { private void checkEntitiesByQuery(EntityDataQuery query, int expectedNumOfDevices, String expectedOwnerName, String expectedOwnerType) throws Exception {
Awaitility.await() await()
.alias("data by query") .alias("data by query")
.atMost(30, TimeUnit.SECONDS) .atMost(30, TimeUnit.SECONDS)
.until(() -> { .until(() -> {
var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {}); var data = findByQuery(query);
var loadedEntities = new ArrayList<>(data.getData()); var loadedEntities = new ArrayList<>(data.getData());
return loadedEntities.size() == expectedNumOfDevices; return loadedEntities.size() == expectedNumOfDevices;
}); });
if (expectedNumOfDevices == 0) { if (expectedNumOfDevices == 0) {
return; return;
} }
var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {}); var data = findByQuery(query);
var loadedEntities = new ArrayList<>(data.getData()); var loadedEntities = new ArrayList<>(data.getData());
Assert.assertEquals(expectedNumOfDevices, loadedEntities.size()); Assert.assertEquals(expectedNumOfDevices, loadedEntities.size());
@ -898,25 +853,37 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
String alarmActiveTime = entity.getLatest().get(EntityKeyType.ATTRIBUTE).getOrDefault("alarmActiveTime", new TsValue(0, "-1")).getValue(); String alarmActiveTime = entity.getLatest().get(EntityKeyType.ATTRIBUTE).getOrDefault("alarmActiveTime", new TsValue(0, "-1")).getValue();
Assert.assertEquals("Device" + i, name); Assert.assertEquals("Device" + i, name);
Assert.assertEquals( expectedOwnerName, ownerName); Assert.assertEquals(expectedOwnerName, ownerName);
Assert.assertEquals( expectedOwnerType, ownerType); Assert.assertEquals(expectedOwnerType, ownerType);
Assert.assertEquals("1" + i, alarmActiveTime); Assert.assertEquals("1" + i, alarmActiveTime);
} }
} }
private void checkEntitiesCount(EntityCountQuery query, int expectedNumOfDevices) { protected PageData<EntityData> findByQuery(EntityDataQuery query) throws Exception {
Awaitility.await() return doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {
.alias("count by query") });
.atMost(30, TimeUnit.SECONDS) }
.until(() -> {
var count = doPost("/api/entitiesQuery/count", query, Integer.class); protected PageData<EntityData> findByQueryAndCheck(EntityDataQuery query, int expectedResultSize) throws Exception {
return count == expectedNumOfDevices; PageData<EntityData> result = findByQuery(query);
}); assertThat(result.getTotalElements()).isEqualTo(expectedResultSize);
} return result;
}
protected Long countByQuery(EntityCountQuery countQuery) throws Exception {
return doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
}
protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult) throws Exception {
Long result = countByQuery(query);
assertThat(result).isEqualTo(expectedResult);
return result;
}
private KeyFilter getEntityFieldStringEqualToKeyFilter(String keyName, String value) { private KeyFilter getEntityFieldStringEqualToKeyFilter(String keyName, String value) {
KeyFilter tenantOwnerNameFilter = new KeyFilter(); KeyFilter tenantOwnerNameFilter = new KeyFilter();
tenantOwnerNameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, keyName)); tenantOwnerNameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, keyName));
tenantOwnerNameFilter.setValueType(EntityKeyValueType.STRING);
StringFilterPredicate ownerNamePredicate = new StringFilterPredicate(); StringFilterPredicate ownerNamePredicate = new StringFilterPredicate();
ownerNamePredicate.setValue(FilterPredicateValue.fromString(value)); ownerNamePredicate.setValue(FilterPredicateValue.fromString(value));
ownerNamePredicate.setOperation(StringFilterPredicate.StringOperation.EQUAL); ownerNamePredicate.setOperation(StringFilterPredicate.StringOperation.EQUAL);
@ -927,6 +894,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
private KeyFilter getServerAttributeNumericGreaterThanKeyFilter(String attribute, int value) { private KeyFilter getServerAttributeNumericGreaterThanKeyFilter(String attribute, int value) {
KeyFilter numericFilter = new KeyFilter(); KeyFilter numericFilter = new KeyFilter();
numericFilter.setKey(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, attribute)); numericFilter.setKey(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, attribute));
numericFilter.setValueType(EntityKeyValueType.NUMERIC);
NumericFilterPredicate predicate = new NumericFilterPredicate(); NumericFilterPredicate predicate = new NumericFilterPredicate();
predicate.setValue(FilterPredicateValue.fromDouble(value)); predicate.setValue(FilterPredicateValue.fromDouble(value));
predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER);

85
application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java

@ -83,7 +83,8 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
@Slf4j @Slf4j
@DaoSqlTest @DaoSqlTest
@TestPropertySource(properties = { @TestPropertySource(properties = {
"server.ws.alarms_per_alarm_status_subscription_cache_size=5" "server.ws.alarms_per_alarm_status_subscription_cache_size=5",
"server.ws.dynamic_page_link.refresh_interval=15"
}) })
public class WebsocketApiTest extends AbstractControllerTest { public class WebsocketApiTest extends AbstractControllerTest {
@Autowired @Autowired
@ -324,6 +325,83 @@ public class WebsocketApiTest extends AbstractControllerTest {
Assert.assertEquals(1, update.getCount()); Assert.assertEquals(1, update.getCount());
} }
@Test
public void testAlarmCountWsCmdWithSingleEntityFilter() throws Exception {
loginTenantAdmin();
SingleEntityFilter singleEntityFilter = new SingleEntityFilter();
singleEntityFilter.setSingleEntity(tenantId);
AlarmCountQuery alarmCountQuery = new AlarmCountQuery(singleEntityFilter);
AlarmCountCmd cmd1 = new AlarmCountCmd(1, alarmCountQuery);
getWsClient().send(cmd1);
AlarmCountUpdate update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply());
Assert.assertEquals(1, update.getCmdId());
Assert.assertEquals(0, update.getCount());
//create alarm, check count = 1
getWsClient().registerWaitForUpdate();
Alarm alarm = new Alarm();
alarm.setOriginator(tenantId);
alarm.setType("TEST ALARM");
alarm.setSeverity(AlarmSeverity.WARNING);
alarm = doPost("/api/alarm", alarm, Alarm.class);
update = getWsClient().parseAlarmCountReply(getWsClient().waitForUpdate());
Assert.assertEquals(1, update.getCmdId());
Assert.assertEquals(1, update.getCount());
// set wrong entity id in filter, check count = 0
singleEntityFilter.setSingleEntity(tenantAdminUserId);
AlarmCountCmd cmd3 = new AlarmCountCmd(2, alarmCountQuery);
getWsClient().send(cmd3);
update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply());
Assert.assertEquals(2, update.getCmdId());
Assert.assertEquals(0, update.getCount());
}
@Test
public void testAlarmCountWsCmdWithDeviceType() throws Exception {
loginTenantAdmin();
DeviceTypeFilter deviceTypeFilter = new DeviceTypeFilter();
deviceTypeFilter.setDeviceTypes(List.of("default"));
AlarmCountQuery alarmCountQuery = new AlarmCountQuery(deviceTypeFilter);
AlarmCountCmd cmd1 = new AlarmCountCmd(1, alarmCountQuery);
getWsClient().send(cmd1);
AlarmCountUpdate update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply());
Assert.assertEquals(1, update.getCmdId());
Assert.assertEquals(0, update.getCount());
getWsClient().registerWaitForUpdate();
Alarm alarm = new Alarm();
alarm.setOriginator(device.getId());
alarm.setType("TEST ALARM");
alarm.setSeverity(AlarmSeverity.WARNING);
alarm = doPost("/api/alarm", alarm, Alarm.class);
update = getWsClient().parseAlarmCountReply(getWsClient().waitForUpdate());
Assert.assertEquals(1, update.getCmdId());
Assert.assertEquals(1, update.getCount());
deviceTypeFilter.setDeviceTypes(List.of("non-existing"));
AlarmCountCmd cmd3 = new AlarmCountCmd(3, alarmCountQuery);
getWsClient().send(cmd3);
update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply());
Assert.assertEquals(3, update.getCmdId());
Assert.assertEquals(0, update.getCount());
}
@Test @Test
public void testAlarmStatusWsCmd() throws Exception { public void testAlarmStatusWsCmd() throws Exception {
loginTenantAdmin(); loginTenantAdmin();
@ -372,17 +450,18 @@ public class WebsocketApiTest extends AbstractControllerTest {
doPost("/api/alarm", alarm2, Alarm.class); doPost("/api/alarm", alarm2, Alarm.class);
AlarmStatusUpdate alarmStatusUpdate3 = JacksonUtil.fromString(getWsClient().waitForReply(), AlarmStatusUpdate.class); AlarmStatusUpdate alarmStatusUpdate3 = JacksonUtil.fromString(getWsClient().waitForUpdate(), AlarmStatusUpdate.class);
Assert.assertEquals(1, alarmStatusUpdate3.getCmdId()); Assert.assertEquals(1, alarmStatusUpdate3.getCmdId());
Assert.assertTrue(alarmStatusUpdate3.isActive()); Assert.assertTrue(alarmStatusUpdate3.isActive());
//change severity //change severity
getWsClient().registerWaitForUpdate();
alarm2.setSeverity(AlarmSeverity.MAJOR); alarm2.setSeverity(AlarmSeverity.MAJOR);
Alarm updatedAlarm = doPost("/api/alarm", alarm2, Alarm.class); Alarm updatedAlarm = doPost("/api/alarm", alarm2, Alarm.class);
Assert.assertNotNull(updatedAlarm); Assert.assertNotNull(updatedAlarm);
Assert.assertEquals(AlarmSeverity.MAJOR, updatedAlarm.getSeverity()); Assert.assertEquals(AlarmSeverity.MAJOR, updatedAlarm.getSeverity());
AlarmStatusUpdate alarmStatusUpdate4 = JacksonUtil.fromString(getWsClient().waitForReply(), AlarmStatusUpdate.class); AlarmStatusUpdate alarmStatusUpdate4 = JacksonUtil.fromString(getWsClient().waitForUpdate(), AlarmStatusUpdate.class);
Assert.assertEquals(1, alarmStatusUpdate4.getCmdId()); Assert.assertEquals(1, alarmStatusUpdate4.getCmdId());
Assert.assertFalse(alarmStatusUpdate4.isActive()); Assert.assertFalse(alarmStatusUpdate4.isActive());

49
application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java

@ -90,14 +90,12 @@ import org.thingsboard.server.gen.edge.v1.EdgeConfiguration;
import org.thingsboard.server.gen.edge.v1.OAuth2ClientUpdateMsg; import org.thingsboard.server.gen.edge.v1.OAuth2ClientUpdateMsg;
import org.thingsboard.server.gen.edge.v1.OAuth2DomainUpdateMsg; import org.thingsboard.server.gen.edge.v1.OAuth2DomainUpdateMsg;
import org.thingsboard.server.gen.edge.v1.QueueUpdateMsg; import org.thingsboard.server.gen.edge.v1.QueueUpdateMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataRequestMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg;
import org.thingsboard.server.gen.edge.v1.SyncCompletedMsg; import org.thingsboard.server.gen.edge.v1.SyncCompletedMsg;
import org.thingsboard.server.gen.edge.v1.TenantProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.TenantProfileUpdateMsg;
import org.thingsboard.server.gen.edge.v1.TenantUpdateMsg; import org.thingsboard.server.gen.edge.v1.TenantUpdateMsg;
import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.edge.v1.UpdateMsgType;
import org.thingsboard.server.gen.edge.v1.UplinkMsg;
import org.thingsboard.server.gen.edge.v1.UserCredentialsUpdateMsg; import org.thingsboard.server.gen.edge.v1.UserCredentialsUpdateMsg;
import org.thingsboard.server.gen.edge.v1.UserUpdateMsg; import org.thingsboard.server.gen.edge.v1.UserUpdateMsg;
@ -142,35 +140,14 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest {
installation(); installation();
edgeImitator = new EdgeImitator("localhost", 7070, edge.getRoutingKey(), edge.getSecret()); edgeImitator = new EdgeImitator("localhost", 7070, edge.getRoutingKey(), edge.getSecret());
edgeImitator.expectMessageAmount(25);
edgeImitator.ignoreType(OAuth2ClientUpdateMsg.class); edgeImitator.ignoreType(OAuth2ClientUpdateMsg.class);
edgeImitator.ignoreType(OAuth2DomainUpdateMsg.class); edgeImitator.ignoreType(OAuth2DomainUpdateMsg.class);
edgeImitator.expectMessageAmount(26);
edgeImitator.connect(); edgeImitator.connect();
requestEdgeRuleChainMetadata();
verifyEdgeConnectionAndInitialData(); verifyEdgeConnectionAndInitialData();
} }
private void requestEdgeRuleChainMetadata() throws Exception {
RuleChainId rootRuleChainId = getEdgeRootRuleChainId();
RuleChainMetadataRequestMsg.Builder builder = RuleChainMetadataRequestMsg.newBuilder()
.setRuleChainIdMSB(rootRuleChainId.getId().getMostSignificantBits())
.setRuleChainIdLSB(rootRuleChainId.getId().getLeastSignificantBits());
testAutoGeneratedCodeByProtobuf(builder);
UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder()
.addRuleChainMetadataRequestMsg(builder.build());
edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build());
}
private RuleChainId getEdgeRootRuleChainId() throws Exception {
return doGetTypedWithPageLink("/api/ruleChains?type={type}&", new TypeReference<PageData<RuleChain>>() {
},
new PageLink(100, 0, "Edge Root Rule Chain"),
"EDGE")
.getData().get(0).getId();
}
@After @After
public void teardownEdgeTest() { public void teardownEdgeTest() {
try { try {
@ -213,6 +190,19 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest {
doPost("/api/ruleChain/metadata", rootRuleChainMetadata, RuleChainMetaData.class); doPost("/api/ruleChain/metadata", rootRuleChainMetadata, RuleChainMetaData.class);
} }
private RuleChainId getEdgeRootRuleChainId() throws Exception {
List<RuleChain> edgeRuleChains = doGetTypedWithPageLink("/api/ruleChains?type={type}&",
new TypeReference<PageData<RuleChain>>() {},
new PageLink(100, 0, "Edge Root Rule Chain"),
"EDGE").getData();
for (RuleChain edgeRuleChain : edgeRuleChains) {
if (edgeRuleChain.isRoot()) {
return edgeRuleChain.getId();
}
}
throw new RuntimeException("Root rule chain not found");
}
protected void extendDeviceProfileData(DeviceProfile deviceProfile) { protected void extendDeviceProfileData(DeviceProfile deviceProfile) {
DeviceProfileData profileData = deviceProfile.getProfileData(); DeviceProfileData profileData = deviceProfile.getProfileData();
List<DeviceProfileAlarm> alarms = new ArrayList<>(); List<DeviceProfileAlarm> alarms = new ArrayList<>();
@ -255,8 +245,8 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest {
validateMsgsCnt(RuleChainUpdateMsg.class, 1); validateMsgsCnt(RuleChainUpdateMsg.class, 1);
UUID ruleChainUUID = validateRuleChains(); UUID ruleChainUUID = validateRuleChains();
// 1 from request message // 1 from rule chain fetcher
validateMsgsCnt(RuleChainMetadataUpdateMsg.class, 2); validateMsgsCnt(RuleChainMetadataUpdateMsg.class, 1);
validateRuleChainMetadataUpdates(ruleChainUUID); validateRuleChainMetadataUpdates(ruleChainUUID);
// 4 messages ('general', 'mail', 'connectivity', 'jwt') // 4 messages ('general', 'mail', 'connectivity', 'jwt')
@ -438,12 +428,11 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest {
} }
private void validateRuleChainMetadataUpdates(UUID expectedRuleChainUUID) { private void validateRuleChainMetadataUpdates(UUID expectedRuleChainUUID) {
Optional<RuleChainMetadataUpdateMsg> ruleChainMetadataUpdateOpt = edgeImitator.findMessageByType(RuleChainMetadataUpdateMsg.class); Optional<RuleChainMetadataUpdateMsg> ruleChainMetadataUpdateMsgOpt = edgeImitator.findMessageByType(RuleChainMetadataUpdateMsg.class);
Assert.assertTrue(ruleChainMetadataUpdateOpt.isPresent()); Assert.assertTrue(ruleChainMetadataUpdateMsgOpt.isPresent());
RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = ruleChainMetadataUpdateOpt.get(); RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = ruleChainMetadataUpdateMsgOpt.get();
Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, ruleChainMetadataUpdateMsg.getMsgType()); Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, ruleChainMetadataUpdateMsg.getMsgType());
RuleChainMetaData ruleChainMetaData = JacksonUtil.fromString(ruleChainMetadataUpdateMsg.getEntity(), RuleChainMetaData.class, true); RuleChainMetaData ruleChainMetaData = JacksonUtil.fromString(ruleChainMetadataUpdateMsg.getEntity(), RuleChainMetaData.class, true);
Assert.assertNotNull(ruleChainMetaData);
Assert.assertEquals(expectedRuleChainUUID, ruleChainMetaData.getRuleChainId().getId()); Assert.assertEquals(expectedRuleChainUUID, ruleChainMetaData.getRuleChainId().getId());
} }

27
application/src/test/java/org/thingsboard/server/edge/AssetEdgeTest.java

@ -169,6 +169,7 @@ public class AssetEdgeTest extends AbstractEdgeTest {
public void testSendAssetToCloud() throws Exception { public void testSendAssetToCloud() throws Exception {
Asset asset = buildAssetForUplinkMsg("Asset Edge 2"); Asset asset = buildAssetForUplinkMsg("Asset Edge 2");
// created asset on edge
UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder();
AssetUpdateMsg.Builder assetUpdateMsgBuilder = AssetUpdateMsg.newBuilder(); AssetUpdateMsg.Builder assetUpdateMsgBuilder = AssetUpdateMsg.newBuilder();
assetUpdateMsgBuilder.setIdMSB(asset.getUuidId().getMostSignificantBits()); assetUpdateMsgBuilder.setIdMSB(asset.getUuidId().getMostSignificantBits());
@ -191,6 +192,32 @@ public class AssetEdgeTest extends AbstractEdgeTest {
Asset foundAsset = doGet("/api/asset/" + asset.getUuidId(), Asset.class); Asset foundAsset = doGet("/api/asset/" + asset.getUuidId(), Asset.class);
Assert.assertNotNull(foundAsset); Assert.assertNotNull(foundAsset);
Assert.assertEquals("Asset Edge 2", foundAsset.getName()); Assert.assertEquals("Asset Edge 2", foundAsset.getName());
// update asset on edge
asset.setName("Asset Edge 2 Updated");
uplinkMsgBuilder = UplinkMsg.newBuilder();
assetUpdateMsgBuilder = AssetUpdateMsg.newBuilder();
assetUpdateMsgBuilder.setIdMSB(asset.getUuidId().getMostSignificantBits());
assetUpdateMsgBuilder.setIdLSB(asset.getUuidId().getLeastSignificantBits());
assetUpdateMsgBuilder.setEntity(JacksonUtil.toString(asset));
assetUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE);
testAutoGeneratedCodeByProtobuf(assetUpdateMsgBuilder);
uplinkMsgBuilder.addAssetUpdateMsg(assetUpdateMsgBuilder.build());
testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder);
edgeImitator.expectResponsesAmount(1);
edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build());
Assert.assertTrue(edgeImitator.waitForResponses());
latestResponseMsg = edgeImitator.getLatestResponseMsg();
Assert.assertTrue(latestResponseMsg.getSuccess());
foundAsset = doGet("/api/asset/" + asset.getUuidId(), Asset.class);
Assert.assertNotNull(foundAsset);
Assert.assertEquals("Asset Edge 2 Updated", foundAsset.getName());
} }
@Test @Test

1
application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java

@ -184,6 +184,7 @@ public class DashboardEdgeTest extends AbstractEdgeTest {
Dashboard dashboard = buildDashboardForUplinkMsg(savedCustomer); Dashboard dashboard = buildDashboardForUplinkMsg(savedCustomer);
// create dashboard on edge
UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder();
DashboardUpdateMsg.Builder dashboardUpdateMsgBuilder = DashboardUpdateMsg.newBuilder(); DashboardUpdateMsg.Builder dashboardUpdateMsgBuilder = DashboardUpdateMsg.newBuilder();
dashboardUpdateMsgBuilder.setIdMSB(dashboard.getUuidId().getMostSignificantBits()); dashboardUpdateMsgBuilder.setIdMSB(dashboard.getUuidId().getMostSignificantBits());

24
application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java

@ -593,8 +593,10 @@ public class DeviceEdgeTest extends AbstractEdgeTest {
@Test @Test
public void testSendDeviceToCloud() throws Exception { public void testSendDeviceToCloud() throws Exception {
Device deviceMsg = buildDeviceForUplinkMsg("Edge Device 2", "test"); String deviceName = "Edge Device 2";
Device deviceMsg = buildDeviceForUplinkMsg(deviceName, "test");
// create device on edge
UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder();
DeviceUpdateMsg.Builder deviceUpdateMsgBuilder = DeviceUpdateMsg.newBuilder(); DeviceUpdateMsg.Builder deviceUpdateMsgBuilder = DeviceUpdateMsg.newBuilder();
deviceUpdateMsgBuilder.setIdMSB(deviceMsg.getUuidId().getMostSignificantBits()); deviceUpdateMsgBuilder.setIdMSB(deviceMsg.getUuidId().getMostSignificantBits());
@ -609,7 +611,25 @@ public class DeviceEdgeTest extends AbstractEdgeTest {
Device device = doGet("/api/device/" + deviceMsg.getId().getId(), Device.class); Device device = doGet("/api/device/" + deviceMsg.getId().getId(), Device.class);
Assert.assertNotNull(device); Assert.assertNotNull(device);
Assert.assertEquals("Edge Device 2", device.getName()); Assert.assertEquals(deviceName, device.getName());
// update device on edge
deviceMsg.setName(deviceName + " Updated");
uplinkMsgBuilder = UplinkMsg.newBuilder();
deviceUpdateMsgBuilder = DeviceUpdateMsg.newBuilder();
deviceUpdateMsgBuilder.setIdMSB(deviceMsg.getUuidId().getMostSignificantBits());
deviceUpdateMsgBuilder.setIdLSB(deviceMsg.getUuidId().getLeastSignificantBits());
deviceUpdateMsgBuilder.setEntity(JacksonUtil.toString(deviceMsg));
deviceUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE);
uplinkMsgBuilder.addDeviceUpdateMsg(deviceUpdateMsgBuilder.build());
edgeImitator.expectResponsesAmount(1);
edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build());
Assert.assertTrue(edgeImitator.waitForResponses());
device = doGet("/api/device/" + deviceMsg.getId().getId(), Device.class);
Assert.assertNotNull(device);
Assert.assertEquals(deviceName + " Updated", device.getName());
} }
@Test @Test

84
application/src/test/java/org/thingsboard/server/edge/RuleChainEdgeTest.java

@ -15,7 +15,7 @@
*/ */
package org.thingsboard.server.edge; package org.thingsboard.server.edge;
import com.google.protobuf.AbstractMessage; import com.datastax.oss.driver.api.core.uuid.Uuids;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
@ -29,16 +29,17 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataRequestMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg;
import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.edge.v1.UpdateMsgType;
import org.thingsboard.server.gen.edge.v1.UplinkMsg; import org.thingsboard.server.gen.edge.v1.UplinkMsg;
import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.UUID;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@ -74,8 +75,9 @@ public class RuleChainEdgeTest extends AbstractEdgeTest {
RuleChainMetaData ruleChainMetaData = JacksonUtil.fromString(ruleChainMetadataUpdateMsg.getEntity(), RuleChainMetaData.class, true); RuleChainMetaData ruleChainMetaData = JacksonUtil.fromString(ruleChainMetadataUpdateMsg.getEntity(), RuleChainMetaData.class, true);
Assert.assertNotNull(ruleChainMetaData); Assert.assertNotNull(ruleChainMetaData);
Assert.assertEquals(ruleChainMetaData.getRuleChainId(), savedRuleChain.getId()); Assert.assertEquals(ruleChainMetaData.getRuleChainId(), savedRuleChain.getId());
for (RuleNode ruleNode : ruleChainMetaData.getNodes()) {
testRuleChainMetadataRequestMsg(savedRuleChain.getId()); Assert.assertEquals(CONFIGURATION_VERSION, ruleNode.getConfigurationVersion());
}
// unassign rule chain from edge // unassign rule chain from edge
edgeImitator.expectMessageAmount(1); edgeImitator.expectMessageAmount(1);
@ -97,60 +99,62 @@ public class RuleChainEdgeTest extends AbstractEdgeTest {
} }
@Test @Test
public void testSendRuleChainMetadataRequestToCloud() throws Exception { public void testRuleChainToCloud() throws Exception {
RuleChainId edgeRootRuleChainId = edge.getRootRuleChainId(); String ruleChainName = "Rule Chain Edge";
UUID uuid = Uuids.timeBased();
// create rule chain on edge
RuleChain edgeRuleChain = new RuleChain();
edgeRuleChain.setTenantId(tenantId);
edgeRuleChain.setId(new RuleChainId(uuid));
edgeRuleChain.setName(ruleChainName);
UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder();
RuleChainMetadataRequestMsg.Builder ruleChainMetadataRequestMsgBuilder = RuleChainMetadataRequestMsg.newBuilder(); RuleChainUpdateMsg.Builder ruleChainUpdateMsgBuilder = RuleChainUpdateMsg.newBuilder();
ruleChainMetadataRequestMsgBuilder.setRuleChainIdMSB(edgeRootRuleChainId.getId().getMostSignificantBits()); ruleChainUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits());
ruleChainMetadataRequestMsgBuilder.setRuleChainIdLSB(edgeRootRuleChainId.getId().getLeastSignificantBits()); ruleChainUpdateMsgBuilder.setIdLSB(uuid.getLeastSignificantBits());
testAutoGeneratedCodeByProtobuf(ruleChainMetadataRequestMsgBuilder); ruleChainUpdateMsgBuilder.setEntity(JacksonUtil.toString(edgeRuleChain));
uplinkMsgBuilder.addRuleChainMetadataRequestMsg(ruleChainMetadataRequestMsgBuilder.build()); ruleChainUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE);
testAutoGeneratedCodeByProtobuf(ruleChainUpdateMsgBuilder);
uplinkMsgBuilder.addRuleChainUpdateMsg(ruleChainUpdateMsgBuilder.build());
testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder); testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder);
edgeImitator.expectResponsesAmount(1); edgeImitator.expectResponsesAmount(1);
edgeImitator.expectMessageAmount(1);
edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build());
Assert.assertTrue(edgeImitator.waitForResponses()); Assert.assertTrue(edgeImitator.waitForResponses());
Assert.assertTrue(edgeImitator.waitForMessages());
AbstractMessage latestMessage = edgeImitator.getLatestMessage(); UplinkResponseMsg latestResponseMsg = edgeImitator.getLatestResponseMsg();
Assert.assertTrue(latestMessage instanceof RuleChainMetadataUpdateMsg); Assert.assertTrue(latestResponseMsg.getSuccess());
RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = (RuleChainMetadataUpdateMsg) latestMessage;
RuleChainMetaData ruleChainMetadataMsg = JacksonUtil.fromString(ruleChainMetadataUpdateMsg.getEntity(), RuleChainMetaData.class, true);
Assert.assertNotNull(ruleChainMetadataMsg);
Assert.assertEquals(edgeRootRuleChainId, ruleChainMetadataMsg.getRuleChainId());
testAutoGeneratedCodeByProtobuf(ruleChainMetadataUpdateMsg); RuleChain ruleChain = doGet("/api/ruleChain/" + uuid, RuleChain.class);
} Assert.assertNotNull(ruleChain);
Assert.assertEquals("Rule Chain Edge", ruleChain.getName());
private void testRuleChainMetadataRequestMsg(RuleChainId ruleChainId) throws Exception { // update rule chain on edge
RuleChainMetadataRequestMsg.Builder ruleChainMetadataRequestMsgBuilder = RuleChainMetadataRequestMsg.newBuilder() edgeRuleChain.setName(ruleChainName + " Updated");
.setRuleChainIdMSB(ruleChainId.getId().getMostSignificantBits()) uplinkMsgBuilder = UplinkMsg.newBuilder();
.setRuleChainIdLSB(ruleChainId.getId().getLeastSignificantBits()); ruleChainUpdateMsgBuilder = RuleChainUpdateMsg.newBuilder();
testAutoGeneratedCodeByProtobuf(ruleChainMetadataRequestMsgBuilder); ruleChainUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits());
ruleChainUpdateMsgBuilder.setIdLSB(uuid.getLeastSignificantBits());
ruleChainUpdateMsgBuilder.setEntity(JacksonUtil.toString(edgeRuleChain));
ruleChainUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE);
testAutoGeneratedCodeByProtobuf(ruleChainUpdateMsgBuilder);
uplinkMsgBuilder.addRuleChainUpdateMsg(ruleChainUpdateMsgBuilder.build());
UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder()
.addRuleChainMetadataRequestMsg(ruleChainMetadataRequestMsgBuilder.build());
testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder); testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder);
edgeImitator.expectResponsesAmount(1); edgeImitator.expectResponsesAmount(1);
edgeImitator.expectMessageAmount(1);
edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build());
Assert.assertTrue(edgeImitator.waitForResponses()); Assert.assertTrue(edgeImitator.waitForResponses());
Assert.assertTrue(edgeImitator.waitForMessages());
AbstractMessage latestMessage = edgeImitator.getLatestMessage(); latestResponseMsg = edgeImitator.getLatestResponseMsg();
Assert.assertTrue(latestMessage instanceof RuleChainMetadataUpdateMsg); Assert.assertTrue(latestResponseMsg.getSuccess());
RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = (RuleChainMetadataUpdateMsg) latestMessage;
RuleChainMetaData ruleChainMetadataMsg = JacksonUtil.fromString(ruleChainMetadataUpdateMsg.getEntity(), RuleChainMetaData.class, true);
Assert.assertNotNull(ruleChainMetadataMsg);
Assert.assertEquals(ruleChainId, ruleChainMetadataMsg.getRuleChainId());
for (RuleNode ruleNode : ruleChainMetadataMsg.getNodes()) { ruleChain = doGet("/api/ruleChain/" + uuid, RuleChain.class);
Assert.assertEquals(CONFIGURATION_VERSION, ruleNode.getConfigurationVersion()); Assert.assertNotNull(ruleChain);
} Assert.assertEquals(ruleChainName + " Updated", ruleChain.getName());
} }
private RuleChainMetaData createRuleChainMetadata(RuleChain ruleChain) { private RuleChainMetaData createRuleChainMetadata(RuleChain ruleChain) {

17
application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java

@ -153,7 +153,7 @@ public class HashPartitionServiceTest {
for (int queueIndex = 0; queueIndex < queueCount; queueIndex++) { for (int queueIndex = 0; queueIndex < queueCount; queueIndex++) {
QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, "queue" + queueIndex, tenantId); QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, "queue" + queueIndex, tenantId);
for (int partition = 0; partition < partitionCount; partition++) { for (int partition = 0; partition < partitionCount; partition++) {
ServiceInfo serviceInfo = partitionService.resolveByPartitionIdx(services, queueKey, partition, Collections.emptyMap()); ServiceInfo serviceInfo = partitionService.resolveByPartitionIdx(services, queueKey, partition, Collections.emptyMap()).get(0);
String serviceId = serviceInfo.getServiceId(); String serviceId = serviceInfo.getServiceId();
map.put(serviceId, map.get(serviceId) + 1); map.put(serviceId, map.get(serviceId) + 1);
} }
@ -308,7 +308,7 @@ public class HashPartitionServiceTest {
partitionService_common.recalculatePartitions(commonRuleEngine, List.of(dedicatedRuleEngine)); partitionService_common.recalculatePartitions(commonRuleEngine, List.of(dedicatedRuleEngine));
verifyPartitionChangeEvent(event -> { verifyPartitionChangeEvent(event -> {
QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, TenantId.SYS_TENANT_ID); QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, TenantId.SYS_TENANT_ID);
return event.getPartitionsMap().get(queueKey).size() == systemQueue.getPartitions(); return event.getNewPartitions().get(queueKey).size() == systemQueue.getPartitions();
}); });
Mockito.reset(applicationEventPublisher); Mockito.reset(applicationEventPublisher);
@ -336,14 +336,14 @@ public class HashPartitionServiceTest {
// expecting event about no partitions for isolated queue key // expecting event about no partitions for isolated queue key
verifyPartitionChangeEvent(event -> { verifyPartitionChangeEvent(event -> {
QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId); QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId);
return event.getPartitionsMap().get(queueKey).isEmpty(); return event.getNewPartitions().get(queueKey).isEmpty();
}); });
partitionService_dedicated.updateQueues(List.of(queueUpdateMsg)); partitionService_dedicated.updateQueues(List.of(queueUpdateMsg));
partitionService_dedicated.recalculatePartitions(dedicatedRuleEngine, List.of(commonRuleEngine)); partitionService_dedicated.recalculatePartitions(dedicatedRuleEngine, List.of(commonRuleEngine));
verifyPartitionChangeEvent(event -> { verifyPartitionChangeEvent(event -> {
QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId); QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId);
return event.getPartitionsMap().get(queueKey).size() == isolatedQueue.getPartitions(); return event.getNewPartitions().get(queueKey).size() == isolatedQueue.getPartitions();
}); });
@ -361,7 +361,7 @@ public class HashPartitionServiceTest {
partitionService_dedicated.removeQueues(List.of(queueDeleteMsg)); partitionService_dedicated.removeQueues(List.of(queueDeleteMsg));
verifyPartitionChangeEvent(event -> { verifyPartitionChangeEvent(event -> {
QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId); QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId);
return event.getPartitionsMap().get(queueKey).isEmpty(); return event.getNewPartitions().get(queueKey).isEmpty();
}); });
} }
@ -381,12 +381,12 @@ public class HashPartitionServiceTest {
Stream.concat(Stream.of(TenantId.SYS_TENANT_ID), Stream.generate(UUID::randomUUID).map(TenantId::new).limit(10)).forEach(tenantId -> { Stream.concat(Stream.of(TenantId.SYS_TENANT_ID), Stream.generate(UUID::randomUUID).map(TenantId::new).limit(10)).forEach(tenantId -> {
List<QueueKey> queues = Stream.generate(() -> RandomStringUtils.randomAlphabetic(10)) List<QueueKey> queues = Stream.generate(() -> RandomStringUtils.randomAlphabetic(10))
.map(queueName -> new QueueKey(ServiceType.TB_RULE_ENGINE, queueName, tenantId)) .map(queueName -> new QueueKey(ServiceType.TB_RULE_ENGINE, queueName, tenantId))
.limit(100).collect(Collectors.toList()); .limit(100).toList();
for (int partition = 0; partition < 10; partition++) { for (int partition = 0; partition < 10; partition++) {
ServiceInfo expectedAssignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, new QueueKey(ServiceType.TB_RULE_ENGINE, tenantId), partition, Collections.emptyMap()); ServiceInfo expectedAssignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, new QueueKey(ServiceType.TB_RULE_ENGINE, tenantId), partition, Collections.emptyMap()).get(0);
for (QueueKey queueKey : queues) { for (QueueKey queueKey : queues) {
ServiceInfo assignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, queueKey, partition, Collections.emptyMap()); ServiceInfo assignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, queueKey, partition, Collections.emptyMap()).get(0);
assertThat(assignedRuleEngine).as(queueKey + "[" + partition + "] should be assigned to " + expectedAssignedRuleEngine.getServiceId()) assertThat(assignedRuleEngine).as(queueKey + "[" + partition + "] should be assigned to " + expectedAssignedRuleEngine.getServiceId())
.isEqualTo(expectedAssignedRuleEngine); .isEqualTo(expectedAssignedRuleEngine);
} }
@ -434,6 +434,7 @@ public class HashPartitionServiceTest {
ReflectionTestUtils.setField(partitionService, "hashFunctionName", hashFunctionName); ReflectionTestUtils.setField(partitionService, "hashFunctionName", hashFunctionName);
ReflectionTestUtils.setField(partitionService, "edgeTopic", "tb.edge"); ReflectionTestUtils.setField(partitionService, "edgeTopic", "tb.edge");
ReflectionTestUtils.setField(partitionService, "edgePartitions", 10); ReflectionTestUtils.setField(partitionService, "edgePartitions", 10);
ReflectionTestUtils.setField(partitionService, "edqsPartitions", 12);
partitionService.init(); partitionService.init();
partitionService.partitionsInit(); partitionService.partitionsInit();
return partitionService; return partitionService;

4
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java

@ -90,7 +90,7 @@ public class ScriptCalculatedFieldStateTest {
state.arguments = new HashMap<>(Map.of("assetHumidity", assetHumidityArgEntry)); state.arguments = new HashMap<>(Map.of("assetHumidity", assetHumidityArgEntry));
Map<String, ArgumentEntry> newArgs = Map.of("deviceTemperature", deviceTemperatureArgEntry); Map<String, ArgumentEntry> newArgs = Map.of("deviceTemperature", deviceTemperatureArgEntry);
boolean stateUpdated = state.updateState(newArgs); boolean stateUpdated = state.updateState(ctx, newArgs);
assertThat(stateUpdated).isTrue(); assertThat(stateUpdated).isTrue();
assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf(
@ -107,7 +107,7 @@ public class ScriptCalculatedFieldStateTest {
SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(ts, new LongDataEntry("assetHumidity", 41L), 349L); SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(ts, new LongDataEntry("assetHumidity", 41L), 349L);
Map<String, ArgumentEntry> newArgs = Map.of("assetHumidity", newArgEntry); Map<String, ArgumentEntry> newArgs = Map.of("assetHumidity", newArgEntry);
boolean stateUpdated = state.updateState(newArgs); boolean stateUpdated = state.updateState(ctx, newArgs);
assertThat(stateUpdated).isTrue(); assertThat(stateUpdated).isTrue();
assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf(

30
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java

@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField
import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.DoubleDataEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry;
import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.dao.usagerecord.ApiLimitService;
@ -87,7 +88,7 @@ public class SimpleCalculatedFieldStateTest {
)); ));
Map<String, ArgumentEntry> newArgs = Map.of("key3", key3ArgEntry); Map<String, ArgumentEntry> newArgs = Map.of("key3", key3ArgEntry);
boolean stateUpdated = state.updateState(newArgs); boolean stateUpdated = state.updateState(ctx, newArgs);
assertThat(stateUpdated).isTrue(); assertThat(stateUpdated).isTrue();
assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf(
@ -105,7 +106,7 @@ public class SimpleCalculatedFieldStateTest {
SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis(), new LongDataEntry("key1", 18L), 190L); SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis(), new LongDataEntry("key1", 18L), 190L);
Map<String, ArgumentEntry> newArgs = Map.of("key1", newArgEntry); Map<String, ArgumentEntry> newArgs = Map.of("key1", newArgEntry);
boolean stateUpdated = state.updateState(newArgs); boolean stateUpdated = state.updateState(ctx, newArgs);
assertThat(stateUpdated).isTrue(); assertThat(stateUpdated).isTrue();
assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf(Map.of("key1", newArgEntry)); assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf(Map.of("key1", newArgEntry));
@ -119,7 +120,7 @@ public class SimpleCalculatedFieldStateTest {
)); ));
Map<String, ArgumentEntry> newArgs = Map.of("key3", new TsRollingArgumentEntry(10, 30000L)); Map<String, ArgumentEntry> newArgs = Map.of("key3", new TsRollingArgumentEntry(10, 30000L));
assertThatThrownBy(() -> state.updateState(newArgs)) assertThatThrownBy(() -> state.updateState(ctx, newArgs))
.isInstanceOf(IllegalArgumentException.class) .isInstanceOf(IllegalArgumentException.class)
.hasMessage("Rolling argument entry is not supported for simple calculated fields."); .hasMessage("Rolling argument entry is not supported for simple calculated fields.");
} }
@ -138,7 +139,7 @@ public class SimpleCalculatedFieldStateTest {
Output output = getCalculatedFieldConfig().getOutput(); Output output = getCalculatedFieldConfig().getOutput();
assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getType()).isEqualTo(output.getType());
assertThat(result.getScope()).isEqualTo(output.getScope()); assertThat(result.getScope()).isEqualTo(output.getScope());
assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49.0))); assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49)));
} }
@Test @Test
@ -154,6 +155,26 @@ public class SimpleCalculatedFieldStateTest {
.hasMessage("Argument 'key2' is not a number."); .hasMessage("Argument 'key2' is not a number.");
} }
@Test
void testPerformCalculationWhenDecimalsByDefault() throws ExecutionException, InterruptedException {
state.arguments = new HashMap<>(Map.of(
"key1", new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new DoubleDataEntry("key1", 11.3456), 145L),
"key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 6, new DoubleDataEntry("key2", 15.1), 165L),
"key3", new SingleValueArgumentEntry(System.currentTimeMillis() - 3, new DoubleDataEntry("key3", 23.1), 184L)
));
Output output = getCalculatedFieldConfig().getOutput();
output.setDecimalsByDefault(3);
ctx.setOutput(output);
CalculatedFieldResult result = state.performCalculation(ctx).get();
assertThat(result).isNotNull();
assertThat(result.getType()).isEqualTo(output.getType());
assertThat(result.getScope()).isEqualTo(output.getScope());
assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49.546)));
}
@Test @Test
void testIsReadyWhenNotAllArgPresent() { void testIsReadyWhenNotAllArgPresent() {
assertThat(state.isReady()).isFalse(); assertThat(state.isReady()).isFalse();
@ -219,6 +240,7 @@ public class SimpleCalculatedFieldStateTest {
output.setName("output"); output.setName("output");
output.setType(OutputType.ATTRIBUTES); output.setType(OutputType.ATTRIBUTES);
output.setScope(AttributeScope.SERVER_SCOPE); output.setScope(AttributeScope.SERVER_SCOPE);
output.setDecimalsByDefault(0);
config.setOutput(output); config.setOutput(output);

2
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java

@ -71,6 +71,6 @@ public class SingleValueArgumentEntryTest {
@Test @Test
void testUpdateEntryWhenValueWasNotChanged() { void testUpdateEntryWhenValueWasNotChanged() {
assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 11L), 237L))).isFalse(); assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 11L), 364L))).isTrue();
} }
} }

128
application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java

@ -0,0 +1,128 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.entitiy;
import com.google.common.collect.Lists;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.TestPropertySource;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.IdBased;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.query.EntityCountQuery;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.RelationsQueryFilter;
import org.thingsboard.server.common.data.relation.EntitySearchDirection;
import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter;
import org.thingsboard.server.common.msg.edqs.EdqsApiService;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.edqs.util.EdqsRocksDb;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static org.awaitility.Awaitility.await;
@DaoSqlTest
@TestPropertySource(properties = {
"queue.edqs.sync.enabled=true",
"queue.edqs.api.supported=true",
"queue.edqs.api.auto_enable=true",
"queue.edqs.mode=local"
})
public class EdqsEntityServiceTest extends EntityServiceTest {
@Autowired
private EdqsApiService edqsApiService;
@MockBean
private EdqsRocksDb edqsRocksDb;
@Before
public void beforeEach() {
await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsApiService.isEnabled());
}
// sql implementation has a bug with data duplication, edqs implementation returns correct value
@Override
@Test
public void testCountHierarchicalEntitiesByMultiRootQuery() throws InterruptedException {
List<Asset> buildings = new ArrayList<>();
List<Asset> apartments = new ArrayList<>();
Map<String, Map<UUID, String>> entityNameByTypeMap = new HashMap<>();
Map<UUID, UUID> childParentRelationMap = new HashMap<>();
createMultiRootHierarchy(buildings, apartments, entityNameByTypeMap, childParentRelationMap);
RelationsQueryFilter filter = new RelationsQueryFilter();
filter.setMultiRoot(true);
filter.setMultiRootEntitiesType(EntityType.ASSET);
filter.setMultiRootEntityIds(buildings.stream().map(IdBased::getId).map(d -> d.getId().toString()).collect(Collectors.toSet()));
filter.setDirection(EntitySearchDirection.FROM);
EntityCountQuery countQuery = new EntityCountQuery(filter);
countByQueryAndCheck(countQuery, 63);
filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("AptToHeat", Collections.singletonList(EntityType.DEVICE))));
countByQueryAndCheck(countQuery, 27);
filter.setMultiRootEntitiesType(EntityType.ASSET);
filter.setMultiRootEntityIds(apartments.stream().map(IdBased::getId).map(d -> d.getId().toString()).collect(Collectors.toSet()));
filter.setDirection(EntitySearchDirection.TO);
filter.setFilters(Lists.newArrayList(
new RelationEntityTypeFilter("buildingToApt", Collections.singletonList(EntityType.ASSET)),
new RelationEntityTypeFilter("AptToEnergy", Collections.singletonList(EntityType.DEVICE))));
countByQueryAndCheck(countQuery, 3);
deviceService.deleteDevicesByTenantId(tenantId);
assetService.deleteAssetsByTenantId(tenantId);
}
@Override
protected PageData<EntityData> findByQueryAndCheck(CustomerId customerId, EntityDataQuery query, long expectedResultSize) {
return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findByQuery(customerId, query),
result -> result.getTotalElements() == expectedResultSize);
}
@Override
protected List<EntityData> findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List<String> expectedTelemetries) {
return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetries),
loadedEntities -> loadedEntities.stream().map(entityData -> entityData.getLatest().get(entityKeyType).get(key).getValue()).toList().containsAll(expectedTelemetries));
}
@Override
protected long countByQueryAndCheck(EntityCountQuery countQuery, int expectedResult) {
return countByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), countQuery, expectedResult);
}
@Override
protected long countByQueryAndCheck(CustomerId customerId, EntityCountQuery query, int expectedResult) {
return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(customerId, query),
result -> result == expectedResult);
}
}

799
dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java → application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java

File diff suppressed because it is too large

93
application/src/test/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerServiceTest.java

@ -532,6 +532,98 @@ public class DefaultTbCoreConsumerServiceTest {
then(statsMock).should(never()).log(inactivityMsg); then(statsMock).should(never()).log(inactivityMsg);
} }
@Test
public void givenProcessingSuccess_whenForwardingInactivityTimeoutUpdateMsgToStateService_thenOnSuccessCallbackIsCalled() {
// GIVEN
var inactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setInactivityTimeout(time)
.build();
doCallRealMethod().when(defaultTbCoreConsumerServiceMock).forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock);
// WHEN
defaultTbCoreConsumerServiceMock.forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock);
// THEN
then(stateServiceMock).should().onDeviceInactivityTimeoutUpdate(tenantId, deviceId, time);
then(tbCallbackMock).should().onSuccess();
then(tbCallbackMock).should(never()).onFailure(any());
}
@Test
public void givenProcessingFailure_whenForwardingInactivityTimeoutUpdateMsgToStateService_thenOnFailureCallbackIsCalled() {
// GIVEN
var inactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setInactivityTimeout(time)
.build();
doCallRealMethod().when(defaultTbCoreConsumerServiceMock).forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock);
var runtimeException = new RuntimeException("Something bad happened!");
doThrow(runtimeException).when(stateServiceMock).onDeviceInactivityTimeoutUpdate(tenantId, deviceId, time);
// WHEN
defaultTbCoreConsumerServiceMock.forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock);
// THEN
then(tbCallbackMock).should(never()).onSuccess();
then(tbCallbackMock).should().onFailure(runtimeException);
}
@Test
public void givenStatsEnabled_whenForwardingInactivityTimeoutUpdateMsgToStateService_thenStatsAreRecorded() {
// GIVEN
ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "stats", statsMock);
ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "statsEnabled", true);
var inactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setInactivityTimeout(time)
.build();
doCallRealMethod().when(defaultTbCoreConsumerServiceMock).forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock);
// WHEN
defaultTbCoreConsumerServiceMock.forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock);
// THEN
then(statsMock).should().log(inactivityTimeoutUpdateMsg);
}
@Test
public void givenStatsDisabled_whenForwardingInactivityTimeoutUpdateMsgToStateService_thenStatsAreNotRecorded() {
// GIVEN
ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "stats", statsMock);
ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "statsEnabled", false);
var inactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setInactivityTimeout(time)
.build();
doCallRealMethod().when(defaultTbCoreConsumerServiceMock).forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock);
// WHEN
defaultTbCoreConsumerServiceMock.forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock);
// THEN
then(statsMock).should(never()).log(inactivityTimeoutUpdateMsg);
}
@Test @Test
public void givenRestApiCallResponseMsgProto_whenForwardToRuleEngineCallService_thenCallOnQueueMsg() { public void givenRestApiCallResponseMsgProto_whenForwardToRuleEngineCallService_thenCallOnQueueMsg() {
// GIVEN // GIVEN
@ -545,4 +637,5 @@ public class DefaultTbCoreConsumerServiceTest {
// THEN // THEN
then(ruleEngineCallServiceMock).should().onQueueMsg(restApiCallResponseMsgProto, tbCallbackMock); then(ruleEngineCallServiceMock).should().onQueueMsg(restApiCallResponseMsgProto, tbCallbackMock);
} }
} }

11
application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java

@ -626,8 +626,7 @@ public class TbRuleEngineQueueConsumerManagerTest {
.until(() -> consumer.subscribed && consumer.getPartitions().equals(expectedPartitions) && consumer.pollingStarted); .until(() -> consumer.subscribed && consumer.getPartitions().equals(expectedPartitions) && consumer.pollingStarted);
verify(consumer, times(1)).subscribe(any()); verify(consumer, times(1)).subscribe(any());
verify(consumer).subscribe(eq(expectedPartitions)); verify(consumer).subscribe(eq(expectedPartitions));
verify(consumer).doSubscribe(argThat(topics -> topics.containsAll(expectedPartitions.stream() verify(consumer).doSubscribe(argThat(topics -> topics.containsAll(expectedPartitions)));
.map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()))));
verify(consumer, atLeastOnce()).poll(eq((long) queue.getPollInterval())); verify(consumer, atLeastOnce()).poll(eq((long) queue.getPollInterval()));
verify(consumer, atLeastOnce()).doPoll(eq((long) queue.getPollInterval())); verify(consumer, atLeastOnce()).doPoll(eq((long) queue.getPollInterval()));
verify(consumer, never()).unsubscribe(); verify(consumer, never()).unsubscribe();
@ -743,9 +742,11 @@ public class TbRuleEngineQueueConsumerManagerTest {
} }
@Override @Override
protected void doSubscribe(List<String> topicNames) { protected void doSubscribe(Set<TopicPartitionInfo> partitions) {
log.debug("doSubscribe({})", topicNames); this.topics = partitions.stream()
this.topics = topicNames; .map(TopicPartitionInfo::getFullTopicName)
.collect(Collectors.toList());
log.debug("doSubscribe({})", topics);
subscribed = true; subscribed = true;
} }

202
application/src/test/java/org/thingsboard/server/service/script/TbelInvokeDocsIoTest.java

@ -21,6 +21,7 @@ import org.thingsboard.script.api.tbel.TbDate;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64; import java.util.Base64;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
@ -146,6 +147,31 @@ class TbelInvokeDocsIoTest extends AbstractTbelInvokeTest {
assertEquals(expected, actual); assertEquals(expected, actual);
} }
@Test
public void mapsImplicitIterationWithoutEntrySet() throws ExecutionException, InterruptedException {
msgStr = msgMapStr;
decoderStr = """
foreach(element : msg){
if(element.getKey() == null){
return raiseError("Bad getKey");
}
if(element.key == null){
return raiseError("Bad key");
}
if(element.getValue() == null){
return raiseError("Bad getValue");
}
if(element.value == null){
return raiseError("Bad value");
}
}
return msg;
""";
LinkedHashMap<String, Object> expected = expectedMap;
Object actual = invokeScript(evalScript(decoderStr), msgStr);
assertEquals(expected, actual);
}
@Test @Test
public void mapsGetInfoSize_Test() throws ExecutionException, InterruptedException { public void mapsGetInfoSize_Test() throws ExecutionException, InterruptedException {
msgStr = msgMapStr; msgStr = msgMapStr;
@ -660,14 +686,13 @@ class TbelInvokeDocsIoTest extends AbstractTbelInvokeTest {
"""; """;
decoderStr = """ decoderStr = """
var list = msg.list; var list = msg.list;
var listAdd = ["thigsboard", 4, 67];
return { return {
list: list.clone(), list: list.clone(),
length: list.length(), length: list.length(),
memorySize: list.memorySize(), memorySize: list.memorySize(),
indOf1: list.indexOf("B", 1), indOf1: list.indexOf("B", 1),
indOf2: list.indexOf(2, 2), indOf2: list.indexOf(2, 2),
sStr: list.validateClazzInArrayIsOnlyString() sStr: list.validateClazzInArrayIsOnlyNumber()
} }
"""; """;
ArrayList list = new ArrayList<>(List.of(67, 2, 2, 2)); ArrayList list = new ArrayList<>(List.of(67, 2, 2, 2));
@ -677,7 +702,7 @@ class TbelInvokeDocsIoTest extends AbstractTbelInvokeTest {
expected.put("memorySize", 32L); expected.put("memorySize", 32L);
expected.put("indOf1", -1); expected.put("indOf1", -1);
expected.put("indOf2", 2); expected.put("indOf2", 2);
expected.put("sStr", false); expected.put("sStr", true);
Object actual = invokeScript(evalScript(decoderStr), msgStr); Object actual = invokeScript(evalScript(decoderStr), msgStr);
assertEquals(expected, actual); assertEquals(expected, actual);
} }
@ -2148,6 +2173,177 @@ class TbelInvokeDocsIoTest extends AbstractTbelInvokeTest {
assertEquals(expected, actual); assertEquals(expected, actual);
} }
@Test
public void toUnmodifiableExecutionArrayList_Test() throws ExecutionException, InterruptedException {
msgStr = "{}";
decoderStr = String.format("""
var original = [];
original.add(0x35);
var unmodifiable = original.toUnmodifiable();
msg.result = unmodifiable;
return {msg: msg};
""");
LinkedHashMap<String, Object> expected = new LinkedHashMap<>();
List expectedList = Arrays.asList(0x35);
LinkedHashMap<String, Object> expectedResult = new LinkedHashMap<>();
expectedResult.put("result", expectedList);
expected.put("msg", expectedResult);
Object actual = invokeScript(evalScript(decoderStr), msgStr);
assertEquals(expected, actual);
decoderStr = String.format("""
var original = [];
original.add(0x67);
var unmodifiable = original.toUnmodifiable();
unmodifiable.add(0x35);
msg.result = unmodifiable;
return {msg: msg};
""");
assertThatThrownBy(() -> {
invokeScript(evalScript(decoderStr), msgStr);
}).hasMessageContaining("Error: unmodifiable.add(0x35): List is unmodifiable");
}
@Test
public void toUnmodifiableExecutionHashMap_Test() throws ExecutionException, InterruptedException {
msgStr = "{}";
decoderStr = String.format("""
var original = {};
original.putIfAbsent("entry1", 73);
var unmodifiable = original.toUnmodifiable();
msg.result = unmodifiable;
return {msg: msg};
""");
LinkedHashMap<String, Object> expected = new LinkedHashMap<>();
LinkedHashMap<String, Object> expectedMap = new LinkedHashMap<>(Map.of("entry1", 73));
LinkedHashMap<String, Object> expectedResult = new LinkedHashMap<>();
expectedResult.put("result", expectedMap);
expected.put("msg", expectedResult);
Object actual = invokeScript(evalScript(decoderStr), msgStr);
assertEquals(expected, actual);
decoderStr = String.format("""
var original = {};
original.humidity = 73;
var unmodifiable = original.toUnmodifiable();
unmodifiable.put("temperature1", 96);
msg.result = unmodifiable;
return {msg: msg};
""");
assertThatThrownBy(() -> {
invokeScript(evalScript(decoderStr), msgStr);
}).hasMessageContaining("Error: unmodifiable.put(\"temperature1\", 96): Map is unmodifiable");
}
@Test
public void tbDateFunction_Test() throws ExecutionException, InterruptedException {
String stringDateUTC = "2024-01-01T10:00:00.00Z";
TbDate d = new TbDate(stringDateUTC);
msgStr = "{}";
decoderStr = String.format("""
var d = new Date("%s"); // TZ => "UTC"
var dIsoY1 = d.toISOString(); // return 2024-01-01T10:00:00Z
d.addYears(1);
var dIsoY2 = d.toISOString(); // return 2025-01-01T10:00:00Z
d.addYears(-2);
var dIsoY3 = d.toISOString(); // return 2023-01-01T10:00:00Z
d.addMonths(2);
var dIsoM1 = d.toISOString(); // return 2023-03-01T10:00:00Z
d.addMonths(10);
var dIsoM2 = d.toISOString(); // return 2024-01-01T10:00:00Z
d.addMonths(-13);
var dIsoM3 = d.toISOString(); // return 2022-12-01T10:00:00Z
d.addWeeks(4);
var dIsoW1 = d.toISOString(); // return 2022-12-29T10:00:00Z
d.addWeeks(-5);
var dIsoW2 = d.toISOString(); // return 2022-11-24T10:00:00Z
d.addDays(6);
var dIsoD1 = d.toISOString(); // return 2022-11-30T10:00:00Z
d.addDays(45);
var dIsoD2 = d.toISOString(); // return 2023-01-14T10:00:00Z
d.addDays(-50);
var dIsoD3 = d.toISOString(); // return 2022-11-25T10:00:00Z
d.addHours(23);
var dIsoH1 = d.toISOString(); // return 2022-11-26T09:00:00Z
d.addHours(-47);
var dIsoH2 = d.toISOString(); // return 2022-11-24T10:00:00Z
d.addMinutes(59);
var dIsoMin1 = d.toISOString(); // return 2022-11-24T10:59:00Z
d.addMinutes(-60);
var dIsoMin2 = d.toISOString(); // return 2022-11-24T09:59:00Z
d.addSeconds(59);
var dIsoS1 = d.toISOString(); // return 2022-11-24T09:59:59Z
d.addSeconds(-60);
var dIsoS2 = d.toISOString(); // return 2022-11-24T09:58:59Z
d.addNanos(999999);
var dIsoN1 = d.toISOString(); // return 2022-11-24T09:58:59.000999999Z
d.addNanos(-1000000);
var dIsoN2 = d.toISOString(); // return 2022-11-24T09:58:58.999999999Z
return {
"dIsoY1": dIsoY1,
"dIsoY2": dIsoY2,
"dIsoY3": dIsoY3,
"dIsoM1": dIsoM1,
"dIsoM2": dIsoM2,
"dIsoM3": dIsoM3,
"dIsoW1": dIsoW1,
"dIsoW2": dIsoW2,
"dIsoD1": dIsoD1,
"dIsoD2": dIsoD2,
"dIsoD3": dIsoD3,
"dIsoH1": dIsoH1,
"dIsoH2": dIsoH2,
"dIsoMin1": dIsoMin1,
"dIsoMin2": dIsoMin2,
"dIsoS1": dIsoS1,
"dIsoS2": dIsoS2,
"dIsoN1": dIsoN1,
"dIsoN2": dIsoN2
}
""", stringDateUTC);
LinkedHashMap<String, Object> expected = new LinkedHashMap<>();
expected.put("dIsoY1", d.toISOString());
d.addYears(1);
expected.put("dIsoY2", d.toISOString());
d.addYears(-2);
expected.put("dIsoY3", d.toISOString());
d.addMonths(2);
expected.put("dIsoM1", d.toISOString());
d.addMonths(10);
expected.put("dIsoM2", d.toISOString());
d.addMonths(-13);
expected.put("dIsoM3", d.toISOString());
d.addWeeks(4);
expected.put("dIsoW1", d.toISOString());
d.addWeeks(-5);
expected.put("dIsoW2", d.toISOString());
d.addDays(6);
expected.put("dIsoD1", d.toISOString());
d.addDays(45);
expected.put("dIsoD2", d.toISOString());
d.addDays(-50);
expected.put("dIsoD3", d.toISOString());
d.addHours(23);
expected.put("dIsoH1", d.toISOString());
d.addHours(-47);
expected.put("dIsoH2", d.toISOString());
d.addMinutes(59);
expected.put("dIsoMin1", d.toISOString());
d.addMinutes(-60);
expected.put("dIsoMin2", d.toISOString());
d.addSeconds(59);
expected.put("dIsoS1", d.toISOString());
d.addSeconds(-60);
expected.put("dIsoS2", d.toISOString());
d.addNanos(999999);
expected.put("dIsoN1", d.toISOString());
d.addNanos(-1000000);
expected.put("dIsoN2", d.toISOString());
Object actual = invokeScript(evalScript(decoderStr), msgStr);
assertEquals(expected, actual);
}
private List splice(List oldList, int start, int deleteCount, Object... values) { private List splice(List oldList, int start, int deleteCount, Object... values) {
start = initStartIndex(oldList, start); start = initStartIndex(oldList, start);

61
application/src/test/java/org/thingsboard/server/service/state/DefaultRuleEngineDeviceStateManagerTest.java → application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateManagerTest.java

@ -52,7 +52,7 @@ import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.never; import static org.mockito.Mockito.never;
@ExtendWith(MockitoExtension.class) @ExtendWith(MockitoExtension.class)
public class DefaultRuleEngineDeviceStateManagerTest { public class DefaultDeviceStateManagerTest {
@Mock @Mock
private DeviceStateService deviceStateServiceMock; private DeviceStateService deviceStateServiceMock;
@ -71,7 +71,7 @@ public class DefaultRuleEngineDeviceStateManagerTest {
@Captor @Captor
private ArgumentCaptor<TbQueueCallback> queueCallbackCaptor; private ArgumentCaptor<TbQueueCallback> queueCallbackCaptor;
private DefaultRuleEngineDeviceStateManager deviceStateManager; private DefaultDeviceStateManager deviceStateManager;
private static final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("57ab2e6c-bc4c-11ee-a506-0242ac120002")); private static final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("57ab2e6c-bc4c-11ee-a506-0242ac120002"));
private static final DeviceId DEVICE_ID = DeviceId.fromString("74a9053e-bc4c-11ee-a506-0242ac120002"); private static final DeviceId DEVICE_ID = DeviceId.fromString("74a9053e-bc4c-11ee-a506-0242ac120002");
@ -82,7 +82,7 @@ public class DefaultRuleEngineDeviceStateManagerTest {
@BeforeEach @BeforeEach
public void setup() { public void setup() {
deviceStateManager = new DefaultRuleEngineDeviceStateManager(serviceInfoProviderMock, partitionServiceMock, Optional.of(deviceStateServiceMock), clusterServiceMock); deviceStateManager = new DefaultDeviceStateManager(serviceInfoProviderMock, partitionServiceMock, Optional.of(deviceStateServiceMock), clusterServiceMock);
} }
@ParameterizedTest @ParameterizedTest
@ -90,7 +90,7 @@ public class DefaultRuleEngineDeviceStateManagerTest {
"when onDeviceX() is called, then should route event to local service and call onSuccess() callback.") "when onDeviceX() is called, then should route event to local service and call onSuccess() callback.")
@MethodSource @MethodSource
public void givenRoutedToLocalAndProcessingSuccess_whenOnDeviceAction_thenShouldCallLocalServiceAndSuccessCallback( public void givenRoutedToLocalAndProcessingSuccess_whenOnDeviceAction_thenShouldCallLocalServiceAndSuccessCallback(
BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback> onDeviceAction, Consumer<DeviceStateService> actionVerification BiConsumer<DefaultDeviceStateManager, TbCallback> onDeviceAction, Consumer<DeviceStateService> actionVerification
) { ) {
// GIVEN // GIVEN
given(serviceInfoProviderMock.isService(ServiceType.TB_CORE)).willReturn(true); given(serviceInfoProviderMock.isService(ServiceType.TB_CORE)).willReturn(true);
@ -109,20 +109,24 @@ public class DefaultRuleEngineDeviceStateManagerTest {
private static Stream<Arguments> givenRoutedToLocalAndProcessingSuccess_whenOnDeviceAction_thenShouldCallLocalServiceAndSuccessCallback() { private static Stream<Arguments> givenRoutedToLocalAndProcessingSuccess_whenOnDeviceAction_thenShouldCallLocalServiceAndSuccessCallback() {
return Stream.of( return Stream.of(
Arguments.of( Arguments.of(
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS) (Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS)
), ),
Arguments.of( Arguments.of(
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS) (Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS)
), ),
Arguments.of( Arguments.of(
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS) (Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS)
), ),
Arguments.of( Arguments.of(
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS) (Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS)
),
Arguments.of(
(BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS)
) )
); );
} }
@ -132,7 +136,7 @@ public class DefaultRuleEngineDeviceStateManagerTest {
"when onDeviceX() is called, then should route event to local service and call onFailure() callback.") "when onDeviceX() is called, then should route event to local service and call onFailure() callback.")
@MethodSource @MethodSource
public void givenRoutedToLocalAndProcessingFailure_whenOnDeviceAction_thenShouldCallLocalServiceAndFailureCallback( public void givenRoutedToLocalAndProcessingFailure_whenOnDeviceAction_thenShouldCallLocalServiceAndFailureCallback(
Consumer<DeviceStateService> exceptionThrowSetup, BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback> onDeviceAction, Consumer<DeviceStateService> actionVerification Consumer<DeviceStateService> exceptionThrowSetup, BiConsumer<DefaultDeviceStateManager, TbCallback> onDeviceAction, Consumer<DeviceStateService> actionVerification
) { ) {
// GIVEN // GIVEN
given(serviceInfoProviderMock.isService(ServiceType.TB_CORE)).willReturn(true); given(serviceInfoProviderMock.isService(ServiceType.TB_CORE)).willReturn(true);
@ -155,23 +159,28 @@ public class DefaultRuleEngineDeviceStateManagerTest {
return Stream.of( return Stream.of(
Arguments.of( Arguments.of(
(Consumer<DeviceStateService>) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS), (Consumer<DeviceStateService>) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS),
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS) (Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS)
), ),
Arguments.of( Arguments.of(
(Consumer<DeviceStateService>) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS), (Consumer<DeviceStateService>) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS),
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS) (Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS)
), ),
Arguments.of( Arguments.of(
(Consumer<DeviceStateService>) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS), (Consumer<DeviceStateService>) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS),
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS) (Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS)
), ),
Arguments.of( Arguments.of(
(Consumer<DeviceStateService>) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS), (Consumer<DeviceStateService>) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS),
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS) (Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS)
),
Arguments.of(
(Consumer<DeviceStateService>) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS),
(BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS)
) )
); );
} }
@ -181,7 +190,7 @@ public class DefaultRuleEngineDeviceStateManagerTest {
"when onDeviceX() is called, then should send correct queue message to external service with correct callback object.") "when onDeviceX() is called, then should send correct queue message to external service with correct callback object.")
@MethodSource @MethodSource
public void givenRoutedToExternal_whenOnDeviceAction_thenShouldSendQueueMsgToExternalServiceWithCorrectCallback( public void givenRoutedToExternal_whenOnDeviceAction_thenShouldSendQueueMsgToExternalServiceWithCorrectCallback(
BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback> onDeviceAction, BiConsumer<TbClusterService, ArgumentCaptor<TbQueueCallback>> actionVerification BiConsumer<DefaultDeviceStateManager, TbCallback> onDeviceAction, BiConsumer<TbClusterService, ArgumentCaptor<TbQueueCallback>> actionVerification
) { ) {
// WHEN // WHEN
ReflectionTestUtils.setField(deviceStateManager, "deviceStateService", Optional.empty()); ReflectionTestUtils.setField(deviceStateManager, "deviceStateService", Optional.empty());
@ -203,7 +212,7 @@ public class DefaultRuleEngineDeviceStateManagerTest {
private static Stream<Arguments> givenRoutedToExternal_whenOnDeviceAction_thenShouldSendQueueMsgToExternalServiceWithCorrectCallback() { private static Stream<Arguments> givenRoutedToExternal_whenOnDeviceAction_thenShouldSendQueueMsgToExternalServiceWithCorrectCallback() {
return Stream.of( return Stream.of(
Arguments.of( Arguments.of(
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<TbClusterService, ArgumentCaptor<TbQueueCallback>>) (clusterServiceMock, queueCallbackCaptor) -> { (BiConsumer<TbClusterService, ArgumentCaptor<TbQueueCallback>>) (clusterServiceMock, queueCallbackCaptor) -> {
var deviceConnectMsg = TransportProtos.DeviceConnectProto.newBuilder() var deviceConnectMsg = TransportProtos.DeviceConnectProto.newBuilder()
.setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits())
@ -219,7 +228,7 @@ public class DefaultRuleEngineDeviceStateManagerTest {
} }
), ),
Arguments.of( Arguments.of(
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<TbClusterService, ArgumentCaptor<TbQueueCallback>>) (clusterServiceMock, queueCallbackCaptor) -> { (BiConsumer<TbClusterService, ArgumentCaptor<TbQueueCallback>>) (clusterServiceMock, queueCallbackCaptor) -> {
var deviceActivityMsg = TransportProtos.DeviceActivityProto.newBuilder() var deviceActivityMsg = TransportProtos.DeviceActivityProto.newBuilder()
.setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits())
@ -235,7 +244,7 @@ public class DefaultRuleEngineDeviceStateManagerTest {
} }
), ),
Arguments.of( Arguments.of(
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<TbClusterService, ArgumentCaptor<TbQueueCallback>>) (clusterServiceMock, queueCallbackCaptor) -> { (BiConsumer<TbClusterService, ArgumentCaptor<TbQueueCallback>>) (clusterServiceMock, queueCallbackCaptor) -> {
var deviceDisconnectMsg = TransportProtos.DeviceDisconnectProto.newBuilder() var deviceDisconnectMsg = TransportProtos.DeviceDisconnectProto.newBuilder()
.setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits())
@ -251,7 +260,7 @@ public class DefaultRuleEngineDeviceStateManagerTest {
} }
), ),
Arguments.of( Arguments.of(
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<TbClusterService, ArgumentCaptor<TbQueueCallback>>) (clusterServiceMock, queueCallbackCaptor) -> { (BiConsumer<TbClusterService, ArgumentCaptor<TbQueueCallback>>) (clusterServiceMock, queueCallbackCaptor) -> {
var deviceInactivityMsg = TransportProtos.DeviceInactivityProto.newBuilder() var deviceInactivityMsg = TransportProtos.DeviceInactivityProto.newBuilder()
.setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits())
@ -265,6 +274,22 @@ public class DefaultRuleEngineDeviceStateManagerTest {
.build(); .build();
then(clusterServiceMock).should().pushMsgToCore(eq(EXTERNAL_TPI), any(UUID.class), eq(toCoreMsg), queueCallbackCaptor.capture()); then(clusterServiceMock).should().pushMsgToCore(eq(EXTERNAL_TPI), any(UUID.class), eq(toCoreMsg), queueCallbackCaptor.capture());
} }
),
Arguments.of(
(BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<TbClusterService, ArgumentCaptor<TbQueueCallback>>) (clusterServiceMock, queueCallbackCaptor) -> {
var deviceInactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder()
.setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits())
.setTenantIdLSB(TENANT_ID.getId().getLeastSignificantBits())
.setDeviceIdMSB(DEVICE_ID.getId().getMostSignificantBits())
.setDeviceIdLSB(DEVICE_ID.getId().getLeastSignificantBits())
.setInactivityTimeout(EVENT_TS)
.build();
var toCoreMsg = TransportProtos.ToCoreMsg.newBuilder()
.setDeviceInactivityTimeoutUpdateMsg(deviceInactivityTimeoutUpdateMsg)
.build();
then(clusterServiceMock).should().pushMsgToCore(eq(EXTERNAL_TPI), any(UUID.class), eq(toCoreMsg), queueCallbackCaptor.capture());
}
) )
); );
} }

60
application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java

@ -38,9 +38,6 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.data.notification.rule.trigger.DeviceActivityTrigger; import org.thingsboard.server.common.data.notification.rule.trigger.DeviceActivityTrigger;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.TsValue;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor;
@ -88,7 +85,6 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import static org.thingsboard.server.service.state.DefaultDeviceStateService.ACTIVITY_STATE; import static org.thingsboard.server.service.state.DefaultDeviceStateService.ACTIVITY_STATE;
import static org.thingsboard.server.service.state.DefaultDeviceStateService.INACTIVITY_ALARM_TIME; import static org.thingsboard.server.service.state.DefaultDeviceStateService.INACTIVITY_ALARM_TIME;
import static org.thingsboard.server.service.state.DefaultDeviceStateService.INACTIVITY_TIMEOUT;
import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAST_ACTIVITY_TIME; import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAST_ACTIVITY_TIME;
import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAST_CONNECT_TIME; import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAST_CONNECT_TIME;
import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAST_DISCONNECT_TIME; import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAST_DISCONNECT_TIME;
@ -211,7 +207,7 @@ public class DefaultDeviceStateServiceTest {
// THEN // THEN
then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(LAST_CONNECT_TIME) && request.getEntries().get(0).getKey().equals(LAST_CONNECT_TIME) &&
request.getEntries().get(0).getValue().equals(lastConnectTime) request.getEntries().get(0).getValue().equals(lastConnectTime)
@ -298,7 +294,7 @@ public class DefaultDeviceStateServiceTest {
// THEN // THEN
then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(LAST_DISCONNECT_TIME) && request.getEntries().get(0).getKey().equals(LAST_DISCONNECT_TIME) &&
request.getEntries().get(0).getValue().equals(lastDisconnectTime) request.getEntries().get(0).getValue().equals(lastDisconnectTime)
@ -421,13 +417,13 @@ public class DefaultDeviceStateServiceTest {
// THEN // THEN
then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) &&
request.getEntries().get(0).getValue().equals(lastInactivityTime) request.getEntries().get(0).getValue().equals(lastInactivityTime)
)); ));
then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) &&
request.getEntries().get(0).getValue().equals(false) request.getEntries().get(0).getValue().equals(false)
@ -465,12 +461,12 @@ public class DefaultDeviceStateServiceTest {
// THEN // THEN
then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME)
)); ));
then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) &&
request.getEntries().get(0).getValue().equals(false) request.getEntries().get(0).getValue().equals(false)
@ -508,42 +504,6 @@ public class DefaultDeviceStateServiceTest {
verify(service).fetchDeviceStateDataUsingSeparateRequests(deviceId); verify(service).fetchDeviceStateDataUsingSeparateRequests(deviceId);
} }
@Test
public void givenPersistToTelemetryAndDefaultInactivityTimeoutFetched_whenTransformingToDeviceStateData_thenTryGetInactivityFromAttribute() {
var defaultInactivityTimeoutInSec = 60L;
var latest =
Map.of(
EntityKeyType.TIME_SERIES, Map.of(INACTIVITY_TIMEOUT, new TsValue(0, Long.toString(defaultInactivityTimeoutInSec * 1000))),
EntityKeyType.SERVER_ATTRIBUTE, Map.of(INACTIVITY_TIMEOUT, new TsValue(0, Long.toString(5000L)))
);
process(latest, defaultInactivityTimeoutInSec);
}
@Test
public void givenPersistToTelemetryAndNoInactivityTimeoutFetchedFromTimeSeries_whenTransformingToDeviceStateData_thenTryGetInactivityFromAttribute() {
var defaultInactivityTimeoutInSec = 60L;
var latest =
Map.of(
EntityKeyType.SERVER_ATTRIBUTE, Map.of(INACTIVITY_TIMEOUT, new TsValue(0, Long.toString(5000L)))
);
process(latest, defaultInactivityTimeoutInSec);
}
private void process(Map<EntityKeyType, Map<String, TsValue>> latest, long defaultInactivityTimeoutInSec) {
service.setDefaultInactivityTimeoutInSec(defaultInactivityTimeoutInSec);
service.setDefaultInactivityTimeoutMs(defaultInactivityTimeoutInSec * 1000);
service.setPersistToTelemetry(true);
var deviceUuid = UUID.randomUUID();
var deviceId = new DeviceId(deviceUuid);
DeviceStateData deviceStateData = service.toDeviceStateData(new EntityData(deviceId, latest, Map.of()), new DeviceIdInfo(TenantId.SYS_TENANT_ID.getId(), UUID.randomUUID(), deviceUuid));
assertThat(deviceStateData.getState().getInactivityTimeout()).isEqualTo(5000L);
}
private void initStateService(long timeout) throws InterruptedException { private void initStateService(long timeout) throws InterruptedException {
service.stop(); service.stop();
reset(service, telemetrySubscriptionService); reset(service, telemetrySubscriptionService);
@ -556,7 +516,7 @@ public class DefaultDeviceStateServiceTest {
.thenReturn(new PageData<>(List.of(deviceIdInfo), 0, 1, false)); .thenReturn(new PageData<>(List.of(deviceIdInfo), 0, 1, false));
PartitionChangeEvent event = new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of( PartitionChangeEvent event = new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of(
new QueueKey(ServiceType.TB_CORE), Collections.singleton(tpi) new QueueKey(ServiceType.TB_CORE), Collections.singleton(tpi)
)); ), Collections.emptyMap());
service.onApplicationEvent(event); service.onApplicationEvent(event);
Thread.sleep(100); Thread.sleep(100);
} }
@ -1002,7 +962,7 @@ public class DefaultDeviceStateServiceTest {
assertThat(actualNotification.isActive()).isFalse(); assertThat(actualNotification.isActive()).isFalse();
then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) &&
request.getEntries().get(0).getValue().equals(expectedLastInactivityAlarmTime) request.getEntries().get(0).getValue().equals(expectedLastInactivityAlarmTime)
@ -1170,7 +1130,7 @@ public class DefaultDeviceStateServiceTest {
assertThat(attributeRequestCaptor.getAllValues()).hasSize(2) assertThat(attributeRequestCaptor.getAllValues()).hasSize(2)
.anySatisfy(request -> { .anySatisfy(request -> {
assertThat(request.getTenantId()).isEqualTo(TenantId.SYS_TENANT_ID); assertThat(request.getTenantId()).isEqualTo(tenantId);
assertThat(request.getEntityId()).isEqualTo(deviceId); assertThat(request.getEntityId()).isEqualTo(deviceId);
assertThat(request.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE); assertThat(request.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE);
assertThat(request.getEntries()).singleElement().satisfies(attributeKvEntry -> { assertThat(request.getEntries()).singleElement().satisfies(attributeKvEntry -> {
@ -1179,7 +1139,7 @@ public class DefaultDeviceStateServiceTest {
}); });
}) })
.anySatisfy(request -> { .anySatisfy(request -> {
assertThat(request.getTenantId()).isEqualTo(TenantId.SYS_TENANT_ID); assertThat(request.getTenantId()).isEqualTo(tenantId);
assertThat(request.getEntityId()).isEqualTo(deviceId); assertThat(request.getEntityId()).isEqualTo(deviceId);
assertThat(request.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE); assertThat(request.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE);
assertThat(request.getEntries()).singleElement().satisfies(attributeKvEntry -> { assertThat(request.getEntries()).singleElement().satisfies(attributeKvEntry -> {

80
application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java

@ -43,6 +43,15 @@ import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.asset.AssetProfile;
import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Argument;
import org.thingsboard.server.common.data.cf.configuration.ArgumentType;
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration;
import org.thingsboard.server.common.data.cf.configuration.Output;
import org.thingsboard.server.common.data.cf.configuration.OutputType;
import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey;
import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration;
import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration;
import org.thingsboard.server.common.data.device.data.DeviceData; import org.thingsboard.server.common.data.device.data.DeviceData;
import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration;
@ -70,6 +79,7 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.sync.ie.DeviceExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
@ -79,6 +89,7 @@ import org.thingsboard.server.common.data.util.ThrowingRunnable;
import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.controller.AbstractControllerTest;
import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetProfileService;
import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.dao.cf.CalculatedFieldService;
import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.dashboard.DashboardService;
import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceProfileService;
@ -145,6 +156,8 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest {
protected TenantService tenantService; protected TenantService tenantService;
@Autowired @Autowired
protected EntityViewService entityViewService; protected EntityViewService entityViewService;
@Autowired
protected CalculatedFieldService calculatedFieldService;
protected TenantId tenantId1; protected TenantId tenantId1;
protected User tenantAdmin1; protected User tenantAdmin1;
@ -191,6 +204,7 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest {
Asset asset = createAsset(tenantId1, null, assetProfile.getId(), "Asset 1"); Asset asset = createAsset(tenantId1, null, assetProfile.getId(), "Asset 1");
DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1");
Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device 1"); Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device 1");
CalculatedField calculatedField = createCalculatedField(tenantId1, device.getId(), asset.getId());
Map<EntityType, EntityExportData> entitiesExportData = Stream.of(customer.getId(), asset.getId(), device.getId(), Map<EntityType, EntityExportData> entitiesExportData = Stream.of(customer.getId(), asset.getId(), device.getId(),
ruleChain.getId(), dashboard.getId(), assetProfile.getId(), deviceProfile.getId()) ruleChain.getId(), dashboard.getId(), assetProfile.getId(), deviceProfile.getId())
@ -198,6 +212,7 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest {
try { try {
return exportEntity(tenantAdmin1, entityId, EntityExportSettings.builder() return exportEntity(tenantAdmin1, entityId, EntityExportSettings.builder()
.exportCredentials(false) .exportCredentials(false)
.exportCalculatedFields(true)
.build()); .build());
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
@ -245,7 +260,6 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest {
verify(entityActionService, Mockito.never()).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset), verify(entityActionService, Mockito.never()).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset),
any(), eq(ActionType.UPDATED), isNull()); any(), eq(ActionType.UPDATED), isNull());
EntityExportData<Asset> updatedAssetEntity = getAndClone(entitiesExportData, EntityType.ASSET); EntityExportData<Asset> updatedAssetEntity = getAndClone(entitiesExportData, EntityType.ASSET);
updatedAssetEntity.getEntity().setLabel("t" + updatedAssetEntity.getEntity().getLabel()); updatedAssetEntity.getEntity().setLabel("t" + updatedAssetEntity.getEntity().getLabel());
Asset updatedAsset = importEntity(tenantAdmin2, updatedAssetEntity).getSavedEntity(); Asset updatedAsset = importEntity(tenantAdmin2, updatedAssetEntity).getSavedEntity();
@ -268,10 +282,27 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest {
importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.DEVICE)); importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.DEVICE));
verify(tbClusterService, Mockito.never()).onDeviceUpdated(eq(importedDevice), eq(importedDevice)); verify(tbClusterService, Mockito.never()).onDeviceUpdated(eq(importedDevice), eq(importedDevice));
// calculated field of imported device:
List<CalculatedField> calculatedFields = calculatedFieldService.findCalculatedFieldsByEntityId(tenantId2, importedDevice.getId());
assertThat(calculatedFields.size()).isOne();
var importedCalculatedField = calculatedFields.get(0);
assertThat(importedCalculatedField.getName()).isEqualTo(calculatedField.getName());
verify(tbClusterService).onCalculatedFieldUpdated(eq(importedCalculatedField), isNull(), any());
EntityExportData<Device> updatedDeviceEntity = getAndClone(entitiesExportData, EntityType.DEVICE); EntityExportData<Device> updatedDeviceEntity = getAndClone(entitiesExportData, EntityType.DEVICE);
updatedDeviceEntity.getEntity().setLabel("t" + updatedDeviceEntity.getEntity().getLabel()); updatedDeviceEntity.getEntity().setLabel("t" + updatedDeviceEntity.getEntity().getLabel());
Device updatedDevice = importEntity(tenantAdmin2, updatedDeviceEntity).getSavedEntity(); Device updatedDevice = importEntity(tenantAdmin2, updatedDeviceEntity).getSavedEntity();
verify(tbClusterService).onDeviceUpdated(eq(updatedDevice), eq(importedDevice)); verify(tbClusterService).onDeviceUpdated(eq(updatedDevice), eq(importedDevice));
// update calculated field:
DeviceExportData deviceExportData = (DeviceExportData) getAndClone(entitiesExportData, EntityType.DEVICE);
deviceExportData.setCalculatedFields(deviceExportData.getCalculatedFields().stream().peek(field -> field.setName("t_" + field.getName())).toList());
importEntity(tenantAdmin2, deviceExportData).getSavedEntity();
calculatedFields = calculatedFieldService.findCalculatedFieldsByEntityId(tenantId2, importedDevice.getId());
assertThat(calculatedFields.size()).isOne();
importedCalculatedField = calculatedFields.get(0);
assertThat(importedCalculatedField.getName()).startsWith("t_");
} }
@Test @Test
@ -290,12 +321,15 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest {
Device device = createDevice(tenantId1, customer.getId(), deviceProfile.getId(), "Device 1"); Device device = createDevice(tenantId1, customer.getId(), deviceProfile.getId(), "Device 1");
EntityView entityView = createEntityView(tenantId1, customer.getId(), device.getId(), "Entity view 1"); EntityView entityView = createEntityView(tenantId1, customer.getId(), device.getId(), "Entity view 1");
CalculatedField calculatedField = createCalculatedField(tenantId1, device.getId(), device.getId());
Map<EntityId, EntityId> ids = new HashMap<>(); Map<EntityId, EntityId> ids = new HashMap<>();
for (EntityId entityId : List.of(customer.getId(), ruleChain.getId(), dashboard.getId(), assetProfile.getId(), asset.getId(), for (EntityId entityId : List.of(customer.getId(), ruleChain.getId(), dashboard.getId(), assetProfile.getId(), asset.getId(),
deviceProfile.getId(), device.getId(), entityView.getId(), ruleChain.getId(), dashboard.getId())) { deviceProfile.getId(), device.getId(), entityView.getId(), ruleChain.getId(), dashboard.getId())) {
EntityExportData exportData = exportEntity(getSecurityUser(tenantAdmin1), entityId); EntityExportData exportData = exportEntity(getSecurityUser(tenantAdmin1), entityId);
EntityImportResult importResult = importEntity(getSecurityUser(tenantAdmin2), exportData, EntityImportSettings.builder() EntityImportResult importResult = importEntity(getSecurityUser(tenantAdmin2), exportData, EntityImportSettings.builder()
.saveCredentials(false) .saveCredentials(false)
.saveCalculatedFields(true)
.build()); .build());
ids.put(entityId, (EntityId) importResult.getSavedEntity().getId()); ids.put(entityId, (EntityId) importResult.getSavedEntity().getId());
} }
@ -325,10 +359,16 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest {
assertThat(exportedDeviceProfile.getDefaultRuleChainId()).isEqualTo(ruleChain.getId()); assertThat(exportedDeviceProfile.getDefaultRuleChainId()).isEqualTo(ruleChain.getId());
assertThat(exportedDeviceProfile.getDefaultDashboardId()).isEqualTo(dashboard.getId()); assertThat(exportedDeviceProfile.getDefaultDashboardId()).isEqualTo(dashboard.getId());
Device exportedDevice = (Device) exportEntity(tenantAdmin2, (DeviceId) ids.get(device.getId())).getEntity(); EntityExportData<Device> entityExportData = exportEntity(tenantAdmin2, (DeviceId) ids.get(device.getId()));
Device exportedDevice = entityExportData.getEntity();
assertThat(exportedDevice.getCustomerId()).isEqualTo(customer.getId()); assertThat(exportedDevice.getCustomerId()).isEqualTo(customer.getId());
assertThat(exportedDevice.getDeviceProfileId()).isEqualTo(deviceProfile.getId()); assertThat(exportedDevice.getDeviceProfileId()).isEqualTo(deviceProfile.getId());
List<CalculatedField> calculatedFields = ((DeviceExportData) entityExportData).getCalculatedFields();
assertThat(calculatedFields.size()).isOne();
CalculatedField field = calculatedFields.get(0);
assertThat(field.getName()).isEqualTo(calculatedField.getName());
EntityView exportedEntityView = (EntityView) exportEntity(tenantAdmin2, (EntityViewId) ids.get(entityView.getId())).getEntity(); EntityView exportedEntityView = (EntityView) exportEntity(tenantAdmin2, (EntityViewId) ids.get(entityView.getId())).getEntity();
assertThat(exportedEntityView.getCustomerId()).isEqualTo(customer.getId()); assertThat(exportedEntityView.getCustomerId()).isEqualTo(customer.getId());
assertThat(exportedEntityView.getEntityId()).isEqualTo(device.getId()); assertThat(exportedEntityView.getEntityId()).isEqualTo(device.getId());
@ -340,7 +380,6 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest {
deviceProfileService.saveDeviceProfile(importedDeviceProfile); deviceProfileService.saveDeviceProfile(importedDeviceProfile);
} }
protected Device createDevice(TenantId tenantId, CustomerId customerId, DeviceProfileId deviceProfileId, String name) { protected Device createDevice(TenantId tenantId, CustomerId customerId, DeviceProfileId deviceProfileId, String name) {
Device device = new Device(); Device device = new Device();
device.setTenantId(tenantId); device.setTenantId(tenantId);
@ -549,9 +588,43 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest {
return relation; return relation;
} }
private CalculatedField createCalculatedField(TenantId tenantId, EntityId entityId, EntityId referencedEntityId) {
CalculatedField calculatedField = new CalculatedField();
calculatedField.setTenantId(tenantId);
calculatedField.setEntityId(entityId);
calculatedField.setType(CalculatedFieldType.SIMPLE);
calculatedField.setName("Test Calculated Field");
calculatedField.setConfigurationVersion(1);
calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId));
calculatedField.setVersion(1L);
return calculatedFieldService.save(calculatedField);
}
private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) {
SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration();
Argument argument = new Argument();
argument.setRefEntityId(referencedEntityId);
ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null);
argument.setRefEntityKey(refEntityKey);
config.setArguments(Map.of("T", argument));
config.setExpression("T - (100 - H) / 5");
Output output = new Output();
output.setName("output");
output.setType(OutputType.TIME_SERIES);
config.setOutput(output);
return config;
}
protected <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> exportEntity(User user, I entityId) throws Exception { protected <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> exportEntity(User user, I entityId) throws Exception {
return exportEntity(user, entityId, EntityExportSettings.builder() return exportEntity(user, entityId, EntityExportSettings.builder()
.exportCredentials(true) .exportCredentials(true)
.exportCalculatedFields(true)
.build()); .build());
} }
@ -562,6 +635,7 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest {
protected <E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> importEntity(User user, EntityExportData<E> exportData) throws Exception { protected <E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> importEntity(User user, EntityExportData<E> exportData) throws Exception {
return importEntity(user, exportData, EntityImportSettings.builder() return importEntity(user, exportData, EntityImportSettings.builder()
.saveCredentials(true) .saveCredentials(true)
.saveCalculatedFields(true)
.build()); .build());
} }

128
application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java

@ -44,6 +44,15 @@ import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.asset.AssetProfile;
import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Argument;
import org.thingsboard.server.common.data.cf.configuration.ArgumentType;
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration;
import org.thingsboard.server.common.data.cf.configuration.Output;
import org.thingsboard.server.common.data.cf.configuration.OutputType;
import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey;
import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration;
import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.debug.DebugSettings;
import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration;
import org.thingsboard.server.common.data.device.data.DeviceData; import org.thingsboard.server.common.data.device.data.DeviceData;
@ -562,6 +571,81 @@ public class VersionControlTest extends AbstractControllerTest {
checkImportedEntity(tenantId1, defaultDeviceProfile, tenantId2, importedDeviceProfile); checkImportedEntity(tenantId1, defaultDeviceProfile, tenantId2, importedDeviceProfile);
} }
@Test
public void testVcWithCalculatedFields_betweenTenants() throws Exception {
Asset asset = createAsset(null, null, "Asset 1");
Device device = createDevice(null, null, "Device 1", "test1");
CalculatedField calculatedField = createCalculatedField("CalculatedField1", device.getId(), asset.getId());
String versionId = createVersion("calculated fields of asset and device", EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE);
loginTenant2();
loadVersion(versionId, config -> {
config.setLoadCredentials(false);
}, EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE);
Asset importedAsset = findAsset(asset.getName());
Device importedDevice = findDevice(device.getName());
checkImportedEntity(tenantId1, device, tenantId2, importedDevice);
checkImportedEntity(tenantId1, asset, tenantId2, importedAsset);
List<CalculatedField> importedCalculatedFields = findCalculatedFieldsByEntityId(importedDevice.getId());
assertThat(importedCalculatedFields).size().isOne();
assertThat(importedCalculatedFields.get(0)).satisfies(importedField -> {
assertThat(importedField.getName()).isEqualTo(calculatedField.getName());
assertThat(importedField.getType()).isEqualTo(calculatedField.getType());
assertThat(importedField.getId()).isNotEqualTo(calculatedField.getId());
});
}
@Test
public void testVcWithReferencedCalculatedFields_betweenTenants() throws Exception {
Asset asset = createAsset(null, null, "Asset 1");
Device device = createDevice(null, null, "Device 1", "test1");
CalculatedField deviceCalculatedField = createCalculatedField("CalculatedField1", device.getId(), asset.getId());
CalculatedField assetCalculatedField = createCalculatedField("CalculatedField2", asset.getId(), device.getId());
String versionId = createVersion("calculated fields of asset and device", EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE);
loginTenant2();
loadVersion(versionId, config -> {
config.setLoadCredentials(false);
}, EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE);
Asset importedAsset = findAsset(asset.getName());
Device importedDevice = findDevice(device.getName());
checkImportedEntity(tenantId1, device, tenantId2, importedDevice);
checkImportedEntity(tenantId1, asset, tenantId2, importedAsset);
List<CalculatedField> importedDeviceCalculatedFields = findCalculatedFieldsByEntityId(importedDevice.getId());
assertThat(importedDeviceCalculatedFields).size().isOne();
assertThat(importedDeviceCalculatedFields.get(0)).satisfies(importedField -> {
assertThat(importedField.getName()).isEqualTo(deviceCalculatedField.getName());
assertThat(importedField.getType()).isEqualTo(deviceCalculatedField.getType());
assertThat(importedField.getId()).isNotEqualTo(deviceCalculatedField.getId());
});
List<CalculatedField> importedAssetCalculatedFields = findCalculatedFieldsByEntityId(importedAsset.getId());
assertThat(importedAssetCalculatedFields).size().isOne();
assertThat(importedAssetCalculatedFields.get(0)).satisfies(importedField -> {
assertThat(importedField.getName()).isEqualTo(assetCalculatedField.getName());
assertThat(importedField.getType()).isEqualTo(assetCalculatedField.getType());
assertThat(importedField.getId()).isNotEqualTo(assetCalculatedField.getId());
});
}
@Test
public void testVcWithCalculatedFields_sameTenant() throws Exception {
Asset asset = createAsset(null, null, "Asset 1");
CalculatedField calculatedField = createCalculatedField("CalculatedField", asset.getId(), asset.getId());
String versionId = createVersion("asset and field", EntityType.ASSET);
loadVersion(versionId, EntityType.ASSET);
CalculatedField importedCalculatedField = findCalculatedFieldByEntityId(asset.getId());
assertThat(importedCalculatedField.getId()).isEqualTo(calculatedField.getId());
assertThat(importedCalculatedField.getName()).isEqualTo(calculatedField.getName());
assertThat(importedCalculatedField.getConfiguration()).isEqualTo(calculatedField.getConfiguration());
assertThat(importedCalculatedField.getType()).isEqualTo(calculatedField.getType());
}
private <E extends ExportableEntity<?> & HasTenantId> void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, E importedEntity) { private <E extends ExportableEntity<?> & HasTenantId> void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, E importedEntity) {
assertThat(initialEntity.getTenantId()).isEqualTo(tenantId1); assertThat(initialEntity.getTenantId()).isEqualTo(tenantId1);
assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2); assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2);
@ -626,10 +710,10 @@ public class VersionControlTest extends AbstractControllerTest {
request.setEntityTypes(Arrays.stream(entityTypes).collect(Collectors.toMap(t -> t, entityType -> { request.setEntityTypes(Arrays.stream(entityTypes).collect(Collectors.toMap(t -> t, entityType -> {
EntityTypeVersionCreateConfig config = new EntityTypeVersionCreateConfig(); EntityTypeVersionCreateConfig config = new EntityTypeVersionCreateConfig();
config.setAllEntities(true); config.setAllEntities(true);
config.setSaveRelations(true); config.setSaveRelations(true);
config.setSaveAttributes(true); config.setSaveAttributes(true);
config.setSaveCredentials(true); config.setSaveCredentials(true);
config.setSaveCalculatedFields(true);
return config; return config;
}))); })));
@ -695,6 +779,7 @@ public class VersionControlTest extends AbstractControllerTest {
config.setLoadAttributes(true); config.setLoadAttributes(true);
config.setLoadRelations(true); config.setLoadRelations(true);
config.setLoadCredentials(true); config.setLoadCredentials(true);
config.setLoadCalculatedFields(true);
config.setRemoveOtherEntities(false); config.setRemoveOtherEntities(false);
config.setFindExistingEntityByName(true); config.setFindExistingEntityByName(true);
configModifier.accept(config); configModifier.accept(config);
@ -941,6 +1026,38 @@ public class VersionControlTest extends AbstractControllerTest {
return doPost("/api/v2/relation", relation, EntityRelation.class); return doPost("/api/v2/relation", relation, EntityRelation.class);
} }
private CalculatedField createCalculatedField(String name, EntityId entityId, EntityId referencedEntityId) {
CalculatedField calculatedField = new CalculatedField();
calculatedField.setEntityId(entityId);
calculatedField.setType(CalculatedFieldType.SIMPLE);
calculatedField.setName(name);
calculatedField.setConfigurationVersion(1);
calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId));
calculatedField.setVersion(1L);
return doPost("/api/calculatedField", calculatedField, CalculatedField.class);
}
private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) {
SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration();
Argument argument = new Argument();
argument.setRefEntityId(referencedEntityId);
ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null);
argument.setRefEntityKey(refEntityKey);
config.setArguments(Map.of("T", argument));
config.setExpression("T - (100 - H) / 5");
Output output = new Output();
output.setName("output");
output.setType(OutputType.TIME_SERIES);
config.setOutput(output);
return config;
}
protected void checkImportedRuleChainData(RuleChain initialRuleChain, RuleChainMetaData initialMetaData, RuleChain importedRuleChain, RuleChainMetaData importedMetaData) { protected void checkImportedRuleChainData(RuleChain initialRuleChain, RuleChainMetaData initialMetaData, RuleChain importedRuleChain, RuleChainMetaData importedMetaData) {
assertThat(importedRuleChain.getType()).isEqualTo(initialRuleChain.getType()); assertThat(importedRuleChain.getType()).isEqualTo(initialRuleChain.getType());
assertThat(importedRuleChain.getName()).isEqualTo(initialRuleChain.getName()); assertThat(importedRuleChain.getName()).isEqualTo(initialRuleChain.getName());
@ -995,11 +1112,18 @@ public class VersionControlTest extends AbstractControllerTest {
private RuleChain findRuleChain(String name) throws Exception { private RuleChain findRuleChain(String name) throws Exception {
return doGetTypedWithPageLink("/api/ruleChains?", new TypeReference<PageData<RuleChain>>() {}, new PageLink(100, 0, name)).getData().get(0); return doGetTypedWithPageLink("/api/ruleChains?", new TypeReference<PageData<RuleChain>>() {}, new PageLink(100, 0, name)).getData().get(0);
} }
private RuleChainMetaData findRuleChainMetaData(RuleChainId ruleChainId) throws Exception { private RuleChainMetaData findRuleChainMetaData(RuleChainId ruleChainId) throws Exception {
return doGet("/api/ruleChain/" + ruleChainId + "/metadata", RuleChainMetaData.class); return doGet("/api/ruleChain/" + ruleChainId + "/metadata", RuleChainMetaData.class);
} }
private CalculatedField findCalculatedFieldByEntityId(EntityId entityId) throws Exception {
return doGetTypedWithPageLink("/api/" + entityId.getEntityType() + "/" + entityId.getId() + "/calculatedFields?", new TypeReference<PageData<CalculatedField>>() {}, new PageLink(100, 0)).getData().get(0);
}
private List<CalculatedField> findCalculatedFieldsByEntityId(EntityId entityId) throws Exception {
return doGetTypedWithPageLink("/api/" + entityId.getEntityType() + "/" + entityId.getId() + "/calculatedFields?", new TypeReference<PageData<CalculatedField>>() {}, new PageLink(100, 0)).getData();
}
} }

26
application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java

@ -174,7 +174,7 @@ class DefaultTelemetrySubscriptionServiceTest {
}).when(calculatedFieldQueueService).pushRequestToQueue(any(TimeseriesSaveRequest.class), any(), any()); }).when(calculatedFieldQueueService).pushRequestToQueue(any(TimeseriesSaveRequest.class), any(), any());
// send partition change event so currentPartitions set is populated // send partition change event so currentPartitions set is populated
telemetryService.onTbApplicationEvent(new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of(new QueueKey(ServiceType.TB_CORE), Set.of(tpi)))); telemetryService.onTbApplicationEvent(new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of(new QueueKey(ServiceType.TB_CORE), Set.of(tpi)), Collections.emptyMap()));
} }
@AfterEach @AfterEach
@ -183,6 +183,28 @@ class DefaultTelemetrySubscriptionServiceTest {
tsCallBackExecutor.shutdownNow(); tsCallBackExecutor.shutdownNow();
} }
/* --- Save time series API --- */
@Test
void shouldThrowErrorWhenTryingToSaveTimeseriesForApiUsageState() {
// GIVEN
var request = TimeseriesSaveRequest.builder()
.tenantId(tenantId)
.customerId(customerId)
.entityId(new ApiUsageStateId(UUID.randomUUID()))
.entries(sampleTimeseries)
.strategy(TimeseriesSaveRequest.Strategy.PROCESS_ALL)
.build();
// WHEN
assertThatThrownBy(() -> telemetryService.saveTimeseries(request))
.isInstanceOf(RuntimeException.class)
.hasMessage("Can't update API Usage State!");
// THEN
then(tsService).shouldHaveNoInteractions();
}
@Test @Test
void shouldReportStorageDataPointsApiUsageWhenTimeSeriesIsSaved() { void shouldReportStorageDataPointsApiUsageWhenTimeSeriesIsSaved() {
// GIVEN // GIVEN
@ -392,6 +414,8 @@ class DefaultTelemetrySubscriptionServiceTest {
); );
} }
/* --- Save attributes API --- */
@ParameterizedTest @ParameterizedTest
@MethodSource("allCombinationsOfThreeBooleans") @MethodSource("allCombinationsOfThreeBooleans")
void shouldCallCorrectSaveAttributesApiBasedOnBooleanFlagsInTheSaveRequest(boolean saveAttributes, boolean sendWsUpdate, boolean processCalculatedFields) { void shouldCallCorrectSaveAttributesApiBasedOnBooleanFlagsInTheSaveRequest(boolean saveAttributes, boolean sendWsUpdate, boolean processCalculatedFields) {

3
application/src/test/resources/application-test.properties

@ -58,3 +58,6 @@ sql.ttl.edge_events.edge_event_ttl=2592000
server.log_controller_error_stack_trace=false server.log_controller_error_stack_trace=false
transport.gateway.dashboard.sync.enabled=false transport.gateway.dashboard.sync.enabled=false
queue.edqs.sync.enabled=false
queue.edqs.api.supported=false

6
application/src/test/resources/update/330/device_profile_001_out.json

@ -64,7 +64,8 @@
"dynamicValue": { "dynamicValue": {
"sourceType": null, "sourceType": null,
"sourceAttribute": null, "sourceAttribute": null,
"inherit": false "inherit": false,
"resolvedValue" : null
} }
} }
} }
@ -103,7 +104,8 @@
"dynamicValue": { "dynamicValue": {
"sourceType": null, "sourceType": null,
"sourceAttribute": null, "sourceAttribute": null,
"inherit": false "inherit": false,
"resolvedValue" : null
} }
} }
} }

2
common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java

@ -26,6 +26,8 @@ public interface TbQueueRequestTemplate<Request extends TbQueueMsg, Response ext
ListenableFuture<Response> send(Request request, long timeoutNs); ListenableFuture<Response> send(Request request, long timeoutNs);
ListenableFuture<Response> send(Request request, Integer partition);
void stop(); void stop();
void setMessagesStats(MessagesStats messagesStats); void setMessagesStats(MessagesStats messagesStats);

10
common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java

@ -15,9 +15,17 @@
*/ */
package org.thingsboard.server.queue; package org.thingsboard.server.queue;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import java.util.Set;
public interface TbQueueResponseTemplate<Request extends TbQueueMsg, Response extends TbQueueMsg> { public interface TbQueueResponseTemplate<Request extends TbQueueMsg, Response extends TbQueueMsg> {
void init(TbQueueHandler<Request, Response> handler); void subscribe();
void subscribe(Set<TopicPartitionInfo> partitions);
void launch(TbQueueHandler<Request, Response> handler);
void stop(); void stop();
} }

2
common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java

@ -118,6 +118,8 @@ public interface AlarmService extends EntityDaoService {
long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query); long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query);
long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query, Collection<EntityId> orderedEntityIds);
PageData<EntitySubtype> findAlarmTypesByTenantId(TenantId tenantId, PageLink pageLink); PageData<EntitySubtype> findAlarmTypesByTenantId(TenantId tenantId, PageLink pageLink);
List<UUID> findActiveOriginatorAlarms(TenantId tenantId, OriginatorAlarmFilter originatorAlarmFilter, int limit); List<UUID> findActiveOriginatorAlarms(TenantId tenantId, OriginatorAlarmFilter originatorAlarmFilter, int limit);

2
common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java

@ -46,6 +46,8 @@ public interface RuleChainService extends EntityDaoService {
RuleChain saveRuleChain(RuleChain ruleChain, boolean publishSaveEvent); RuleChain saveRuleChain(RuleChain ruleChain, boolean publishSaveEvent);
RuleChain saveRuleChain(RuleChain ruleChain, boolean publishSaveEvent, boolean doValidate);
boolean setRootRuleChain(TenantId tenantId, RuleChainId ruleChainId); boolean setRootRuleChain(TenantId tenantId, RuleChainId ruleChainId);
RuleChainUpdateResult saveRuleChainMetaData(TenantId tenantId, RuleChainMetaData ruleChainMetaData, Function<RuleNode, RuleNode> ruleNodeUpdater); RuleChainUpdateResult saveRuleChainMetaData(TenantId tenantId, RuleChainMetaData ruleChainMetaData, Function<RuleNode, RuleNode> ruleNodeUpdater);

89
common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java

@ -0,0 +1,89 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
public enum ObjectType {
TENANT,
TENANT_PROFILE,
CUSTOMER,
QUEUE,
RPC,
RULE_CHAIN,
OTA_PACKAGE,
RESOURCE,
EVENT,
RULE_NODE,
USER,
EDGE,
WIDGETS_BUNDLE,
WIDGET_TYPE,
DASHBOARD,
DEVICE_PROFILE,
DEVICE,
DEVICE_CREDENTIALS,
ASSET_PROFILE,
ASSET,
ENTITY_VIEW,
ALARM,
ENTITY_ALARM,
OAUTH2_CLIENT,
OAUTH2_DOMAIN,
OAUTH2_MOBILE,
USER_SETTINGS,
NOTIFICATION_TARGET,
NOTIFICATION_TEMPLATE,
NOTIFICATION_RULE,
ALARM_COMMENT,
API_USAGE_STATE,
QUEUE_STATS,
AUDIT_LOG,
RELATION,
ATTRIBUTE_KV,
LATEST_TS_KV;
public static final Set<ObjectType> edqsTenantTypes = EnumSet.of(
TENANT, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD,
RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, API_USAGE_STATE, QUEUE_STATS
);
public static final Set<ObjectType> edqsTypes = EnumSet.copyOf(edqsTenantTypes);
public static final Set<ObjectType> edqsSystemTypes = EnumSet.of(TENANT, USER, DASHBOARD,
API_USAGE_STATE, ATTRIBUTE_KV, LATEST_TS_KV);
public static final Set<ObjectType> unversionedTypes = EnumSet.of(
QUEUE_STATS // created once, never updated
);
static {
edqsTypes.addAll(List.of(RELATION, ATTRIBUTE_KV, LATEST_TS_KV));
}
public EntityType toEntityType() {
return EntityType.valueOf(name());
}
public static ObjectType fromEntityType(EntityType entityType) {
try {
return ObjectType.valueOf(entityType.name());
} catch (Exception e) {
return null;
}
}
}

1
common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java

@ -19,7 +19,6 @@ import lombok.Data;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.AssetProfileId;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;

26
common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java

@ -23,7 +23,6 @@ import lombok.EqualsAndHashCode;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.BaseData;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.HasDebugSettings; import org.thingsboard.server.common.data.HasDebugSettings;
import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasName;
import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.HasTenantId;
@ -37,11 +36,14 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.Length;
import org.thingsboard.server.common.data.validation.NoXss; import org.thingsboard.server.common.data.validation.NoXss;
import java.io.Serial;
@Schema @Schema
@Data @Data
@EqualsAndHashCode(callSuper = true) @EqualsAndHashCode(callSuper = true)
public class CalculatedField extends BaseData<CalculatedFieldId> implements HasName, HasTenantId, HasVersion, ExportableEntity<CalculatedFieldId>, HasDebugSettings { public class CalculatedField extends BaseData<CalculatedFieldId> implements HasName, HasTenantId, HasVersion, HasDebugSettings {
@Serial
private static final long serialVersionUID = 4491966747773381420L; private static final long serialVersionUID = 4491966747773381420L;
private TenantId tenantId; private TenantId tenantId;
@ -66,9 +68,6 @@ public class CalculatedField extends BaseData<CalculatedFieldId> implements HasN
@Getter @Getter
@Setter @Setter
private Long version; private Long version;
@Getter
@Setter
private CalculatedFieldId externalId;
public CalculatedField() { public CalculatedField() {
super(); super();
@ -78,7 +77,7 @@ public class CalculatedField extends BaseData<CalculatedFieldId> implements HasN
super(id); super(id);
} }
public CalculatedField(TenantId tenantId, EntityId entityId, CalculatedFieldType type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version, CalculatedFieldId externalId) { public CalculatedField(TenantId tenantId, EntityId entityId, CalculatedFieldType type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version) {
this.tenantId = tenantId; this.tenantId = tenantId;
this.entityId = entityId; this.entityId = entityId;
this.type = type; this.type = type;
@ -86,7 +85,19 @@ public class CalculatedField extends BaseData<CalculatedFieldId> implements HasN
this.configurationVersion = configurationVersion; this.configurationVersion = configurationVersion;
this.configuration = configuration; this.configuration = configuration;
this.version = version; this.version = version;
this.externalId = externalId; }
public CalculatedField(CalculatedField calculatedField) {
super(calculatedField);
this.tenantId = calculatedField.tenantId;
this.entityId = calculatedField.entityId;
this.type = calculatedField.type;
this.name = calculatedField.name;
this.debugMode = calculatedField.debugMode;
this.debugSettings = calculatedField.debugSettings;
this.configurationVersion = calculatedField.configurationVersion;
this.configuration = calculatedField.configuration;
this.version = calculatedField.version;
} }
@Schema(description = "JSON object with the Calculated Field Id. Referencing non-existing Calculated Field Id will cause error.") @Schema(description = "JSON object with the Calculated Field Id. Referencing non-existing Calculated Field Id will cause error.")
@ -112,7 +123,6 @@ public class CalculatedField extends BaseData<CalculatedFieldId> implements HasN
.append(", configurationVersion=").append(configurationVersion) .append(", configurationVersion=").append(configurationVersion)
.append(", configuration=").append(configuration) .append(", configuration=").append(configuration)
.append(", version=").append(version) .append(", version=").append(version)
.append(", externalId=").append(externalId)
.append(", createdTime=").append(createdTime) .append(", createdTime=").append(createdTime)
.append(", id=").append(id).append(']') .append(", id=").append(id).append(']')
.toString(); .toString();

1
common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java

@ -26,5 +26,6 @@ public class Output {
private String name; private String name;
private OutputType type; private OutputType type;
private AttributeScope scope; private AttributeScope scope;
private Integer decimalsByDefault;
} }

75
common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java

@ -0,0 +1,75 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.edqs;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.ObjectType;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.KvEntry;
@Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class AttributeKv implements EdqsObject {
private EntityId entityId;
private AttributeScope scope;
private String key;
private Long version;
private DataPoint dataPoint; // optional (on deletion)
private Long lastUpdateTs; // only for serialization
private KvEntry value; // only for serialization
public AttributeKv(EntityId entityId, AttributeScope scope, AttributeKvEntry attributeKvEntry, long version) {
this.entityId = entityId;
this.scope = scope;
this.key = attributeKvEntry.getKey();
this.version = version;
this.lastUpdateTs = attributeKvEntry.getLastUpdateTs();
this.value = attributeKvEntry;
}
public AttributeKv(EntityId entityId, AttributeScope scope, String key, long version) {
this.entityId = entityId;
this.scope = scope;
this.key = key;
this.version = version;
}
@Override
public String key() {
return "a_" + entityId + "_" + scope + "_" + key;
}
@Override
public Long version() {
return version;
}
@Override
public ObjectType type() {
return ObjectType.ATTRIBUTE_KV;
}
}

40
common/data/src/main/java/org/thingsboard/server/common/data/edqs/DataPoint.java

@ -0,0 +1,40 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.edqs;
import org.thingsboard.server.common.data.kv.DataType;
public interface DataPoint {
String NOT_SUPPORTED = "Not supported!";
long getTs();
DataType getType();
String getStr();
long getLong();
double getDouble();
boolean getBool();
String getJson();
String valueToString();
}

34
common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEvent.java

@ -0,0 +1,34 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.edqs;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import org.thingsboard.server.common.data.ObjectType;
import org.thingsboard.server.common.data.id.TenantId;
@Data
@AllArgsConstructor
@Builder
public class EdqsEvent {
private final TenantId tenantId;
private final ObjectType objectType;
private final EdqsEventType eventType;
private final EdqsObject object;
}

21
common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEventType.java

@ -0,0 +1,21 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.edqs;
public enum EdqsEventType {
UPDATED,
DELETED
}

32
common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java

@ -0,0 +1,32 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.edqs;
import com.fasterxml.jackson.annotation.JsonIgnore;
import org.thingsboard.server.common.data.ObjectType;
public interface EdqsObject {
@JsonIgnore
String key();
@JsonIgnore
Long version();
@JsonIgnore
ObjectType type();
}

24
common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsSyncRequest.java

@ -0,0 +1,24 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.edqs;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;
@Data
@JsonIgnoreProperties
public class EdqsSyncRequest {
}

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save