Browse Source

Save attributes strategies: merge with master

pull/12764/head
Dmytro Skarzhynets 1 year ago
parent
commit
e2cf33e6cb
No known key found for this signature in database GPG Key ID: 2B51652F224037DF
  1. 6
      .github/workflows/check-configuration-files.yml
  2. 4
      application/pom.xml
  3. 2
      application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json
  4. 17
      application/src/main/data/upgrade/basic/schema_update.sql
  5. 14
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  6. 39
      application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java
  7. 3
      application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java
  8. 10
      application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java
  9. 21
      application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java
  10. 3
      application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java
  11. 11
      application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java
  12. 1
      application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java
  13. 2
      application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java
  14. 33
      application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java
  15. 19
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java
  16. 2
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java
  17. 4
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java
  18. 15
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java
  19. 12
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java
  20. 2
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java
  21. 5
      application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java
  22. 12
      application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java
  23. 4
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeEntityProcessor.java
  24. 82
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/BaseRuleChainProcessor.java
  25. 98
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java
  26. 5
      application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java
  27. 117
      application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java
  28. 298
      application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java
  29. 61
      application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java
  30. 284
      application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java
  31. 42
      application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java
  32. 35
      application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java
  33. 10
      application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java
  34. 45
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
  35. 14
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java
  36. 2
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java
  37. 8
      application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java
  38. 2
      application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java
  39. 180
      application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateManager.java
  40. 108
      application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java
  41. 196
      application/src/main/java/org/thingsboard/server/service/state/DefaultRuleEngineDeviceStateManager.java
  42. 6
      application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java
  43. 36
      application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java
  44. 119
      application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java
  45. 3
      application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java
  46. 5
      application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java
  47. 1
      application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java
  48. 21
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java
  49. 6
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java
  50. 6
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java
  51. 57
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java
  52. 9
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java
  53. 6
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java
  54. 10
      application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
  55. 2
      application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java
  56. 5
      application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java
  57. 1
      application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java
  58. 2
      application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java
  59. 3
      application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java
  60. 1
      application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java
  61. 66
      application/src/main/resources/thingsboard.yml
  62. 2
      application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java
  63. 72
      application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java
  64. 198
      application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java
  65. 85
      application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java
  66. 49
      application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java
  67. 27
      application/src/test/java/org/thingsboard/server/edge/AssetEdgeTest.java
  68. 1
      application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java
  69. 24
      application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java
  70. 84
      application/src/test/java/org/thingsboard/server/edge/RuleChainEdgeTest.java
  71. 17
      application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java
  72. 4
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java
  73. 30
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java
  74. 2
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java
  75. 128
      application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java
  76. 799
      application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java
  77. 93
      application/src/test/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerServiceTest.java
  78. 11
      application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java
  79. 202
      application/src/test/java/org/thingsboard/server/service/script/TbelInvokeDocsIoTest.java
  80. 61
      application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateManagerTest.java
  81. 60
      application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java
  82. 80
      application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java
  83. 128
      application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java
  84. 26
      application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java
  85. 3
      application/src/test/resources/application-test.properties
  86. 6
      application/src/test/resources/update/330/device_profile_001_out.json
  87. 2
      common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java
  88. 10
      common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java
  89. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java
  90. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java
  91. 89
      common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java
  92. 1
      common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java
  93. 26
      common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java
  94. 1
      common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java
  95. 75
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java
  96. 40
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/DataPoint.java
  97. 34
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEvent.java
  98. 21
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEventType.java
  99. 32
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java
  100. 24
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsSyncRequest.java

6
.github/workflows/check-configuration-files.yml

@ -32,14 +32,14 @@ on:
jobs:
build:
name: Check thingsboard.yml file
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Set up Python 3.10
- name: Set up Python 3.13
uses: actions/setup-python@v3
with:
python-version: "3.10.2"
python-version: "3.13.2"
architecture: "x64"
env:
AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache

4
application/pom.xml

@ -124,6 +124,10 @@
<groupId>org.thingsboard.common</groupId>
<artifactId>edge-api</artifactId>
</dependency>
<dependency>
<groupId>org.thingsboard.common</groupId>
<artifactId>edqs</artifactId>
</dependency>
<dependency>
<groupId>org.thingsboard</groupId>
<artifactId>dao</artifactId>

2
application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json

@ -122,7 +122,7 @@
"type": "org.thingsboard.rule.engine.edge.TbMsgPushToCloudNode",
"name": "Push to cloud",
"configuration": {
"scope": "SERVER_SCOPE"
"scope": "CLIENT_SCOPE"
},
"externalId": null
},

17
application/src/main/data/upgrade/basic/schema_update.sql

@ -66,3 +66,20 @@ WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode'
-- UPDATE SAVE ATTRIBUTES NODES END
ALTER TABLE api_usage_state ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1;
-- UPDATE TENANT PROFILE CALCULATED FIELD LIMITS START
UPDATE tenant_profile
SET profile_data = profile_data
|| jsonb_build_object(
'configuration', profile_data->'configuration' || jsonb_build_object(
'maxCalculatedFieldsPerEntity', COALESCE(profile_data->'configuration'->>'maxCalculatedFieldsPerEntity', '5')::bigint,
'maxArgumentsPerCF', COALESCE(profile_data->'configuration'->>'maxArgumentsPerCF', '10')::bigint,
'maxDataPointsPerRollingArg', COALESCE(profile_data->'configuration'->>'maxDataPointsPerRollingArg', '1000')::bigint,
'maxStateSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxStateSizeInKBytes', '32')::bigint,
'maxSingleValueArgumentSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxSingleValueArgumentSizeInKBytes', '2')::bigint
)
)
WHERE profile_data->'configuration'->>'maxCalculatedFieldsPerEntity' IS NULL;
-- UPDATE TENANT PROFILE CALCULATED FIELD LIMITS END

14
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@ -33,7 +33,7 @@ import org.springframework.stereotype.Component;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.rule.engine.api.MailService;
import org.thingsboard.rule.engine.api.NotificationCenter;
import org.thingsboard.rule.engine.api.RuleEngineDeviceStateManager;
import org.thingsboard.rule.engine.api.DeviceStateManager;
import org.thingsboard.rule.engine.api.SmsService;
import org.thingsboard.rule.engine.api.notification.SlackService;
import org.thingsboard.rule.engine.api.sms.SmsSenderFactory;
@ -109,6 +109,7 @@ import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.cf.CalculatedFieldProcessingService;
import org.thingsboard.server.service.cf.CalculatedFieldQueueService;
import org.thingsboard.server.service.cf.CalculatedFieldStateService;
import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache;
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry;
@ -233,7 +234,7 @@ public class ActorSystemContext {
@Autowired(required = false)
@Getter
private RuleEngineDeviceStateManager deviceStateManager;
private DeviceStateManager deviceStateManager;
@Autowired
@Getter
@ -544,6 +545,11 @@ public class ActorSystemContext {
@Getter
private CalculatedFieldStateService calculatedFieldStateService;
@Lazy
@Autowired(required = false)
@Getter
private CalculatedFieldQueueService calculatedFieldQueueService;
@Lazy
@Autowired(required = false)
@Getter
@ -644,6 +650,10 @@ public class ActorSystemContext {
@Getter
private String deviceStateNodeRateLimitConfig;
@Value("${actors.calculated_fields.calculation_timeout:5}")
@Getter
private long cfCalculationResultTimeout;
@Getter
@Setter
private TbActorSystem actorSystem;

39
application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java

@ -242,7 +242,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
justRestored = true;
}
if (state.isSizeOk()) {
if (state.updateState(newArgValues) || justRestored) {
if (state.updateState(ctx, newArgValues) || justRestored) {
cfIdList = new ArrayList<>(cfIdList);
cfIdList.add(ctx.getCfId());
processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback);
@ -274,36 +274,34 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
private void processStateIfReady(CalculatedFieldCtx ctx, List<CalculatedFieldId> cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) throws CalculatedFieldException {
CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId);
boolean stateSizeOk;
if (ctx.isInitialized() && state.isReady()) {
try {
CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS);
boolean stateSizeChecked = false;
try {
if (ctx.isInitialized() && state.isReady()) {
CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(systemContext.getCfCalculationResultTimeout(), TimeUnit.SECONDS);
state.checkStateSize(ctxId, ctx.getMaxStateSize());
stateSizeOk = state.isSizeOk();
if (stateSizeOk) {
stateSizeChecked = true;
if (state.isSizeOk()) {
cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback);
if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) {
systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null);
}
}
} catch (Exception e) {
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).msgId(tbMsgId).msgType(tbMsgType).arguments(state.getArguments()).cause(e).build();
}
} else {
state.checkStateSize(ctxId, ctx.getMaxStateSize());
stateSizeOk = state.isSizeOk();
if (stateSizeOk) {
callback.onSuccess(); // State was updated but no calculation performed;
} catch (Exception e) {
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).msgId(tbMsgId).msgType(tbMsgType).arguments(state.getArguments()).cause(e).build();
} finally {
if (!stateSizeChecked) {
state.checkStateSize(ctxId, ctx.getMaxStateSize());
}
if (state.isSizeOk()) {
cfStateService.persistState(ctxId, state, callback);
} else {
removeStateAndRaiseSizeException(ctxId, CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(), callback);
}
}
if (stateSizeOk) {
cfStateService.persistState(ctxId, state, callback);
} else {
removeStateAndRaiseSizeException(ctxId, CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(), callback);
}
}
private void removeStateAndRaiseSizeException(CalculatedFieldEntityCtxId ctxId, CalculatedFieldException ex, TbCallback callback) {
private void removeStateAndRaiseSizeException(CalculatedFieldEntityCtxId ctxId, CalculatedFieldException ex, TbCallback callback) throws CalculatedFieldException {
// We remove the state, but remember that it is over-sized in a local map.
cfStateService.removeState(ctxId, new TbCallback() {
@Override
@ -316,6 +314,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
callback.onFailure(ex);
}
});
throw ex;
}
private Map<String, ArgumentEntry> mapToArguments(CalculatedFieldCtx ctx, List<TsKvProto> data) {

3
application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java

@ -16,7 +16,6 @@
package org.thingsboard.server.actors.calculatedField;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.DebugModeUtil;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.actors.TbActorCtx;
import org.thingsboard.server.actors.TbActorRef;
@ -51,8 +50,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CopyOnWriteArrayList;
import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto;

10
application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java

@ -28,8 +28,9 @@ import org.thingsboard.rule.engine.api.NotificationCenter;
import org.thingsboard.rule.engine.api.RuleEngineAlarmService;
import org.thingsboard.rule.engine.api.RuleEngineApiUsageStateService;
import org.thingsboard.rule.engine.api.RuleEngineAssetProfileCache;
import org.thingsboard.rule.engine.api.RuleEngineCalculatedFieldQueueService;
import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache;
import org.thingsboard.rule.engine.api.RuleEngineDeviceStateManager;
import org.thingsboard.rule.engine.api.DeviceStateManager;
import org.thingsboard.rule.engine.api.RuleEngineRpcService;
import org.thingsboard.rule.engine.api.RuleEngineTelemetryService;
import org.thingsboard.rule.engine.api.ScriptEngine;
@ -725,7 +726,7 @@ public class DefaultTbContext implements TbContext {
}
@Override
public RuleEngineDeviceStateManager getDeviceStateManager() {
public DeviceStateManager getDeviceStateManager() {
return mainCtx.getDeviceStateManager();
}
@ -902,6 +903,11 @@ public class DefaultTbContext implements TbContext {
return mainCtx.getCalculatedFieldService();
}
@Override
public RuleEngineCalculatedFieldQueueService getCalculatedFieldQueueService() {
return mainCtx.getCalculatedFieldQueueService();
}
@Override
public boolean isExternalNodeForceAck() {
return mainCtx.isExternalNodeForceAck();

21
application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java

@ -20,6 +20,8 @@ import io.swagger.v3.oas.annotations.media.Schema;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
@ -27,6 +29,7 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UserId;
@ -38,6 +41,8 @@ import org.thingsboard.server.common.data.query.EntityCountQuery;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataPageLink;
import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.msg.edqs.EdqsApiService;
import org.thingsboard.server.common.msg.edqs.EdqsService;
import org.thingsboard.server.config.annotations.ApiOperation;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.query.EntityQueryService;
@ -55,6 +60,10 @@ public class EntityQueryController extends BaseController {
@Autowired
private EntityQueryService entityQueryService;
@Autowired
private EdqsService edqsService;
@Autowired
private EdqsApiService edqsApiService;
private static final int MAX_PAGE_SIZE = 100;
@ -133,4 +142,16 @@ public class EntityQueryController extends BaseController {
return entityQueryService.getKeysByQuery(getCurrentUser(), tenantId, query, isTimeseries, isAttributes, scope);
}
@PreAuthorize("hasAnyAuthority('SYS_ADMIN')")
@PostMapping("/edqs/system/request")
public void processSystemEdqsRequest(@RequestBody ToCoreEdqsRequest request) {
edqsService.processSystemRequest(request);
}
@PreAuthorize("hasAnyAuthority('SYS_ADMIN')")
@GetMapping("/edqs/enabled")
public boolean isEdqsApiEnabled() {
return edqsApiService.isEnabled();
}
}

3
application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java

@ -18,13 +18,14 @@ package org.thingsboard.server.service.cf;
import com.google.common.util.concurrent.FutureCallback;
import org.thingsboard.rule.engine.api.AttributesDeleteRequest;
import org.thingsboard.rule.engine.api.AttributesSaveRequest;
import org.thingsboard.rule.engine.api.RuleEngineCalculatedFieldQueueService;
import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest;
import org.thingsboard.rule.engine.api.TimeseriesSaveRequest;
import org.thingsboard.server.common.data.kv.TimeseriesSaveResult;
import java.util.List;
public interface CalculatedFieldQueueService {
public interface CalculatedFieldQueueService extends RuleEngineCalculatedFieldQueueService {
/**
* Filter CFs based on the request entity. Push to the queue if any matching CF exist;

11
application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java

@ -15,22 +15,29 @@
*/
package org.thingsboard.server.service.cf;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import org.rocksdb.Options;
import org.rocksdb.WriteOptions;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Component;
import org.thingsboard.server.utils.TbRocksDb;
import org.thingsboard.server.edqs.util.TbRocksDb;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='in-memory'")
public class CfRocksDb extends TbRocksDb {
public CfRocksDb(@Value("${queue.calculated_fields.rocks_db_path:${user.home}/.rocksdb/cf_states}") String path) throws Exception {
public CfRocksDb(@Value("${queue.calculated_fields.rocks_db_path:${user.home}/.rocksdb/cf_states}") String path) {
super(path, new Options().setCreateIfMissing(true), new WriteOptions().setSync(true));
}
@PostConstruct
@Override
public void init() {
super.init();
}
@PreDestroy
@Override
public void close() {

1
application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java

@ -117,7 +117,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache {
CalculatedField calculatedField = getCalculatedField(calculatedFieldId);
if (calculatedField != null) {
ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService, apiLimitService);
ctx.init();
calculatedFieldsCtx.put(calculatedFieldId, ctx);
log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx);
}

2
application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java

@ -125,7 +125,7 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP
}
return Futures.whenAllComplete(argFutures.values()).call(() -> {
var result = createStateByType(ctx);
result.updateState(argFutures.entrySet().stream()
result.updateState(ctx, argFutures.entrySet().stream()
.collect(Collectors.toMap(
Entry::getKey, // Keep the key as is
entry -> {

33
application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java

@ -47,6 +47,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import org.thingsboard.server.service.profile.TbAssetProfileCache;
import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
@ -89,6 +90,11 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS
() -> toCalculatedFieldTelemetryMsgProto(request, result), callback);
}
@Override
public void pushRequestToQueue(TimeseriesSaveRequest request, FutureCallback<Void> callback) {
pushRequestToQueue(request, null, callback);
}
@Override
public void pushRequestToQueue(AttributesSaveRequest request, List<Long> result, FutureCallback<Void> callback) {
var tenantId = request.getTenantId();
@ -97,6 +103,11 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS
() -> toCalculatedFieldTelemetryMsgProto(request, result), callback);
}
@Override
public void pushRequestToQueue(AttributesSaveRequest request, FutureCallback<Void> callback) {
pushRequestToQueue(request, null, callback);
}
@Override
public void pushRequestToQueue(AttributesDeleteRequest request, List<String> result, FutureCallback<Void> callback) {
var tenantId = request.getTenantId();
@ -156,15 +167,19 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS
ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder();
CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType());
List<TsKvEntry> entries = request.getEntries();
List<Long> versions = result.getVersions();
List<Long> versions = result != null ? result.getVersions() : Collections.emptyList();
for (int i = 0; i < entries.size(); i++) {
long tsVersion = versions.get(i);
TsKvProto tsProto = toTsKvProto(entries.get(i)).toBuilder().setVersion(tsVersion).build();
telemetryMsg.addTsData(tsProto);
TsKvProto.Builder tsProtoBuilder = toTsKvProto(entries.get(i)).toBuilder();
if (result != null) {
tsProtoBuilder.setVersion(versions.get(i));
}
telemetryMsg.addTsData(tsProtoBuilder.build());
}
msg.setTelemetryMsg(telemetryMsg.build());
msg.setTelemetryMsg(telemetryMsg.build());
return msg.build();
}
@ -175,9 +190,11 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS
telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name()));
List<AttributeKvEntry> entries = request.getEntries();
for (int i = 0; i < entries.size(); i++) {
long attrVersion = versions.get(i);
AttributeValueProto attrProto = ProtoUtils.toProto(entries.get(i)).toBuilder().setVersion(attrVersion).build();
telemetryMsg.addAttrData(attrProto);
AttributeValueProto.Builder attrProtoBuilder = ProtoUtils.toProto(entries.get(i)).toBuilder();
if (versions != null) {
attrProtoBuilder.setVersion(versions.get(i));
}
telemetryMsg.addAttrData(attrProtoBuilder.build());
}
msg.setTelemetryMsg(telemetryMsg.build());

19
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java

@ -25,6 +25,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArgumentProto;
@Data
@AllArgsConstructor
public abstract class BaseCalculatedFieldState implements CalculatedFieldState {
@ -43,7 +45,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState {
}
@Override
public boolean updateState(Map<String, ArgumentEntry> argumentValues) {
public boolean updateState(CalculatedFieldCtx ctx, Map<String, ArgumentEntry> argumentValues) {
if (arguments == null) {
arguments = new HashMap<>();
}
@ -53,6 +55,9 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState {
for (Map.Entry<String, ArgumentEntry> entry : argumentValues.entrySet()) {
String key = entry.getKey();
ArgumentEntry newEntry = entry.getValue();
checkArgumentSize(key, newEntry, ctx);
ArgumentEntry existingEntry = arguments.get(key);
if (existingEntry == null || newEntry.isForceResetPrevious()) {
@ -81,6 +86,18 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState {
}
}
@Override
public void checkArgumentSize(String name, ArgumentEntry entry, CalculatedFieldCtx ctx) {
if (entry instanceof TsRollingArgumentEntry) {
return;
}
if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) {
if (ctx.getMaxSingleValueArgumentSize() > 0 && toSingleValueArgumentProto(name, singleValueArgumentEntry).getSerializedSize() > ctx.getMaxSingleValueArgumentSize()) {
throw new IllegalArgumentException("Single value size exceeds the maximum allowed limit. The argument will not be used for calculation.");
}
}
}
protected abstract void validateNewEntry(ArgumentEntry newEntry);
}

2
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java

@ -71,6 +71,7 @@ public class CalculatedFieldCtx {
private long maxDataPointsPerRollingArg;
private long maxStateSize;
private long maxSingleValueArgumentSize;
public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService, ApiLimitService apiLimitService) {
this.calculatedField = calculatedField;
@ -104,6 +105,7 @@ public class CalculatedFieldCtx {
this.maxDataPointsPerRollingArg = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg);
this.maxStateSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes) * 1024;
this.maxSingleValueArgumentSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxSingleValueArgumentSizeInKBytes) * 1024;
}
public void init() {

4
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java

@ -44,7 +44,7 @@ public interface CalculatedFieldState {
void setRequiredArguments(List<String> requiredArguments);
boolean updateState(Map<String, ArgumentEntry> argumentValues);
boolean updateState(CalculatedFieldCtx ctx, Map<String, ArgumentEntry> argumentValues);
ListenableFuture<CalculatedFieldResult> performCalculation(CalculatedFieldCtx ctx);
@ -60,4 +60,6 @@ public interface CalculatedFieldState {
void checkStateSize(CalculatedFieldEntityCtxId ctxId, long maxStateSize);
void checkArgumentSize(String name, ArgumentEntry entry, CalculatedFieldCtx ctx);
}

15
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java

@ -20,6 +20,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.script.api.tbel.TbUtils;
import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Output;
import org.thingsboard.server.common.data.kv.BasicKvEntry;
@ -64,7 +65,19 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState {
double expressionResult = expr.evaluate();
Output output = ctx.getOutput();
return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), JacksonUtil.valueToTree(Map.of(output.getName(), expressionResult))));
Object result;
Integer decimals = output.getDecimalsByDefault();
if (decimals != null) {
if (decimals.equals(0)) {
result = TbUtils.toInt(expressionResult);
} else {
result = TbUtils.toFixed(expressionResult, decimals);
}
} else {
result = expressionResult;
}
return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), JacksonUtil.valueToTree(Map.of(output.getName(), result))));
}
}

12
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java

@ -42,13 +42,17 @@ public class SingleValueArgumentEntry implements ArgumentEntry {
public SingleValueArgumentEntry(TsKvProto entry) {
this.ts = entry.getTs();
this.version = entry.getVersion();
if (entry.hasVersion()) {
this.version = entry.getVersion();
}
this.kvEntryValue = ProtoUtils.fromProto(entry.getKv());
}
public SingleValueArgumentEntry(AttributeValueProto entry) {
this.ts = entry.getLastUpdateTs();
this.version = entry.getVersion();
if (entry.hasVersion()) {
this.version = entry.getVersion();
}
this.kvEntryValue = ProtoUtils.basicKvEntryFromProto(entry);
}
@ -100,10 +104,6 @@ public class SingleValueArgumentEntry implements ArgumentEntry {
if (newVersion == null || this.version == null || newVersion > this.version) {
this.ts = singleValueEntry.getTs();
this.version = newVersion;
BasicKvEntry newValue = singleValueEntry.getKvEntryValue();
if (this.kvEntryValue != null && this.kvEntryValue.getValue().equals(newValue.getValue())) {
return false;
}
this.kvEntryValue = singleValueEntry.getKvEntryValue();
return true;
}

2
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java

@ -89,7 +89,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry {
for (var e : tsRecords.entrySet()) {
values.add(new TbelCfTsDoubleVal(e.getKey(), e.getValue()));
}
return new TbelCfTsRollingArg(limit, timeWindow, values);
return new TbelCfTsRollingArg(timeWindow, values);
}
@Override

5
application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java

@ -82,6 +82,11 @@ public class EdgeEventSourcingListener {
@TransactionalEventListener(fallbackExecution = true)
public void handleEvent(SaveEntityEvent<?> event) {
if (Boolean.FALSE.equals(event.getBroadcastEvent())) {
log.trace("Ignoring event {}", event);
return;
}
try {
if (!isValidSaveEntityEventForEdgeProcessing(event)) {
return;

12
application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java

@ -74,6 +74,8 @@ import org.thingsboard.server.gen.edge.v1.RequestMsgType;
import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg;
import org.thingsboard.server.gen.edge.v1.ResponseMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataRequestMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg;
import org.thingsboard.server.gen.edge.v1.SyncCompletedMsg;
import org.thingsboard.server.gen.edge.v1.UplinkMsg;
import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg;
@ -820,6 +822,16 @@ public abstract class EdgeGrpcSession implements Closeable {
result.add(ctx.getAssetProcessor().processAssetMsgFromEdge(edge.getTenantId(), edge, assetUpdateMsg));
}
}
if (uplinkMsg.getRuleChainUpdateMsgCount() > 0) {
for (RuleChainUpdateMsg ruleChainUpdateMsg : uplinkMsg.getRuleChainUpdateMsgList()) {
result.add(ctx.getRuleChainProcessor().processRuleChainMsgFromEdge(edge.getTenantId(), edge, ruleChainUpdateMsg));
}
}
if (uplinkMsg.getRuleChainMetadataUpdateMsgCount() > 0) {
for (RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg : uplinkMsg.getRuleChainMetadataUpdateMsgList()) {
result.add(ctx.getRuleChainProcessor().processRuleChainMetadataMsgFromEdge(edge.getTenantId(), edge, ruleChainMetadataUpdateMsg));
}
}
if (uplinkMsg.getEntityViewUpdateMsgCount() > 0) {
for (EntityViewUpdateMsg entityViewUpdateMsg : uplinkMsg.getEntityViewUpdateMsgList()) {
result.add(ctx.getEntityViewProcessor().processEntityViewMsgFromEdge(edge.getTenantId(), edge, entityViewUpdateMsg));

4
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeEntityProcessor.java

@ -49,8 +49,12 @@ public class EdgeEntityProcessor extends BaseEdgeProcessor {
@Override
public ListenableFuture<Void> processEntityNotification(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto edgeNotificationMsg) {
try {
EdgeId originatorEdgeId = safeGetEdgeId(edgeNotificationMsg.getOriginatorEdgeIdMSB(), edgeNotificationMsg.getOriginatorEdgeIdLSB());
EdgeEventActionType actionType = EdgeEventActionType.valueOf(edgeNotificationMsg.getAction());
EdgeId edgeId = new EdgeId(new UUID(edgeNotificationMsg.getEntityIdMSB(), edgeNotificationMsg.getEntityIdLSB()));
if (edgeId.equals(originatorEdgeId)) {
return Futures.immediateFuture(null);
}
switch (actionType) {
case ASSIGNED_TO_CUSTOMER: {
CustomerId customerId = JacksonUtil.fromString(edgeNotificationMsg.getBody(), CustomerId.class);

82
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/BaseRuleChainProcessor.java

@ -0,0 +1,82 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.edge.rpc.processor.rule;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.util.Pair;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.dao.service.DataValidator;
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg;
import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor;
import java.util.function.Function;
@Slf4j
public class BaseRuleChainProcessor extends BaseEdgeProcessor {
@Autowired
private DataValidator<RuleChain> ruleChainValidator;
protected Pair<Boolean, Boolean> saveOrUpdateRuleChain(TenantId tenantId, RuleChainId ruleChainId, RuleChainUpdateMsg ruleChainUpdateMsg, RuleChainType ruleChainType) {
boolean created = false;
RuleChain ruleChainFromDb = edgeCtx.getRuleChainService().findRuleChainById(tenantId, ruleChainId);
if (ruleChainFromDb == null) {
created = true;
}
RuleChain ruleChain = JacksonUtil.fromString(ruleChainUpdateMsg.getEntity(), RuleChain.class, true);
if (ruleChain == null) {
throw new RuntimeException("[{" + tenantId + "}] ruleChainUpdateMsg {" + ruleChainUpdateMsg + "} cannot be converted to rule chain");
}
boolean isRoot = ruleChain.isRoot();
if (RuleChainType.CORE.equals(ruleChainType)) {
ruleChain.setRoot(false);
} else {
ruleChain.setRoot(ruleChainFromDb == null ? false : ruleChainFromDb.isRoot());
}
ruleChain.setType(ruleChainType);
ruleChainValidator.validate(ruleChain, RuleChain::getTenantId);
if (created) {
ruleChain.setId(ruleChainId);
}
edgeCtx.getRuleChainService().saveRuleChain(ruleChain, true, false);
return Pair.of(created, isRoot);
}
protected void saveOrUpdateRuleChainMetadata(TenantId tenantId, RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg) {
RuleChainMetaData ruleChainMetadata = JacksonUtil.fromString(ruleChainMetadataUpdateMsg.getEntity(), RuleChainMetaData.class, true);
if (ruleChainMetadata == null) {
throw new RuntimeException("[{" + tenantId + "}] ruleChainMetadataUpdateMsg {" + ruleChainMetadataUpdateMsg + "} cannot be converted to rule chain metadata");
}
if (!ruleChainMetadata.getNodes().isEmpty()) {
ruleChainMetadata.setVersion(null);
for (RuleNode ruleNode : ruleChainMetadata.getNodes()) {
ruleNode.setRuleChainId(null);
ruleNode.setId(null);
}
edgeCtx.getRuleChainService().saveRuleChainMetaData(tenantId, ruleChainMetadata, Function.identity(), true);
}
}
}

98
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java

@ -15,29 +15,123 @@
*/
package org.thingsboard.server.service.edge.rpc.processor.rule;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.util.Pair;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.EdgeUtils;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.edge.EdgeEvent;
import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg;
import org.thingsboard.server.gen.edge.v1.UpdateMsgType;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils;
import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor;
import java.util.UUID;
import static org.thingsboard.server.dao.edge.EdgeServiceImpl.EDGE_IS_ROOT_BODY_KEY;
@Slf4j
@Component
@TbCoreComponent
public class RuleChainEdgeProcessor extends BaseEdgeProcessor {
public class RuleChainEdgeProcessor extends BaseRuleChainProcessor {
public ListenableFuture<Void> processRuleChainMsgFromEdge(TenantId tenantId, Edge edge, RuleChainUpdateMsg ruleChainUpdateMsg) {
log.trace("[{}] executing processRuleChainMsgFromEdge [{}] from edge [{}]", tenantId, ruleChainUpdateMsg, edge.getName());
RuleChainId ruleChainId = new RuleChainId(new UUID(ruleChainUpdateMsg.getIdMSB(), ruleChainUpdateMsg.getIdLSB()));
try {
edgeSynchronizationManager.getEdgeId().set(edge.getId());
switch (ruleChainUpdateMsg.getMsgType()) {
case ENTITY_CREATED_RPC_MESSAGE:
case ENTITY_UPDATED_RPC_MESSAGE:
return saveOrUpdateRuleChain(tenantId, ruleChainId, ruleChainUpdateMsg, edge);
case ENTITY_DELETED_RPC_MESSAGE:
RuleChain ruleChainToDelete = edgeCtx.getRuleChainService().findRuleChainById(tenantId, ruleChainId);
if (ruleChainToDelete != null) {
edgeCtx.getRuleChainService().unassignRuleChainFromEdge(tenantId, ruleChainId, edge.getId(), false);
}
return Futures.immediateFuture(null);
case UNRECOGNIZED:
default:
return handleUnsupportedMsgType(ruleChainUpdateMsg.getMsgType());
}
} catch (DataValidationException e) {
if (e.getMessage().contains("limit reached")) {
log.warn("[{}] Number of allowed rule chains violated {}", tenantId, ruleChainUpdateMsg, e);
return Futures.immediateFuture(null);
} else {
return Futures.immediateFailedFuture(e);
}
} finally {
edgeSynchronizationManager.getEdgeId().remove();
}
}
private ListenableFuture<Void> saveOrUpdateRuleChain(TenantId tenantId, RuleChainId ruleChainId, RuleChainUpdateMsg ruleChainUpdateMsg, Edge edge) {
try {
Pair<Boolean, Boolean> resultPair = super.saveOrUpdateRuleChain(tenantId, ruleChainId, ruleChainUpdateMsg, RuleChainType.EDGE);
Boolean created = resultPair.getFirst();
if (created) {
createRelationFromEdge(tenantId, edge.getId(), ruleChainId);
pushRuleChainCreatedEventToRuleEngine(tenantId, edge, ruleChainId, ruleChainUpdateMsg.getEntity());
edgeCtx.getRuleChainService().assignRuleChainToEdge(tenantId, ruleChainId, edge.getId());
}
Boolean isRoot = resultPair.getSecond();
if (isRoot) {
edge = edgeCtx.getEdgeService().findEdgeById(tenantId, edge.getId());
edgeCtx.getEdgeService().setEdgeRootRuleChain(tenantId, edge, ruleChainId);
}
} catch (Exception e) {
log.error("Failed to save or update rule chain", e);
return Futures.immediateFailedFuture(e);
}
return Futures.immediateFuture(null);
}
private void pushRuleChainCreatedEventToRuleEngine(TenantId tenantId, Edge edge, RuleChainId ruleChainId, String ruleChainAsString) {
try {
TbMsgMetaData msgMetaData = getEdgeActionTbMsgMetaData(edge, null);
pushEntityEventToRuleEngine(tenantId, ruleChainId, null, TbMsgType.ENTITY_CREATED, ruleChainAsString, msgMetaData);
} catch (Exception e) {
log.warn("[{}][{}] Failed to push rule chain action to rule engine: {}", tenantId, ruleChainId, TbMsgType.ENTITY_CREATED.name(), e);
}
}
public ListenableFuture<Void> processRuleChainMetadataMsgFromEdge(TenantId tenantId, Edge edge, RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg) {
log.trace("[{}] executing processRuleChainMetadataMsgFromEdge [{}] from edge [{}]", tenantId, ruleChainMetadataUpdateMsg, edge.getName());
try {
edgeSynchronizationManager.getEdgeId().set(edge.getId());
switch (ruleChainMetadataUpdateMsg.getMsgType()) {
case ENTITY_CREATED_RPC_MESSAGE:
case ENTITY_UPDATED_RPC_MESSAGE:
saveOrUpdateRuleChainMetadata(tenantId, ruleChainMetadataUpdateMsg);
return Futures.immediateFuture(null);
case UNRECOGNIZED:
default:
return handleUnsupportedMsgType(ruleChainMetadataUpdateMsg.getMsgType());
}
} catch (Exception e) {
String errMsg = String.format("Can't process rule chain metadata update msg %s", ruleChainMetadataUpdateMsg);
log.error(errMsg, e);
return Futures.immediateFailedFuture(new RuntimeException(errMsg, e));
} finally {
edgeSynchronizationManager.getEdgeId().remove();
}
}
@Override
public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) {

5
application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java

@ -152,8 +152,7 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService {
entityData = new HashMap<>();
attributes = JacksonUtil.newObjectNode();
for (AttributeKvEntry attr : ssAttributes) {
if (DefaultDeviceStateService.PERSISTENT_ATTRIBUTES.contains(attr.getKey())
&& !DefaultDeviceStateService.INACTIVITY_TIMEOUT.equals(attr.getKey())) {
if (DefaultDeviceStateService.ACTIVITY_KEYS_WITHOUT_INACTIVITY_TIMEOUT.contains(attr.getKey())) {
continue;
}
if (attr.getDataType() == DataType.BOOLEAN && attr.getBooleanValue().isPresent()) {
@ -200,7 +199,7 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService {
}
Map<Long, Map<String, Object>> tsData = new HashMap<>();
for (TsKvEntry tsKvEntry : tsKvEntries) {
if (DefaultDeviceStateService.PERSISTENT_ATTRIBUTES.contains(tsKvEntry.getKey())) {
if (DefaultDeviceStateService.ACTIVITY_KEYS_WITH_INACTIVITY_TIMEOUT.contains(tsKvEntry.getKey())) {
continue;
}
tsData.computeIfAbsent(tsKvEntry.getTs(), k -> new HashMap<>()).put(tsKvEntry.getKey(), tsKvEntry.getValue());

117
application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java

@ -0,0 +1,117 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.edqs;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.edqs.query.EdqsRequest;
import org.thingsboard.server.common.data.edqs.query.EdqsResponse;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.edqs.EdqsApiService;
import org.thingsboard.server.edqs.state.EdqsPartitionService;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg;
import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.provider.EdqsClientQueueFactory;
import java.util.UUID;
@Service
@Slf4j
@RequiredArgsConstructor
@ConditionalOnExpression("'${queue.edqs.api.supported:true}' == 'true' && ('${service.type:null}' == 'monolith' || '${service.type:null}' == 'tb-core')")
public class DefaultEdqsApiService implements EdqsApiService {
private final EdqsPartitionService edqsPartitionService;
private final EdqsClientQueueFactory queueFactory;
private TbQueueRequestTemplate<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> requestTemplate;
@Value("${queue.edqs.api.auto_enable:true}")
private boolean autoEnable;
private Boolean apiEnabled = null;
@PostConstruct
private void init() {
requestTemplate = queueFactory.createEdqsRequestTemplate();
requestTemplate.init();
}
@Override
public ListenableFuture<EdqsResponse> processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) {
var requestMsg = ToEdqsMsg.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setTs(System.currentTimeMillis())
.setRequestMsg(TransportProtos.EdqsRequestMsg.newBuilder()
.setValue(JacksonUtil.toString(request))
.build());
if (customerId != null && !customerId.isNullUid()) {
requestMsg.setCustomerIdMSB(customerId.getId().getMostSignificantBits());
requestMsg.setCustomerIdLSB(customerId.getId().getLeastSignificantBits());
}
Integer partition = edqsPartitionService.resolvePartition(tenantId);
ListenableFuture<TbProtoQueueMsg<FromEdqsMsg>> resultFuture = requestTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), requestMsg.build()), partition);
return Futures.transform(resultFuture, msg -> {
TransportProtos.EdqsResponseMsg responseMsg = msg.getValue().getResponseMsg();
return JacksonUtil.fromString(responseMsg.getValue(), EdqsResponse.class);
}, MoreExecutors.directExecutor());
}
@Override
public boolean isEnabled() {
return Boolean.TRUE.equals(apiEnabled);
}
@Override
public void setEnabled(boolean enabled) {
if (enabled) {
log.info("Enabling EDQS API");
} else {
log.info("Disabling EDQS API");
}
apiEnabled = enabled;
}
@Override
public boolean isSupported() {
return true;
}
@Override
public boolean isAutoEnable() {
return autoEnable;
}
@PreDestroy
private void stop() {
requestTemplate.stop();
}
}

298
application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java

@ -0,0 +1,298 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.edqs;
import com.google.protobuf.ByteString;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ObjectType;
import org.thingsboard.server.common.data.edqs.EdqsEventType;
import org.thingsboard.server.common.data.edqs.EdqsObject;
import org.thingsboard.server.common.data.edqs.EdqsSyncRequest;
import org.thingsboard.server.common.data.edqs.Entity;
import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg;
import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
import org.thingsboard.server.common.data.kv.JsonDataEntry;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.msg.edqs.EdqsApiService;
import org.thingsboard.server.common.msg.edqs.EdqsService;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.edqs.processor.EdqsProducer;
import org.thingsboard.server.edqs.state.EdqsPartitionService;
import org.thingsboard.server.edqs.util.EdqsConverter;
import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsCoreServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg;
import org.thingsboard.server.queue.discovery.HashPartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.discovery.TopicService;
import org.thingsboard.server.queue.edqs.EdqsQueue;
import org.thingsboard.server.queue.environment.DistributedLock;
import org.thingsboard.server.queue.environment.DistributedLockService;
import org.thingsboard.server.queue.provider.EdqsClientQueueFactory;
import org.thingsboard.server.queue.util.AfterStartUp;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
@Service
@RequiredArgsConstructor
@Slf4j
@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "true")
public class DefaultEdqsService implements EdqsService {
private final EdqsClientQueueFactory queueFactory;
private final EdqsConverter edqsConverter;
private final EdqsSyncService edqsSyncService;
private final EdqsApiService edqsApiService;
private final DistributedLockService distributedLockService;
private final AttributesService attributesService;
private final EdqsPartitionService edqsPartitionService;
private final TopicService topicService;
private final TbServiceInfoProvider serviceInfoProvider;
@Autowired @Lazy
private TbClusterService clusterService;
@Autowired @Lazy
private HashPartitionService hashPartitionService;
private EdqsProducer eventsProducer;
private ExecutorService executor;
private DistributedLock syncLock;
@PostConstruct
private void init() {
executor = ThingsBoardExecutors.newWorkStealingPool(12, getClass());
eventsProducer = EdqsProducer.builder()
.queue(EdqsQueue.EVENTS)
.partitionService(edqsPartitionService)
.topicService(topicService)
.producer(queueFactory.createEdqsMsgProducer(EdqsQueue.EVENTS))
.build();
syncLock = distributedLockService.getLock("edqs_sync");
}
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
public void onStartUp() {
if (!serviceInfoProvider.isService(ServiceType.TB_CORE)) {
return;
}
executor.submit(() -> {
try {
EdqsSyncState syncState = getSyncState();
if (edqsSyncService.isSyncNeeded() || syncState == null || syncState.getStatus() != EdqsSyncStatus.FINISHED) {
if (hashPartitionService.isSystemPartitionMine(ServiceType.TB_CORE)) {
processSystemRequest(ToCoreEdqsRequest.builder()
.syncRequest(new EdqsSyncRequest())
.build());
}
} else if (edqsApiService.isSupported() && edqsApiService.isAutoEnable()) {
// only if topic/RocksDB is not empty and sync is finished
edqsApiService.setEnabled(true);
}
} catch (Throwable e) {
log.error("Failed to start EDQS service", e);
}
});
}
@Override
public void processSystemRequest(ToCoreEdqsRequest request) {
log.info("Processing system request {}", request);
if (request.getSyncRequest() != null) {
saveSyncState(EdqsSyncStatus.REQUESTED);
}
broadcast(request.toInternalMsg());
}
@Override
public void processSystemMsg(ToCoreEdqsMsg msg) {
executor.submit(() -> {
log.info("Processing system msg {}", msg);
try {
if (msg.getApiEnabled() != null) {
edqsApiService.setEnabled(msg.getApiEnabled());
}
if (msg.getSyncRequest() != null) {
syncLock.lock();
try {
EdqsSyncState syncState = getSyncState();
if (syncState != null && syncState.getStatus() == EdqsSyncStatus.FINISHED) {
log.info("EDQS sync is already finished");
return;
}
saveSyncState(EdqsSyncStatus.STARTED);
edqsSyncService.sync();
saveSyncState(EdqsSyncStatus.FINISHED);
if (edqsApiService.isSupported())
if (edqsApiService.isAutoEnable()) {
log.info("EDQS sync is finished, auto-enabling API");
broadcast(ToCoreEdqsMsg.builder()
.apiEnabled(Boolean.TRUE)
.build());
} else {
log.info("EDQS sync is finished, but leaving API disabled");
}
} catch (Exception e) {
log.error("Failed to complete sync", e);
saveSyncState(EdqsSyncStatus.FAILED);
} finally {
syncLock.unlock();
}
}
} catch (Throwable e) {
log.error("Failed to process msg {}", msg, e);
}
});
}
@Override
public void onUpdate(TenantId tenantId, EntityId entityId, Object entity) {
EntityType entityType = entityId.getEntityType();
ObjectType objectType = ObjectType.fromEntityType(entityType);
if (!isEdqsType(tenantId, objectType)) {
log.trace("[{}][{}] Ignoring update event, type {} not supported", tenantId, entityId, entityType);
return;
}
onUpdate(tenantId, objectType, edqsConverter.toEntity(entityType, entity));
}
@Override
public void onUpdate(TenantId tenantId, ObjectType objectType, EdqsObject object) {
processEvent(tenantId, objectType, EdqsEventType.UPDATED, object);
}
@Override
public void onDelete(TenantId tenantId, EntityId entityId) {
EntityType entityType = entityId.getEntityType();
ObjectType objectType = ObjectType.fromEntityType(entityType);
if (!isEdqsType(tenantId, objectType)) {
log.trace("[{}][{}] Ignoring deletion event, type {} not supported", tenantId, entityId, entityType);
return;
}
onDelete(tenantId, objectType, new Entity(entityType, entityId.getId(), Long.MAX_VALUE));
}
@Override
public void onDelete(TenantId tenantId, ObjectType objectType, EdqsObject object) {
processEvent(tenantId, objectType, EdqsEventType.DELETED, object);
}
protected void processEvent(TenantId tenantId, ObjectType objectType, EdqsEventType eventType, EdqsObject object) {
executor.submit(() -> {
try {
String key = object.key();
Long version = object.version();
EdqsEventMsg.Builder eventMsg = EdqsEventMsg.newBuilder()
.setKey(key)
.setObjectType(objectType.name())
.setData(ByteString.copyFrom(edqsConverter.serialize(objectType, object)))
.setEventType(eventType.name());
if (version != null) {
eventMsg.setVersion(version);
}
eventsProducer.send(tenantId, objectType, key, ToEdqsMsg.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setTs(System.currentTimeMillis())
.setEventMsg(eventMsg)
.build());
} catch (Throwable e) {
log.error("[{}] Failed to push {} event for {} {}", tenantId, eventType, objectType, object, e);
}
});
}
private boolean isEdqsType(TenantId tenantId, ObjectType objectType) {
if (objectType == null) {
return false;
}
if (!tenantId.isSysTenantId()) {
return ObjectType.edqsTypes.contains(objectType);
} else {
return ObjectType.edqsSystemTypes.contains(objectType);
}
}
private void broadcast(ToCoreEdqsMsg msg) {
clusterService.broadcastToCore(ToCoreNotificationMsg.newBuilder()
.setToEdqsCoreServiceMsg(ToEdqsCoreServiceMsg.newBuilder()
.setValue(ByteString.copyFrom(JacksonUtil.writeValueAsBytes(msg))))
.build());
}
@SneakyThrows
private EdqsSyncState getSyncState() {
EdqsSyncState state = attributesService.find(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, AttributeScope.SERVER_SCOPE, "edqsSyncState").get(30, TimeUnit.SECONDS)
.flatMap(KvEntry::getJsonValue)
.map(value -> JacksonUtil.fromString(value, EdqsSyncState.class))
.orElse(null);
log.info("EDQS sync state: {}", state);
return state;
}
@SneakyThrows
private void saveSyncState(EdqsSyncStatus status) {
EdqsSyncState state = new EdqsSyncState(status);
log.info("New EDQS sync state: {}", state);
attributesService.save(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, AttributeScope.SERVER_SCOPE, new BaseAttributeKvEntry(
new JsonDataEntry("edqsSyncState", JacksonUtil.toString(state)),
System.currentTimeMillis())).get(30, TimeUnit.SECONDS);
}
@PreDestroy
private void stop() {
executor.shutdown();
eventsProducer.stop();
}
@Data
@AllArgsConstructor
@NoArgsConstructor
private static class EdqsSyncState {
private EdqsSyncStatus status;
}
private enum EdqsSyncStatus {
REQUESTED,
STARTED,
FINISHED,
FAILED
}
}

61
application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java

@ -0,0 +1,61 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.edqs;
import lombok.RequiredArgsConstructor;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import org.springframework.transaction.event.TransactionalEventListener;
import org.thingsboard.server.common.data.ObjectType;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.msg.edqs.EdqsService;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.RelationActionEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "true")
public class EdqsListener {
private final EdqsService edqsService;
@TransactionalEventListener(fallbackExecution = true)
public void onUpdate(SaveEntityEvent<?> event) {
if (event.getEntityId() == null || event.getEntity() == null) {
return;
}
edqsService.onUpdate(event.getTenantId(), event.getEntityId(), event.getEntity());
}
@TransactionalEventListener(fallbackExecution = true)
public void onDelete(DeleteEntityEvent<?> event) {
if (event.getEntityId() == null) {
return;
}
edqsService.onDelete(event.getTenantId(), event.getEntityId());
}
@TransactionalEventListener(fallbackExecution = true)
public void handleEvent(RelationActionEvent relationEvent) {
if (relationEvent.getActionType() == ActionType.RELATION_ADD_OR_UPDATE) {
edqsService.onUpdate(relationEvent.getTenantId(), ObjectType.RELATION, relationEvent.getRelation());
} else if (relationEvent.getActionType() == ActionType.RELATION_DELETED) {
edqsService.onDelete(relationEvent.getTenantId(), ObjectType.RELATION, relationEvent.getRelation());
}
}
}

284
application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java

@ -0,0 +1,284 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.edqs;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ObjectType;
import org.thingsboard.server.common.data.edqs.AttributeKv;
import org.thingsboard.server.common.data.edqs.EdqsEventType;
import org.thingsboard.server.common.data.edqs.EdqsObject;
import org.thingsboard.server.common.data.edqs.Entity;
import org.thingsboard.server.common.data.edqs.LatestTsKv;
import org.thingsboard.server.common.data.edqs.fields.EntityFields;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageDataIterable;
import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.dao.Dao;
import org.thingsboard.server.dao.attributes.AttributesDao;
import org.thingsboard.server.dao.dictionary.KeyDictionaryDao;
import org.thingsboard.server.dao.entity.EntityDaoRegistry;
import org.thingsboard.server.dao.model.sql.AttributeKvEntity;
import org.thingsboard.server.dao.model.sql.RelationEntity;
import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry;
import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity;
import org.thingsboard.server.dao.sql.relation.RelationRepository;
import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV;
import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV;
import static org.thingsboard.server.common.data.ObjectType.RELATION;
import static org.thingsboard.server.common.data.ObjectType.edqsTenantTypes;
@Slf4j
public abstract class EdqsSyncService {
@Value("${queue.edqs.sync.entity_batch_size:10000}")
private int entityBatchSize;
@Value("${queue.edqs.sync.ts_batch_size:10000}")
private int tsBatchSize;
@Autowired
private EntityDaoRegistry entityDaoRegistry;
@Autowired
private AttributesDao attributesDao;
@Autowired
private KeyDictionaryDao keyDictionaryDao;
@Autowired
private RelationRepository relationRepository;
@Autowired
private TsKvLatestRepository tsKvLatestRepository;
@Autowired
@Lazy
private DefaultEdqsService edqsService;
private final ConcurrentHashMap<UUID, EntityIdInfo> entityInfoMap = new ConcurrentHashMap<>();
private final ConcurrentHashMap<Integer, String> keys = new ConcurrentHashMap<>();
private final Map<ObjectType, AtomicInteger> counters = new ConcurrentHashMap<>();
public abstract boolean isSyncNeeded();
public void sync() {
log.info("Synchronizing data to EDQS");
long startTs = System.currentTimeMillis();
counters.clear();
syncTenantEntities();
syncRelations();
loadKeyDictionary();
syncAttributes();
syncLatestTimeseries();
counters.clear();
log.info("Finishing synchronizing data to EDQS in {} ms", (System.currentTimeMillis() - startTs));
}
private void process(TenantId tenantId, ObjectType type, EdqsObject object) {
AtomicInteger counter = counters.computeIfAbsent(type, t -> new AtomicInteger());
if (counter.incrementAndGet() % 10000 == 0) {
log.info("Processed {} {} objects", counter.get(), type);
}
edqsService.processEvent(tenantId, type, EdqsEventType.UPDATED, object);
}
private void syncTenantEntities() {
for (ObjectType type : edqsTenantTypes) {
log.info("Synchronizing {} entities to EDQS", type);
long ts = System.currentTimeMillis();
EntityType entityType = type.toEntityType();
Dao<?> dao = entityDaoRegistry.getDao(entityType);
UUID lastId = UUID.fromString("00000000-0000-0000-0000-000000000000");
while (true) {
var batch = dao.findNextBatch(lastId, entityBatchSize);
if (batch.isEmpty()) {
break;
}
for (EntityFields entityFields : batch) {
TenantId tenantId = TenantId.fromUUID(entityFields.getTenantId());
entityInfoMap.put(entityFields.getId(), new EntityIdInfo(entityType, tenantId));
process(tenantId, type, new Entity(entityType, entityFields));
}
EntityFields lastRecord = batch.get(batch.size() - 1);
lastId = lastRecord.getId();
}
log.info("Finished synchronizing {} entities to EDQS in {} ms", type, (System.currentTimeMillis() - ts));
}
}
private void syncRelations() {
log.info("Synchronizing relations to EDQS");
long ts = System.currentTimeMillis();
UUID lastFromEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000");
String lastFromEntityType = "";
String lastRelationTypeGroup = "";
String lastRelationType = "";
UUID lastToEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000");
String lastToEntityType = "";
while (true) {
List<RelationEntity> batch = relationRepository.findNextBatch(lastFromEntityId, lastFromEntityType, lastRelationTypeGroup,
lastRelationType, lastToEntityId, lastToEntityType, entityBatchSize);
if (batch.isEmpty()) {
break;
}
processRelationBatch(batch);
RelationEntity lastRecord = batch.get(batch.size() - 1);
lastFromEntityId = lastRecord.getFromId();
lastFromEntityType = lastRecord.getFromType();
lastRelationTypeGroup = lastRecord.getRelationTypeGroup();
lastRelationType = lastRecord.getRelationType();
lastToEntityId = lastRecord.getToId();
lastToEntityType = lastRecord.getToType();
}
log.info("Finished synchronizing relations to EDQS in {} ms", (System.currentTimeMillis() - ts));
}
private void processRelationBatch(List<RelationEntity> relations) {
for (RelationEntity relation : relations) {
if (RelationTypeGroup.COMMON.name().equals(relation.getRelationTypeGroup())) {
EntityIdInfo entityIdInfo = entityInfoMap.get(relation.getFromId());
if (entityIdInfo != null) {
process(entityIdInfo.tenantId(), RELATION, relation.toData());
} else {
log.info("Relation from id not found: {} ", relation);
}
}
}
}
private void loadKeyDictionary() {
log.info("Loading key dictionary");
long ts = System.currentTimeMillis();
var keyDictionaryEntries = new PageDataIterable<>(keyDictionaryDao::findAll, 10000);
for (KeyDictionaryEntry keyDictionaryEntry : keyDictionaryEntries) {
keys.put(keyDictionaryEntry.getKeyId(), keyDictionaryEntry.getKey());
}
log.info("Finished loading key dictionary in {} ms", (System.currentTimeMillis() - ts));
}
private void syncAttributes() {
log.info("Synchronizing attributes to EDQS");
long ts = System.currentTimeMillis();
UUID lastEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000");
int lastAttributeType = Integer.MIN_VALUE;
int lastAttributeKey = Integer.MIN_VALUE;
while (true) {
List<AttributeKvEntity> batch = attributesDao.findNextBatch(lastEntityId, lastAttributeType, lastAttributeKey, tsBatchSize);
if (batch.isEmpty()) {
break;
}
processAttributeBatch(batch);
AttributeKvEntity lastRecord = batch.get(batch.size() - 1);
lastEntityId = lastRecord.getId().getEntityId();
lastAttributeType = lastRecord.getId().getAttributeType();
lastAttributeKey = lastRecord.getId().getAttributeKey();
}
log.info("Finished synchronizing attributes to EDQS in {} ms", (System.currentTimeMillis() - ts));
}
private void processAttributeBatch(List<AttributeKvEntity> batch) {
for (AttributeKvEntity attribute : batch) {
attribute.setStrKey(getStrKeyOrFetchFromDb(attribute.getId().getAttributeKey()));
UUID entityId = attribute.getId().getEntityId();
EntityIdInfo entityIdInfo = entityInfoMap.get(entityId);
if (entityIdInfo == null) {
log.debug("Skipping attribute with entity UUID {} as it is not found in entityInfoMap", entityId);
continue;
}
AttributeKv attributeKv = new AttributeKv(
EntityIdFactory.getByTypeAndUuid(entityIdInfo.entityType(), entityId),
AttributeScope.valueOf(attribute.getId().getAttributeType()),
attribute.toData(),
attribute.getVersion());
process(entityIdInfo.tenantId(), ATTRIBUTE_KV, attributeKv);
}
}
private void syncLatestTimeseries() {
log.info("Synchronizing latest timeseries to EDQS");
long ts = System.currentTimeMillis();
UUID lastEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000");
int lastKey = Integer.MIN_VALUE;
while (true) {
List<TsKvLatestEntity> batch = tsKvLatestRepository.findNextBatch(lastEntityId, lastKey, tsBatchSize);
if (batch.isEmpty()) {
break;
}
processTsKvLatestBatch(batch);
TsKvLatestEntity lastRecord = batch.get(batch.size() - 1);
lastEntityId = lastRecord.getEntityId();
lastKey = lastRecord.getKey();
}
log.info("Finished synchronizing latest timeseries to EDQS in {} ms", (System.currentTimeMillis() - ts));
}
private void processTsKvLatestBatch(List<TsKvLatestEntity> tsKvLatestEntities) {
for (TsKvLatestEntity tsKvLatestEntity : tsKvLatestEntities) {
try {
String strKey = getStrKeyOrFetchFromDb(tsKvLatestEntity.getKey());
if (strKey == null) {
log.debug("Skipping latest timeseries with key {} as it is not found in key dictionary", tsKvLatestEntity.getKey());
continue;
}
tsKvLatestEntity.setStrKey(strKey);
UUID entityUuid = tsKvLatestEntity.getEntityId();
EntityIdInfo entityIdInfo = entityInfoMap.get(entityUuid);
if (entityIdInfo != null) {
EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityIdInfo.entityType(), entityUuid);
LatestTsKv latestTsKv = new LatestTsKv(entityId, tsKvLatestEntity.toData(), tsKvLatestEntity.getVersion());
process(entityIdInfo.tenantId(), LATEST_TS_KV, latestTsKv);
}
} catch (Exception e) {
log.error("Failed to sync latest timeseries: {}", tsKvLatestEntity, e);
}
}
}
private String getStrKeyOrFetchFromDb(int key) {
String strKey = keys.get(key);
if (strKey != null) {
return strKey;
} else {
strKey = keyDictionaryDao.getKey(key);
if (strKey != null) {
keys.put(key, strKey);
}
}
return strKey;
}
public record EntityIdInfo(EntityType entityType, TenantId tenantId) {
}
}

42
application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java

@ -0,0 +1,42 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.edqs;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Service;
import org.thingsboard.server.queue.edqs.EdqsQueue;
import org.thingsboard.server.queue.kafka.TbKafkaAdmin;
import org.thingsboard.server.queue.kafka.TbKafkaSettings;
import java.util.Collections;
@Service
@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'kafka'")
public class KafkaEdqsSyncService extends EdqsSyncService {
private final boolean syncNeeded;
public KafkaEdqsSyncService(TbKafkaSettings kafkaSettings) {
TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, Collections.emptyMap());
this.syncNeeded = kafkaAdmin.isTopicEmpty(EdqsQueue.EVENTS.getTopic());
}
@Override
public boolean isSyncNeeded() {
return syncNeeded;
}
}

35
application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java

@ -0,0 +1,35 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.edqs;
import lombok.RequiredArgsConstructor;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Service;
import org.thingsboard.server.edqs.util.EdqsRocksDb;
@Service
@RequiredArgsConstructor
@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'in-memory'")
public class LocalEdqsSyncService extends EdqsSyncService {
private final EdqsRocksDb db;
@Override
public boolean isSyncNeeded() {
return db.isNew();
}
}

10
application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java

@ -52,6 +52,7 @@ import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg;
import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNotificationMsg;
import org.thingsboard.server.dao.edge.EdgeSynchronizationManager;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
@ -67,6 +68,7 @@ public class EntityStateSourcingListener {
private final TenantService tenantService;
private final TbClusterService tbClusterService;
private final EdgeSynchronizationManager edgeSynchronizationManager;
@PostConstruct
public void init() {
@ -75,6 +77,11 @@ public class EntityStateSourcingListener {
@TransactionalEventListener(fallbackExecution = true)
public void handleEvent(SaveEntityEvent<?> event) {
if (Boolean.FALSE.equals(event.getBroadcastEvent())) {
log.trace("Ignoring event {}", event);
return;
}
TenantId tenantId = event.getTenantId();
EntityId entityId = event.getEntityId();
if (entityId == null) {
@ -270,6 +277,9 @@ public class EntityStateSourcingListener {
private void onEdgeEvent(TenantId tenantId, EntityId entityId, Object entity, ComponentLifecycleEvent lifecycleEvent) {
if (entity instanceof Edge) {
if (entityId.equals(edgeSynchronizationManager.getEdgeId().get())) {
return;
}
tbClusterService.onEdgeStateChangeEvent(new ComponentLifecycleMsg(tenantId, entityId, lifecycleEvent));
} else if (entity instanceof EdgeEvent edgeEvent) {
tbClusterService.onEdgeEventUpdate(new EdgeEventUpdateMsg(tenantId, edgeEvent.getEdgeId()));

45
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@ -20,7 +20,6 @@ import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
@ -33,6 +32,7 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.JavaSerDesUtil;
import org.thingsboard.server.common.data.alarm.AlarmInfo;
import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg;
import org.thingsboard.server.common.data.event.ErrorEvent;
import org.thingsboard.server.common.data.event.Event;
import org.thingsboard.server.common.data.event.LifecycleEvent;
@ -45,6 +45,7 @@ import org.thingsboard.server.common.data.queue.QueueConfig;
import org.thingsboard.server.common.data.rpc.RpcError;
import org.thingsboard.server.common.msg.MsgType;
import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.edqs.EdqsService;
import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
@ -76,6 +77,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceM
import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager;
import org.thingsboard.server.queue.common.consumer.QueueConsumerManager;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.QueueKey;
@ -88,7 +90,6 @@ import org.thingsboard.server.service.notification.NotificationSchedulerService;
import org.thingsboard.server.service.ota.OtaPackageStateService;
import org.thingsboard.server.service.profile.TbAssetProfileCache;
import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager;
import org.thingsboard.server.service.queue.processing.AbstractConsumerService;
import org.thingsboard.server.service.queue.processing.IdMsgPair;
import org.thingsboard.server.service.resource.TbImageService;
@ -146,9 +147,10 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
private final TbCoreQueueFactory queueFactory;
private final TbImageService imageService;
private final RuleEngineCallService ruleEngineCallService;
private final EdqsService edqsService;
private final TbCoreConsumerStats stats;
private MainQueueConsumerManager<TbProtoQueueMsg<ToCoreMsg>, CoreQueueConfig> mainConsumer;
private MainQueueConsumerManager<TbProtoQueueMsg<ToCoreMsg>, QueueConfig> mainConsumer;
private QueueConsumerManager<TbProtoQueueMsg<ToUsageStatsServiceMsg>> usageStatsConsumer;
private QueueConsumerManager<TbProtoQueueMsg<ToOtaPackageStateServiceMsg>> firmwareStatesConsumer;
@ -175,7 +177,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
NotificationRuleProcessor notificationRuleProcessor,
TbImageService imageService,
RuleEngineCallService ruleEngineCallService,
CalculatedFieldCache calculatedFieldCache) {
CalculatedFieldCache calculatedFieldCache,
EdqsService edqsService) {
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService,
eventPublisher, jwtSettingsService);
this.stateService = stateService;
@ -191,6 +194,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
this.imageService = imageService;
this.ruleEngineCallService = ruleEngineCallService;
this.queueFactory = tbCoreQueueFactory;
this.edqsService = edqsService;
}
@PostConstruct
@ -198,9 +202,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
super.init("tb-core");
this.deviceActivityEventsExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-device-activity-events-executor")));
this.mainConsumer = MainQueueConsumerManager.<TbProtoQueueMsg<ToCoreMsg>, CoreQueueConfig>builder()
this.mainConsumer = MainQueueConsumerManager.<TbProtoQueueMsg<ToCoreMsg>, QueueConfig>builder()
.queueKey(new QueueKey(ServiceType.TB_CORE))
.config(CoreQueueConfig.of(consumerPerPartition, (int) pollInterval))
.config(QueueConfig.of(consumerPerPartition, pollInterval))
.msgPackProcessor(this::processMsgs)
.consumerCreator((config, partitionId) -> queueFactory.createToCoreMsgConsumer())
.consumerExecutor(consumersExecutor)
@ -251,7 +255,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
.collect(Collectors.toSet()));
}
private void processMsgs(List<TbProtoQueueMsg<ToCoreMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> consumer, CoreQueueConfig config) throws Exception {
private void processMsgs(List<TbProtoQueueMsg<ToCoreMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> consumer, QueueConfig config) throws Exception {
List<IdMsgPair<ToCoreMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList();
ConcurrentMap<UUID, TbProtoQueueMsg<ToCoreMsg>> pendingMap = orderedMsgList.stream().collect(
Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg));
@ -289,6 +293,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
} else if (toCoreMsg.hasDeviceInactivityMsg()) {
log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceInactivityMsg());
forwardToStateService(toCoreMsg.getDeviceInactivityMsg(), callback);
} else if (toCoreMsg.hasDeviceInactivityTimeoutUpdateMsg()) {
log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceInactivityTimeoutUpdateMsg());
forwardToStateService(toCoreMsg.getDeviceInactivityTimeoutUpdateMsg(), callback);
} else if (toCoreMsg.hasToDeviceActorNotification()) {
TbActorMsg actorMsg = ProtoUtils.fromProto(toCoreMsg.getToDeviceActorNotification());
if (actorMsg != null) {
@ -389,6 +396,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
callback.onSuccess();
} else if (toCoreNotification.hasResourceCacheInvalidateMsg()) {
forwardToResourceService(toCoreNotification.getResourceCacheInvalidateMsg(), callback);
} else if (toCoreNotification.hasToEdqsCoreServiceMsg()) {
edqsService.processSystemMsg(JacksonUtil.fromBytes(toCoreNotification.getToEdqsCoreServiceMsg().getValue().toByteArray(), ToCoreEdqsMsg.class));
callback.onSuccess();
}
if (statsEnabled) {
stats.log(toCoreNotification);
@ -660,6 +670,21 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
});
}
void forwardToStateService(TransportProtos.DeviceInactivityTimeoutUpdateProto deviceInactivityTimeoutUpdateMsg, TbCallback callback) {
if (statsEnabled) {
stats.log(deviceInactivityTimeoutUpdateMsg);
}
var tenantId = toTenantId(deviceInactivityTimeoutUpdateMsg.getTenantIdMSB(), deviceInactivityTimeoutUpdateMsg.getTenantIdLSB());
var deviceId = new DeviceId(new UUID(deviceInactivityTimeoutUpdateMsg.getDeviceIdMSB(), deviceInactivityTimeoutUpdateMsg.getDeviceIdLSB()));
ListenableFuture<?> future = deviceActivityEventsExecutor.submit(() -> stateService.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, deviceInactivityTimeoutUpdateMsg.getInactivityTimeout()));
DonAsynchron.withCallback(future,
__ -> callback.onSuccess(),
t -> {
log.warn("[{}] Failed to process device inactivity timeout update message for device [{}]", tenantId.getId(), deviceId.getId(), t);
callback.onFailure(t);
});
}
private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) {
TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB());
NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB()));
@ -732,10 +757,4 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
firmwareStatesConsumer.stop();
}
@Data(staticConstructor = "of")
public static class CoreQueueConfig implements QueueConfig {
private final boolean consumerPerPartition;
private final int pollInterval;
}
}

14
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java

@ -85,7 +85,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService<ToEdge
private final EdgeContextComponent edgeCtx;
private final EdgeConsumerStats stats;
private MainQueueConsumerManager<TbProtoQueueMsg<ToEdgeMsg>, EdgeQueueConfig> mainConsumer;
private MainQueueConsumerManager<TbProtoQueueMsg<ToEdgeMsg>, QueueConfig> mainConsumer;
public DefaultTbEdgeConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext,
StatsFactory statsFactory, EdgeContextComponent edgeCtx) {
@ -100,9 +100,9 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService<ToEdge
public void init() {
super.init("tb-edge");
this.mainConsumer = MainQueueConsumerManager.<TbProtoQueueMsg<ToEdgeMsg>, EdgeQueueConfig>builder()
this.mainConsumer = MainQueueConsumerManager.<TbProtoQueueMsg<ToEdgeMsg>, QueueConfig>builder()
.queueKey(new QueueKey(ServiceType.TB_CORE).withQueueName(DataConstants.EDGE_QUEUE_NAME))
.config(EdgeQueueConfig.of(consumerPerPartition, pollInterval))
.config(QueueConfig.of(consumerPerPartition, pollInterval))
.msgPackProcessor(this::processMsgs)
.consumerCreator((config, partitionId) -> queueFactory.createEdgeMsgConsumer())
.consumerExecutor(consumersExecutor)
@ -128,7 +128,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService<ToEdge
mainConsumer.update(partitions);
}
private void processMsgs(List<TbProtoQueueMsg<ToEdgeMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToEdgeMsg>> consumer, EdgeQueueConfig edgeQueueConfig) throws InterruptedException {
private void processMsgs(List<TbProtoQueueMsg<ToEdgeMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToEdgeMsg>> consumer, QueueConfig edgeQueueConfig) throws InterruptedException {
List<IdMsgPair<ToEdgeMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList();
ConcurrentMap<UUID, TbProtoQueueMsg<ToEdgeMsg>> pendingMap = orderedMsgList.stream().collect(
Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg));
@ -285,10 +285,4 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService<ToEdge
mainConsumer.awaitStop();
}
@Data(staticConstructor = "of")
public static class EdgeQueueConfig implements QueueConfig {
private final boolean consumerPerPartition;
private final int pollInterval;
}
}

2
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java

@ -107,7 +107,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
@Override
protected void onTbApplicationEvent(PartitionChangeEvent event) {
event.getPartitionsMap().forEach((queueKey, partitions) -> {
event.getNewPartitions().forEach((queueKey, partitions) -> {
if (CollectionsUtil.isOneOf(queueKey, QueueKey.CF, QueueKey.CF_STATES)) {
return;
}

8
application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java

@ -40,6 +40,7 @@ public class TbCoreConsumerStats {
public static final String DEVICE_ACTIVITIES = "deviceActivity";
public static final String DEVICE_DISCONNECTS = "deviceDisconnect";
public static final String DEVICE_INACTIVITIES = "deviceInactivity";
public static final String DEVICE_INACTIVITY_TIMEOUT_UPDATES = "deviceInactivityTimeoutUpdate";
public static final String TO_CORE_NF_OTHER = "coreNfOther"; // normally, there is no messages when codebase is fine
public static final String TO_CORE_NF_COMPONENT_LIFECYCLE = "coreNfCompLfcl";
@ -65,6 +66,7 @@ public class TbCoreConsumerStats {
private final StatsCounter deviceActivitiesCounter;
private final StatsCounter deviceDisconnectsCounter;
private final StatsCounter deviceInactivitiesCounter;
private final StatsCounter deviceInactivityTimeoutUpdatesCounter;
private final StatsCounter toCoreNfOtherCounter;
private final StatsCounter toCoreNfComponentLifecycleCounter;
@ -95,6 +97,7 @@ public class TbCoreConsumerStats {
this.deviceActivitiesCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_ACTIVITIES));
this.deviceDisconnectsCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_DISCONNECTS));
this.deviceInactivitiesCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_INACTIVITIES));
this.deviceInactivityTimeoutUpdatesCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_INACTIVITY_TIMEOUT_UPDATES));
// Core notification counters
this.toCoreNfOtherCounter = register(statsFactory.createStatsCounter(statsKey, TO_CORE_NF_OTHER));
@ -163,6 +166,11 @@ public class TbCoreConsumerStats {
deviceInactivitiesCounter.increment();
}
public void log(TransportProtos.DeviceInactivityTimeoutUpdateProto msg) {
totalCounter.increment();
deviceInactivityTimeoutUpdatesCounter.increment();
}
public void log(TransportProtos.SubscriptionMgrMsgProto msg) {
totalCounter.increment();
subscriptionMsgCounter.increment();

2
application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java

@ -72,7 +72,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager<T
ExecutorService consumerExecutor,
ScheduledExecutorService scheduler,
ExecutorService taskExecutor) {
super(queueKey, null, null, ctx.getQueueFactory()::createToRuleEngineMsgConsumer, consumerExecutor, scheduler, taskExecutor);
super(queueKey, null, null, ctx.getQueueFactory()::createToRuleEngineMsgConsumer, consumerExecutor, scheduler, taskExecutor, null);
this.ctx = ctx;
this.stats = new TbRuleEngineConsumerStats(queueKey, ctx.getStatsFactory());
}

180
application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateManager.java

@ -0,0 +1,180 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.state;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.thingsboard.rule.engine.api.DeviceStateManager;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.common.SimpleTbQueueCallback;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import java.util.Optional;
import java.util.function.Consumer;
import java.util.function.Supplier;
@Slf4j
@Service
@RequiredArgsConstructor
public class DefaultDeviceStateManager implements DeviceStateManager {
private final TbServiceInfoProvider serviceInfoProvider;
private final PartitionService partitionService;
private final Optional<DeviceStateService> deviceStateService;
private final TbClusterService clusterService;
@Override
public void onDeviceConnect(TenantId tenantId, DeviceId deviceId, long connectTime, TbCallback callback) {
forwardToDeviceStateService(tenantId, deviceId,
deviceStateService -> {
log.debug("[{}][{}] Forwarding device connect event to local service. Connect time: [{}].", tenantId.getId(), deviceId.getId(), connectTime);
deviceStateService.onDeviceConnect(tenantId, deviceId, connectTime);
},
() -> {
log.debug("[{}][{}] Sending device connect message to core. Connect time: [{}].", tenantId.getId(), deviceId.getId(), connectTime);
var deviceConnectMsg = TransportProtos.DeviceConnectProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setLastConnectTime(connectTime)
.build();
return TransportProtos.ToCoreMsg.newBuilder()
.setDeviceConnectMsg(deviceConnectMsg)
.build();
}, callback);
}
@Override
public void onDeviceActivity(TenantId tenantId, DeviceId deviceId, long activityTime, TbCallback callback) {
forwardToDeviceStateService(tenantId, deviceId,
deviceStateService -> {
log.debug("[{}][{}] Forwarding device activity event to local service. Activity time: [{}].", tenantId.getId(), deviceId.getId(), activityTime);
deviceStateService.onDeviceActivity(tenantId, deviceId, activityTime);
},
() -> {
log.debug("[{}][{}] Sending device activity message to core. Activity time: [{}].", tenantId.getId(), deviceId.getId(), activityTime);
var deviceActivityMsg = TransportProtos.DeviceActivityProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setLastActivityTime(activityTime)
.build();
return TransportProtos.ToCoreMsg.newBuilder()
.setDeviceActivityMsg(deviceActivityMsg)
.build();
}, callback);
}
@Override
public void onDeviceDisconnect(TenantId tenantId, DeviceId deviceId, long disconnectTime, TbCallback callback) {
forwardToDeviceStateService(tenantId, deviceId,
deviceStateService -> {
log.debug("[{}][{}] Forwarding device disconnect event to local service. Disconnect time: [{}].", tenantId.getId(), deviceId.getId(), disconnectTime);
deviceStateService.onDeviceDisconnect(tenantId, deviceId, disconnectTime);
},
() -> {
log.debug("[{}][{}] Sending device disconnect message to core. Disconnect time: [{}].", tenantId.getId(), deviceId.getId(), disconnectTime);
var deviceDisconnectMsg = TransportProtos.DeviceDisconnectProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setLastDisconnectTime(disconnectTime)
.build();
return TransportProtos.ToCoreMsg.newBuilder()
.setDeviceDisconnectMsg(deviceDisconnectMsg)
.build();
}, callback);
}
@Override
public void onDeviceInactivity(TenantId tenantId, DeviceId deviceId, long inactivityTime, TbCallback callback) {
forwardToDeviceStateService(tenantId, deviceId,
deviceStateService -> {
log.debug("[{}][{}] Forwarding device inactivity event to local service. Inactivity time: [{}].", tenantId.getId(), deviceId.getId(), inactivityTime);
deviceStateService.onDeviceInactivity(tenantId, deviceId, inactivityTime);
},
() -> {
log.debug("[{}][{}] Sending device inactivity message to core. Inactivity time: [{}].", tenantId.getId(), deviceId.getId(), inactivityTime);
var deviceInactivityMsg = TransportProtos.DeviceInactivityProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setLastInactivityTime(inactivityTime)
.build();
return TransportProtos.ToCoreMsg.newBuilder()
.setDeviceInactivityMsg(deviceInactivityMsg)
.build();
}, callback);
}
@Override
public void onDeviceInactivityTimeoutUpdate(TenantId tenantId, DeviceId deviceId, long inactivityTimeout, TbCallback callback) {
forwardToDeviceStateService(tenantId, deviceId,
deviceStateService -> {
log.debug("[{}][{}] Forwarding device inactivity timeout update to local service. Updated inactivity timeout: [{}].", tenantId.getId(), deviceId.getId(), inactivityTimeout);
deviceStateService.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, inactivityTimeout);
},
() -> {
log.debug("[{}][{}] Sending device inactivity timeout update message to core. Updated inactivity timeout: [{}].", tenantId.getId(), deviceId.getId(), inactivityTimeout);
var deviceInactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setInactivityTimeout(inactivityTimeout)
.build();
return TransportProtos.ToCoreMsg.newBuilder()
.setDeviceInactivityTimeoutUpdateMsg(deviceInactivityTimeoutUpdateMsg)
.build();
}, callback);
}
private void forwardToDeviceStateService(
TenantId tenantId, DeviceId deviceId,
Consumer<DeviceStateService> toDeviceStateService,
Supplier<TransportProtos.ToCoreMsg> toCore,
TbCallback callback
) {
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId);
if (serviceInfoProvider.isService(ServiceType.TB_CORE) && tpi.isMyPartition() && deviceStateService.isPresent()) {
try {
toDeviceStateService.accept(deviceStateService.get());
} catch (Exception e) {
log.error("[{}][{}] Failed to process device connectivity event.", tenantId.getId(), deviceId.getId(), e);
callback.onFailure(e);
return;
}
callback.onSuccess();
} else {
TransportProtos.ToCoreMsg toCoreMsg = toCore.get();
clusterService.pushMsgToCore(tpi, deviceId.getId(), toCoreMsg, new SimpleTbQueueCallback(__ -> callback.onSuccess(), callback::onFailure));
}
}
}

108
application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java

@ -96,6 +96,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
@ -129,11 +130,10 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
private static final List<EntityKey> PERSISTENT_TELEMETRY_KEYS = Arrays.asList(
new EntityKey(EntityKeyType.TIME_SERIES, LAST_ACTIVITY_TIME),
new EntityKey(EntityKeyType.TIME_SERIES, INACTIVITY_ALARM_TIME),
new EntityKey(EntityKeyType.TIME_SERIES, INACTIVITY_TIMEOUT),
new EntityKey(EntityKeyType.TIME_SERIES, ACTIVITY_STATE),
new EntityKey(EntityKeyType.TIME_SERIES, LAST_CONNECT_TIME),
new EntityKey(EntityKeyType.TIME_SERIES, LAST_DISCONNECT_TIME),
new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, INACTIVITY_TIMEOUT));
new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, INACTIVITY_TIMEOUT)); // inactivity timeout is always a server attribute, even when activity data is stored as time series
private static final List<EntityKey> PERSISTENT_ATTRIBUTE_KEYS = Arrays.asList(
new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, LAST_ACTIVITY_TIME),
@ -143,8 +143,14 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, LAST_CONNECT_TIME),
new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, LAST_DISCONNECT_TIME));
public static final List<String> PERSISTENT_ATTRIBUTES = Arrays.asList(ACTIVITY_STATE, LAST_CONNECT_TIME,
LAST_DISCONNECT_TIME, LAST_ACTIVITY_TIME, INACTIVITY_ALARM_TIME, INACTIVITY_TIMEOUT);
public static final Set<String> ACTIVITY_KEYS_WITHOUT_INACTIVITY_TIMEOUT = Set.of(
ACTIVITY_STATE, LAST_CONNECT_TIME, LAST_DISCONNECT_TIME, LAST_ACTIVITY_TIME, INACTIVITY_ALARM_TIME
);
public static final Set<String> ACTIVITY_KEYS_WITH_INACTIVITY_TIMEOUT = Set.of(
ACTIVITY_STATE, LAST_CONNECT_TIME, LAST_DISCONNECT_TIME, LAST_ACTIVITY_TIME, INACTIVITY_ALARM_TIME, INACTIVITY_TIMEOUT
);
private static final List<EntityKey> PERSISTENT_ENTITY_FIELDS = Arrays.asList(
new EntityKey(EntityKeyType.ENTITY_FIELD, "name"),
new EntityKey(EntityKeyType.ENTITY_FIELD, "type"),
@ -251,7 +257,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
}
log.trace("[{}][{}] On device connect: processing connect event with ts [{}].", tenantId.getId(), deviceId.getId(), lastConnectTime);
stateData.getState().setLastConnectTime(lastConnectTime);
save(deviceId, LAST_CONNECT_TIME, lastConnectTime);
save(tenantId, deviceId, LAST_CONNECT_TIME, lastConnectTime);
pushRuleEngineMessage(stateData, TbMsgType.CONNECT_EVENT);
checkAndUpdateState(deviceId, stateData);
}
@ -271,14 +277,14 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
void updateActivityState(DeviceId deviceId, DeviceStateData stateData, long lastReportedActivity) {
log.trace("updateActivityState - fetched state {} for device {}, lastReportedActivity {}", stateData, deviceId, lastReportedActivity);
if (stateData != null) {
save(deviceId, LAST_ACTIVITY_TIME, lastReportedActivity);
save(stateData.getTenantId(), deviceId, LAST_ACTIVITY_TIME, lastReportedActivity);
DeviceState state = stateData.getState();
state.setLastActivityTime(lastReportedActivity);
if (!state.isActive()) {
state.setActive(true);
if (lastReportedActivity <= state.getLastInactivityAlarmTime()) {
state.setLastInactivityAlarmTime(0);
save(deviceId, INACTIVITY_ALARM_TIME, 0);
save(stateData.getTenantId(), deviceId, INACTIVITY_ALARM_TIME, 0);
}
onDeviceActivityStatusChange(deviceId, true, stateData);
}
@ -307,7 +313,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
}
log.trace("[{}][{}] On device disconnect: processing disconnect event with ts [{}].", tenantId.getId(), deviceId.getId(), lastDisconnectTime);
stateData.getState().setLastDisconnectTime(lastDisconnectTime);
save(deviceId, LAST_DISCONNECT_TIME, lastDisconnectTime);
save(tenantId, deviceId, LAST_DISCONNECT_TIME, lastDisconnectTime);
pushRuleEngineMessage(stateData, TbMsgType.DISCONNECT_EVENT);
}
@ -418,7 +424,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
private void initializeActivityState(DeviceId deviceId, DeviceStateData fetchedState) {
DeviceStateData cachedState = deviceStates.putIfAbsent(fetchedState.getDeviceId(), fetchedState);
boolean activityState = Objects.requireNonNullElse(cachedState, fetchedState).getState().isActive();
save(deviceId, ACTIVITY_STATE, activityState);
save(fetchedState.getTenantId(), deviceId, ACTIVITY_STATE, activityState);
}
@Override
@ -496,7 +502,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
updateActivityState(deviceId, state, deviceState.getLastActivityTime());
if (deviceState.getLastInactivityAlarmTime() != 0L && deviceState.getLastInactivityAlarmTime() >= deviceState.getLastActivityTime()) {
deviceState.setLastInactivityAlarmTime(0L);
save(deviceId, INACTIVITY_ALARM_TIME, 0L);
save(state.getTenantId(), deviceId, INACTIVITY_ALARM_TIME, 0L);
}
}
}
@ -583,7 +589,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
DeviceState state = stateData.getState();
state.setActive(false);
state.setLastInactivityAlarmTime(ts);
save(deviceId, INACTIVITY_ALARM_TIME, ts);
save(stateData.getTenantId(), deviceId, INACTIVITY_ALARM_TIME, ts);
onDeviceActivityStatusChange(deviceId, false, stateData);
}
@ -611,7 +617,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
}
private void onDeviceActivityStatusChange(DeviceId deviceId, boolean active, DeviceStateData stateData) {
save(deviceId, ACTIVITY_STATE, active);
save(stateData.getTenantId(), deviceId, ACTIVITY_STATE, active);
pushRuleEngineMessage(stateData, active ? TbMsgType.ACTIVITY_EVENT : TbMsgType.INACTIVITY_EVENT);
TbMsgMetaData metaData = stateData.getMetaData();
notificationRuleProcessor.process(DeviceActivityTrigger.builder()
@ -643,41 +649,45 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
deviceStates.remove(deviceId);
}
private ListenableFuture<DeviceStateData> fetchDeviceState(Device device) {
ListenableFuture<DeviceStateData> future;
if (persistToTelemetry) {
ListenableFuture<List<TsKvEntry>> tsData = tsService.findLatest(TenantId.SYS_TENANT_ID, device.getId(), PERSISTENT_ATTRIBUTES);
future = Futures.transform(tsData, extractDeviceStateData(device), MoreExecutors.directExecutor());
ListenableFuture<List<TsKvEntry>> timeseriesActivityDataFuture = tsService.findLatest(TenantId.SYS_TENANT_ID, device.getId(), ACTIVITY_KEYS_WITHOUT_INACTIVITY_TIMEOUT);
ListenableFuture<Optional<AttributeKvEntry>> inactivityTimeoutAttributeFuture = attributesService.find(
TenantId.SYS_TENANT_ID, device.getId(), AttributeScope.SERVER_SCOPE, INACTIVITY_TIMEOUT
);
ListenableFuture<List<? extends KvEntry>> fullActivityDataFuture = Futures.whenAllSucceed(timeseriesActivityDataFuture, inactivityTimeoutAttributeFuture).call(() -> {
List<TsKvEntry> activityTimeseries = Futures.getDone(timeseriesActivityDataFuture);
Optional<AttributeKvEntry> inactivityTimeoutAttribute = Futures.getDone(inactivityTimeoutAttributeFuture);
List<KvEntry> result;
if (inactivityTimeoutAttribute.isPresent()) {
result = new ArrayList<>(activityTimeseries.size() + 1);
result.addAll(activityTimeseries);
inactivityTimeoutAttribute.ifPresent(result::add);
} else {
return activityTimeseries;
}
return result;
}, deviceStateCallbackExecutor);
future = Futures.transform(fullActivityDataFuture, extractDeviceStateData(device), MoreExecutors.directExecutor());
} else {
ListenableFuture<List<AttributeKvEntry>> attrData = attributesService.find(TenantId.SYS_TENANT_ID, device.getId(), AttributeScope.SERVER_SCOPE, PERSISTENT_ATTRIBUTES);
future = Futures.transform(attrData, extractDeviceStateData(device), MoreExecutors.directExecutor());
ListenableFuture<List<AttributeKvEntry>> attributesActivityDataFuture = attributesService.find(
TenantId.SYS_TENANT_ID, device.getId(), AttributeScope.SERVER_SCOPE, ACTIVITY_KEYS_WITH_INACTIVITY_TIMEOUT
);
future = Futures.transform(attributesActivityDataFuture, extractDeviceStateData(device), MoreExecutors.directExecutor());
}
return transformInactivityTimeout(future);
return future;
}
private ListenableFuture<DeviceStateData> transformInactivityTimeout(ListenableFuture<DeviceStateData> future) {
return Futures.transformAsync(future, deviceStateData -> {
if (!persistToTelemetry || deviceStateData.getState().getInactivityTimeout() != defaultInactivityTimeoutMs) {
return future; //fail fast
}
var attributesFuture = attributesService.find(TenantId.SYS_TENANT_ID, deviceStateData.getDeviceId(), AttributeScope.SERVER_SCOPE, INACTIVITY_TIMEOUT);
return Futures.transform(attributesFuture, attributes -> {
attributes.flatMap(KvEntry::getLongValue).ifPresent((inactivityTimeout) -> {
if (inactivityTimeout > 0) {
deviceStateData.getState().setInactivityTimeout(inactivityTimeout);
}
});
return deviceStateData;
}, MoreExecutors.directExecutor());
}, deviceStateCallbackExecutor);
}
private <T extends KvEntry> Function<List<T>, DeviceStateData> extractDeviceStateData(Device device) {
private Function<List<? extends KvEntry>, DeviceStateData> extractDeviceStateData(Device device) {
return new Function<>() {
@Nonnull
@Override
public DeviceStateData apply(@Nullable List<T> data) {
public DeviceStateData apply(@Nullable List<? extends KvEntry> data) {
try {
long lastActivityTime = getEntryValue(data, LAST_ACTIVITY_TIME, 0L);
long inactivityAlarmTime = getEntryValue(data, INACTIVITY_ALARM_TIME, 0L);
@ -690,7 +700,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
.lastDisconnectTime(getEntryValue(data, LAST_DISCONNECT_TIME, 0L))
.lastActivityTime(lastActivityTime)
.lastInactivityAlarmTime(inactivityAlarmTime)
.inactivityTimeout(inactivityTimeout)
.inactivityTimeout(inactivityTimeout > 0 ? inactivityTimeout : defaultInactivityTimeoutMs)
.build();
TbMsgMetaData md = new TbMsgMetaData();
md.putValue("deviceName", device.getName());
@ -761,12 +771,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
DeviceStateData toDeviceStateData(EntityData ed, DeviceIdInfo deviceIdInfo) {
long lastActivityTime = getEntryValue(ed, getKeyType(), LAST_ACTIVITY_TIME, 0L);
long inactivityAlarmTime = getEntryValue(ed, getKeyType(), INACTIVITY_ALARM_TIME, 0L);
long inactivityTimeout = getEntryValue(ed, getKeyType(), INACTIVITY_TIMEOUT, defaultInactivityTimeoutMs);
if (persistToTelemetry && inactivityTimeout == defaultInactivityTimeoutMs) {
log.trace("[{}] default value for inactivity timeout fetched {}, going to fetch inactivity timeout from attributes",
deviceIdInfo.getDeviceId(), inactivityTimeout);
inactivityTimeout = getEntryValue(ed, EntityKeyType.SERVER_ATTRIBUTE, INACTIVITY_TIMEOUT, defaultInactivityTimeoutMs);
}
long inactivityTimeout = getEntryValue(ed, EntityKeyType.SERVER_ATTRIBUTE, INACTIVITY_TIMEOUT, defaultInactivityTimeoutMs);
// Actual active state by wall-clock will be updated outside this method. This method is only for fetching persistent state
final boolean active = getEntryValue(ed, getKeyType(), ACTIVITY_STATE, false);
DeviceState deviceState = DeviceState.builder()
@ -874,18 +879,18 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
}
}
private void save(DeviceId deviceId, String key, long value) {
save(deviceId, new LongDataEntry(key, value), getCurrentTimeMillis());
private void save(TenantId tenantId, DeviceId deviceId, String key, long value) {
save(tenantId, deviceId, new LongDataEntry(key, value), getCurrentTimeMillis());
}
private void save(DeviceId deviceId, String key, boolean value) {
save(deviceId, new BooleanDataEntry(key, value), getCurrentTimeMillis());
private void save(TenantId tenantId, DeviceId deviceId, String key, boolean value) {
save(tenantId, deviceId, new BooleanDataEntry(key, value), getCurrentTimeMillis());
}
private void save(DeviceId deviceId, KvEntry kvEntry, long ts) {
private void save(TenantId tenantId, DeviceId deviceId, KvEntry kvEntry, long ts) {
if (persistToTelemetry) {
tsSubService.saveTimeseriesInternal(TimeseriesSaveRequest.builder()
.tenantId(TenantId.SYS_TENANT_ID)
.tenantId(tenantId)
.entityId(deviceId)
.entry(new BasicTsKvEntry(ts, kvEntry))
.ttl(telemetryTtl)
@ -893,7 +898,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
.build());
} else {
tsSubService.saveAttributes(AttributesSaveRequest.builder()
.tenantId(TenantId.SYS_TENANT_ID)
.tenantId(tenantId)
.entityId(deviceId)
.scope(AttributeScope.SERVER_SCOPE)
.entry(new BaseAttributeKvEntry(ts, kvEntry))
@ -925,4 +930,5 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
log.warn("[{}] Failed to update entry {}", deviceId, kvEntry, t);
}
}
}

196
application/src/main/java/org/thingsboard/server/service/state/DefaultRuleEngineDeviceStateManager.java

@ -1,196 +0,0 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.state;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.thingsboard.rule.engine.api.RuleEngineDeviceStateManager;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.common.SimpleTbQueueCallback;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.util.TbRuleEngineComponent;
import java.util.Optional;
import java.util.UUID;
@Slf4j
@Service
@TbRuleEngineComponent
public class DefaultRuleEngineDeviceStateManager implements RuleEngineDeviceStateManager {
private final TbServiceInfoProvider serviceInfoProvider;
private final PartitionService partitionService;
private final Optional<DeviceStateService> deviceStateService;
private final TbClusterService clusterService;
public DefaultRuleEngineDeviceStateManager(
TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService,
Optional<DeviceStateService> deviceStateServiceOptional, TbClusterService clusterService
) {
this.serviceInfoProvider = serviceInfoProvider;
this.partitionService = partitionService;
this.deviceStateService = deviceStateServiceOptional;
this.clusterService = clusterService;
}
@Getter
private abstract static class ConnectivityEventInfo {
private final TenantId tenantId;
private final DeviceId deviceId;
private final long eventTime;
private ConnectivityEventInfo(TenantId tenantId, DeviceId deviceId, long eventTime) {
this.tenantId = tenantId;
this.deviceId = deviceId;
this.eventTime = eventTime;
}
abstract void forwardToLocalService();
abstract TransportProtos.ToCoreMsg toQueueMsg();
}
@Override
public void onDeviceConnect(TenantId tenantId, DeviceId deviceId, long connectTime, TbCallback callback) {
routeEvent(new ConnectivityEventInfo(tenantId, deviceId, connectTime) {
@Override
void forwardToLocalService() {
deviceStateService.ifPresent(service -> service.onDeviceConnect(tenantId, deviceId, connectTime));
}
@Override
TransportProtos.ToCoreMsg toQueueMsg() {
var deviceConnectMsg = TransportProtos.DeviceConnectProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setLastConnectTime(connectTime)
.build();
return TransportProtos.ToCoreMsg.newBuilder()
.setDeviceConnectMsg(deviceConnectMsg)
.build();
}
}, callback);
}
@Override
public void onDeviceActivity(TenantId tenantId, DeviceId deviceId, long activityTime, TbCallback callback) {
routeEvent(new ConnectivityEventInfo(tenantId, deviceId, activityTime) {
@Override
void forwardToLocalService() {
deviceStateService.ifPresent(service -> service.onDeviceActivity(tenantId, deviceId, activityTime));
}
@Override
TransportProtos.ToCoreMsg toQueueMsg() {
var deviceActivityMsg = TransportProtos.DeviceActivityProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setLastActivityTime(activityTime)
.build();
return TransportProtos.ToCoreMsg.newBuilder()
.setDeviceActivityMsg(deviceActivityMsg)
.build();
}
}, callback);
}
@Override
public void onDeviceDisconnect(TenantId tenantId, DeviceId deviceId, long disconnectTime, TbCallback callback) {
routeEvent(new ConnectivityEventInfo(tenantId, deviceId, disconnectTime) {
@Override
void forwardToLocalService() {
deviceStateService.ifPresent(service -> service.onDeviceDisconnect(tenantId, deviceId, disconnectTime));
}
@Override
TransportProtos.ToCoreMsg toQueueMsg() {
var deviceDisconnectMsg = TransportProtos.DeviceDisconnectProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setLastDisconnectTime(disconnectTime)
.build();
return TransportProtos.ToCoreMsg.newBuilder()
.setDeviceDisconnectMsg(deviceDisconnectMsg)
.build();
}
}, callback);
}
@Override
public void onDeviceInactivity(TenantId tenantId, DeviceId deviceId, long inactivityTime, TbCallback callback) {
routeEvent(new ConnectivityEventInfo(tenantId, deviceId, inactivityTime) {
@Override
void forwardToLocalService() {
deviceStateService.ifPresent(service -> service.onDeviceInactivity(tenantId, deviceId, inactivityTime));
}
@Override
TransportProtos.ToCoreMsg toQueueMsg() {
var deviceInactivityMsg = TransportProtos.DeviceInactivityProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setLastInactivityTime(inactivityTime)
.build();
return TransportProtos.ToCoreMsg.newBuilder()
.setDeviceInactivityMsg(deviceInactivityMsg)
.build();
}
}, callback);
}
private void routeEvent(ConnectivityEventInfo eventInfo, TbCallback callback) {
var tenantId = eventInfo.getTenantId();
var deviceId = eventInfo.getDeviceId();
long eventTime = eventInfo.getEventTime();
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId);
if (serviceInfoProvider.isService(ServiceType.TB_CORE) && tpi.isMyPartition() && deviceStateService.isPresent()) {
log.debug("[{}][{}] Forwarding device connectivity event to local service. Event time: [{}].", tenantId.getId(), deviceId.getId(), eventTime);
try {
eventInfo.forwardToLocalService();
} catch (Exception e) {
log.error("[{}][{}] Failed to process device connectivity event. Event time: [{}].", tenantId.getId(), deviceId.getId(), eventTime, e);
callback.onFailure(e);
return;
}
callback.onSuccess();
} else {
TransportProtos.ToCoreMsg msg = eventInfo.toQueueMsg();
log.debug("[{}][{}] Sending device connectivity message to core. Event time: [{}].", tenantId.getId(), deviceId.getId(), eventTime);
clusterService.pushMsgToCore(tpi, UUID.randomUUID(), msg, new SimpleTbQueueCallback(__ -> callback.onSuccess(), callback::onFailure));
}
}
}

6
application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java

@ -160,9 +160,6 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene
@Override
public void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, TbCallback callback) {
onTimeSeriesUpdate(entityId, ts);
if (entityId.getEntityType() == EntityType.DEVICE) {
updateDeviceInactivityTimeout(tenantId, entityId, ts);
}
callback.onSuccess();
}
@ -170,9 +167,6 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene
public void onTimeSeriesDelete(TenantId tenantId, EntityId entityId, List<String> keys, TbCallback callback) {
onTimeSeriesUpdate(entityId,
keys.stream().map(key -> new BasicTsKvEntry(0, new StringDataEntry(key, ""))).collect(Collectors.toList()));
if (entityId.getEntityType() == EntityType.DEVICE) {
deleteDeviceInactivityTimeout(tenantId, entityId, keys);
}
callback.onSuccess();
}

36
application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java

@ -33,7 +33,7 @@ import org.springframework.stereotype.Service;
import org.springframework.web.socket.CloseStatus;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.alarm.AlarmInfo;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult;
@ -41,7 +41,6 @@ import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.query.AlarmDataQuery;
import org.thingsboard.server.common.data.query.ComparisonTsValue;
import org.thingsboard.server.common.data.query.OriginatorAlarmFilter;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityKey;
@ -55,17 +54,16 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.executors.DbCallbackExecutorService;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.ws.WebSocketService;
import org.thingsboard.server.service.ws.WebSocketSessionRef;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.AggHistoryCmd;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.AggKey;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.AggTimeSeriesCmd;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmCountCmd;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmCountUpdate;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataCmd;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataUpdate;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmStatusCmd;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.CmdUpdate;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityCountCmd;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataCmd;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataUpdate;
@ -74,7 +72,6 @@ import org.thingsboard.server.service.ws.telemetry.cmd.v2.GetTsCmd;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.LatestValueCmd;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.TimeSeriesCmd;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.UnsubscribeCmd;
import org.thingsboard.server.service.ws.telemetry.sub.AlarmSubscriptionUpdate;
import java.util.ArrayList;
import java.util.Arrays;
@ -83,7 +80,6 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
@ -430,13 +426,25 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc
long start = System.currentTimeMillis();
ctx.fetchData();
long end = System.currentTimeMillis();
stats.getAlarmQueryInvocationCnt().incrementAndGet();
stats.getAlarmQueryTimeSpent().addAndGet(end - start);
TbAlarmCountSubCtx finalCtx = ctx;
ScheduledFuture<?> task = scheduler.scheduleWithFixedDelay(
() -> refreshDynamicQuery(finalCtx),
dynamicPageLinkRefreshInterval, dynamicPageLinkRefreshInterval, TimeUnit.SECONDS);
finalCtx.setRefreshTask(task);
stats.getRegularQueryInvocationCnt().incrementAndGet();
stats.getRegularQueryTimeSpent().addAndGet(end - start);
Set<EntityId> entitiesIds = ctx.getEntitiesIds();
ctx.cancelTasks();
ctx.clearAlarmSubscriptions();
if (entitiesIds != null && entitiesIds.isEmpty()) {
AlarmCountUpdate update = new AlarmCountUpdate(cmd.getCmdId(), 0);
ctx.sendWsMsg(update);
} else {
ctx.doFetchAlarmCount();
if (entitiesIds != null) {
ctx.createAlarmSubscriptions();
}
TbAlarmCountSubCtx finalCtx = ctx;
ScheduledFuture<?> task = scheduler.scheduleWithFixedDelay(
() -> refreshDynamicQuery(finalCtx),
dynamicPageLinkRefreshInterval, dynamicPageLinkRefreshInterval, TimeUnit.SECONDS);
finalCtx.setRefreshTask(task);
}
} else {
log.debug("[{}][{}] Received duplicate command: {}", session.getSessionId(), cmd.getCmdId(), cmd);
}
@ -555,7 +563,7 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc
private TbAlarmCountSubCtx createSubCtx(WebSocketSessionRef sessionRef, AlarmCountCmd cmd) {
Map<Integer, TbAbstractSubCtx> sessionSubs = subscriptionsBySessionId.computeIfAbsent(sessionRef.getSessionId(), k -> new ConcurrentHashMap<>());
TbAlarmCountSubCtx ctx = new TbAlarmCountSubCtx(serviceId, wsService, entityService, localSubscriptionService,
attributesService, stats, alarmService, sessionRef, cmd.getCmdId());
attributesService, stats, alarmService, sessionRef, cmd.getCmdId(), maxEntitiesPerAlarmSubscription, maxAlarmQueriesPerRefreshInterval);
if (cmd.getQuery() != null) {
ctx.setAndResolveQuery(cmd.getQuery());
}

119
application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java

@ -19,49 +19,152 @@ import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.query.AlarmCountQuery;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataPageLink;
import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityDataSortOrder;
import org.thingsboard.server.common.data.query.EntityKey;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.dao.alarm.AlarmService;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.service.ws.WebSocketService;
import org.thingsboard.server.service.ws.WebSocketSessionRef;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmCountUpdate;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@Slf4j
@ToString(callSuper = true)
public class TbAlarmCountSubCtx extends TbAbstractEntityQuerySubCtx<AlarmCountQuery> {
private final AlarmService alarmService;
protected final Map<Integer, EntityId> subToEntityIdMap;
@Getter
private LinkedHashSet<EntityId> entitiesIds;
private final int maxEntitiesPerAlarmSubscription;
private final int maxAlarmQueriesPerRefreshInterval;
@Getter
@Setter
private volatile int result;
@Getter
@Setter
private boolean tooManyEntities;
private int alarmCountInvocationAttempts;
public TbAlarmCountSubCtx(String serviceId, WebSocketService wsService,
EntityService entityService, TbLocalSubscriptionService localSubscriptionService,
AttributesService attributesService, SubscriptionServiceStatistics stats, AlarmService alarmService,
WebSocketSessionRef sessionRef, int cmdId) {
WebSocketSessionRef sessionRef, int cmdId, int maxEntitiesPerAlarmSubscription, int maxAlarmQueriesPerRefreshInterval) {
super(serviceId, wsService, entityService, localSubscriptionService, attributesService, stats, sessionRef, cmdId);
this.alarmService = alarmService;
this.subToEntityIdMap = new ConcurrentHashMap<>();
this.maxEntitiesPerAlarmSubscription = maxEntitiesPerAlarmSubscription;
this.maxAlarmQueriesPerRefreshInterval = maxAlarmQueriesPerRefreshInterval;
this.entitiesIds = null;
}
@Override
public void clearSubscriptions() {
clearAlarmSubscriptions();
}
@Override
public void fetchData() {
result = (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query);
sendWsMsg(new AlarmCountUpdate(cmdId, result));
resetInvocationCounter();
if (query.getEntityFilter() != null) {
entitiesIds = new LinkedHashSet<>();
log.trace("[{}] Fetching data: {}", cmdId, alarmCountInvocationAttempts);
PageData<EntityData> data = entityService.findEntityDataByQuery(getTenantId(), getCustomerId(), buildEntityDataQuery());
entitiesIds.clear();
tooManyEntities = data.hasNext();
for (EntityData entityData : data.getData()) {
entitiesIds.add(entityData.getEntityId());
}
}
}
@Override
protected void update() {
int newCount = (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query);
if (newCount != result) {
result = newCount;
sendWsMsg(new AlarmCountUpdate(cmdId, result));
}
resetInvocationCounter();
fetchAlarmCount();
}
@Override
public boolean isDynamic() {
return true;
}
public void fetchAlarmCount() {
alarmCountInvocationAttempts++;
log.trace("[{}] Fetching alarms: {}", cmdId, alarmCountInvocationAttempts);
if (alarmCountInvocationAttempts <= maxAlarmQueriesPerRefreshInterval) {
int newCount = (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query, entitiesIds);
if (newCount != result) {
result = newCount;
sendWsMsg(new AlarmCountUpdate(cmdId, result));
}
} else {
log.trace("[{}] Ignore alarm count fetch due to rate limit: [{}] of maximum [{}]", cmdId, alarmCountInvocationAttempts, maxAlarmQueriesPerRefreshInterval);
}
}
public void doFetchAlarmCount() {
result = (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query, entitiesIds);
sendWsMsg(new AlarmCountUpdate(cmdId, result));
}
private EntityDataQuery buildEntityDataQuery() {
EntityDataPageLink edpl = new EntityDataPageLink(maxEntitiesPerAlarmSubscription, 0, null,
new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, ModelConstants.CREATED_TIME_PROPERTY)));
return new EntityDataQuery(query.getEntityFilter(), edpl, null, null, query.getKeyFilters());
}
private void resetInvocationCounter() {
alarmCountInvocationAttempts = 0;
}
public void createAlarmSubscriptions() {
for (EntityId entityId : entitiesIds) {
createAlarmSubscriptionForEntity(entityId);
}
}
private void createAlarmSubscriptionForEntity(EntityId entityId) {
int subIdx = sessionRef.getSessionSubIdSeq().incrementAndGet();
subToEntityIdMap.put(subIdx, entityId);
log.trace("[{}][{}][{}] Creating alarms subscription for [{}] ", serviceId, cmdId, subIdx, entityId);
TbAlarmsSubscription subscription = TbAlarmsSubscription.builder()
.serviceId(serviceId)
.sessionId(sessionRef.getSessionId())
.subscriptionId(subIdx)
.tenantId(sessionRef.getSecurityCtx().getTenantId())
.entityId(entityId)
.updateProcessor((sub, update) -> fetchAlarmCount())
.build();
localSubscriptionService.addSubscription(subscription, sessionRef);
}
public void clearAlarmSubscriptions() {
if (subToEntityIdMap != null) {
for (Integer subId : subToEntityIdMap.keySet()) {
localSubscriptionService.cancelSubscription(getTenantId(), getSessionId(), subId);
}
subToEntityIdMap.clear();
}
}
}

3
application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java

@ -39,6 +39,7 @@ import org.thingsboard.server.dao.alarm.AlarmService;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.sql.query.EntityKeyMapping;
import org.thingsboard.server.service.ws.WebSocketService;
import org.thingsboard.server.service.ws.WebSocketSessionRef;
import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataUpdate;
@ -359,7 +360,7 @@ public class TbAlarmDataSubCtx extends TbAbstractDataSubCtx<AlarmDataQuery> {
EntityDataSortOrder sortOrder = query.getPageLink().getSortOrder();
EntityDataSortOrder entitiesSortOrder;
if (sortOrder == null || sortOrder.getKey().getType().equals(EntityKeyType.ALARM_FIELD)) {
entitiesSortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, ModelConstants.CREATED_TIME_PROPERTY));
entitiesSortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, EntityKeyMapping.CREATED_TIME));
} else {
entitiesSortOrder = sortOrder;
}

5
application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java

@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.common.data.util.ThrowingRunnable;
import org.thingsboard.server.dao.cf.CalculatedFieldService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.queue.util.TbCoreComponent;
@ -61,6 +62,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS
private final Map<EntityType, EntityImportService<?, ?, ?>> importServices = new HashMap<>();
private final RelationService relationService;
private final CalculatedFieldService calculatedFieldService;
private final RateLimitService rateLimitService;
private final TbLogEntityActionService logEntityActionService;
@ -72,7 +74,6 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS
EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_RULE
);
@Override
public <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> exportEntity(EntitiesExportCtx<?> ctx, I entityId) throws ThingsboardException {
if (!rateLimitService.checkRateLimit(LimitedApi.ENTITY_EXPORT, ctx.getTenantId())) {
@ -129,13 +130,11 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS
}
}
@Override
public Comparator<EntityType> getEntityTypeComparatorForImport() {
return Comparator.comparing(SUPPORTED_ENTITY_TYPES::indexOf);
}
@SuppressWarnings("unchecked")
private <I extends EntityId, E extends ExportableEntity<I>, D extends EntityExportData<E>> EntityExportService<I, E, D> getExportService(EntityType entityType) {
EntityExportService<?, ?, ?> exportService = exportServices.get(entityType);

1
application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java

@ -32,7 +32,6 @@ public interface EntitiesExportImportService {
<E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> importEntity(EntitiesImportCtx ctx, EntityExportData<E> exportData) throws ThingsboardException;
void saveReferencesAndRelations(EntitiesImportCtx ctx) throws ThingsboardException;
Comparator<EntityType> getEntityTypeComparatorForImport();

21
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java

@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.HasVersion;
import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
@ -31,6 +32,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.common.data.sync.ie.AttributeExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.cf.CalculatedFieldService;
import org.thingsboard.server.dao.relation.RelationDao;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.ie.exporting.EntityExportService;
@ -59,6 +61,8 @@ public class DefaultEntityExportService<I extends EntityId, E extends Exportable
private RelationDao relationDao;
@Autowired
private AttributesService attributesService;
@Autowired
private CalculatedFieldService calculatedFieldService;
@Override
public final D getExportData(EntitiesExportCtx<?> ctx, I entityId) throws ThingsboardException {
@ -98,6 +102,10 @@ public class DefaultEntityExportService<I extends EntityId, E extends Exportable
Map<String, List<AttributeExportData>> attributes = exportAttributes(ctx, entity);
exportData.setAttributes(attributes);
}
if (ctx.getSettings().isExportCalculatedFields()) {
List<CalculatedField> calculatedFields = exportCalculatedFields(ctx, entity.getId());
exportData.setCalculatedFields(calculatedFields);
}
}
private List<EntityRelation> exportRelations(EntitiesExportCtx<?> ctx, E entity) throws ThingsboardException {
@ -141,6 +149,19 @@ public class DefaultEntityExportService<I extends EntityId, E extends Exportable
return attributes;
}
private List<CalculatedField> exportCalculatedFields(EntitiesExportCtx<?> ctx, EntityId entityId) {
List<CalculatedField> calculatedFields = calculatedFieldService.findCalculatedFieldsByEntityId(ctx.getTenantId(), entityId);
calculatedFields.forEach(calculatedField -> {
calculatedField.setEntityId(getExternalIdOrElseInternal(ctx, entityId));
calculatedField.getConfiguration().getArguments().values().forEach(argument -> {
if (argument.getRefEntityId() != null) {
argument.setRefEntityId(getExternalIdOrElseInternal(ctx, argument.getRefEntityId()));
}
});
});
return calculatedFields;
}
protected <ID extends EntityId> ID getExternalIdOrElseInternal(EntitiesExportCtx<?> ctx, ID internalId) {
if (internalId == null || internalId.isNullUid()) return internalId;
var result = ctx.getExternalId(internalId);

6
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java

@ -47,7 +47,11 @@ public class AssetImportService extends BaseEntityImportService<AssetId, Asset,
@Override
protected Asset saveOrUpdate(EntitiesImportCtx ctx, Asset asset, EntityExportData<Asset> exportData, IdProvider idProvider) {
return assetService.saveAsset(asset);
Asset savedAsset = assetService.saveAsset(asset);
if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) {
importCalculatedFields(ctx, savedAsset, exportData, idProvider);
}
return savedAsset;
}
@Override

6
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java

@ -50,7 +50,11 @@ public class AssetProfileImportService extends BaseEntityImportService<AssetProf
@Override
protected AssetProfile saveOrUpdate(EntitiesImportCtx ctx, AssetProfile assetProfile, EntityExportData<AssetProfile> exportData, IdProvider idProvider) {
return assetProfileService.saveAssetProfile(assetProfile);
AssetProfile saved = assetProfileService.saveAssetProfile(assetProfile);
if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) {
importCalculatedFields(ctx, saved, exportData, idProvider);
}
return saved;
}
@Override

57
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java

@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.HasDefaultOption;
import org.thingsboard.server.common.data.HasVersion;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
@ -50,6 +51,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.common.data.sync.ie.AttributeExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.dao.cf.CalculatedFieldService;
import org.thingsboard.server.dao.relation.RelationDao;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.service.action.EntityActionService;
@ -78,6 +80,8 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
@Lazy
private ExportableEntitiesService entitiesService;
@Autowired
private CalculatedFieldService calculatedFieldService;
@Autowired
private RelationService relationService;
@Autowired
private RelationDao relationDao;
@ -134,7 +138,7 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
}
protected boolean updateRelatedEntitiesIfUnmodified(EntitiesImportCtx ctx, E prepared, D exportData, IdProvider idProvider) {
return false;
return importCalculatedFields(ctx, prepared, exportData, idProvider);
}
@Override
@ -278,12 +282,59 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
});
}
protected boolean importCalculatedFields(EntitiesImportCtx ctx, E savedEntity, D exportData, IdProvider idProvider) {
if (exportData.getCalculatedFields() == null || !ctx.isSaveCalculatedFields()) {
return false;
}
boolean updated = false;
List<CalculatedField> existing = calculatedFieldService.findCalculatedFieldsByEntityId(ctx.getTenantId(), savedEntity.getId());
List<CalculatedField> fieldsToSave = exportData.getCalculatedFields().stream()
.peek(calculatedField -> {
calculatedField.setTenantId(ctx.getTenantId());
calculatedField.setEntityId(savedEntity.getId());
calculatedField.getConfiguration().getArguments().values().forEach(argument -> {
if (argument.getRefEntityId() != null) {
argument.setRefEntityId(idProvider.getInternalId(argument.getRefEntityId(), ctx.isFinalImportAttempt()));
}
});
}).toList();
for (CalculatedField existingField : existing) {
boolean found = fieldsToSave.stream().anyMatch(importedField -> compareCalculatedFields(existingField, importedField));
if (!found) {
calculatedFieldService.deleteCalculatedField(ctx.getTenantId(), existingField.getId());
updated = true;
}
}
for (CalculatedField calculatedField : fieldsToSave) {
boolean found = existing.stream().anyMatch(existingField -> compareCalculatedFields(existingField, calculatedField));
if (!found) {
calculatedFieldService.save(calculatedField);
updated = true;
}
}
return updated;
}
private boolean compareCalculatedFields(CalculatedField existingField, CalculatedField newField) {
CalculatedField oldCopy = new CalculatedField(existingField);
CalculatedField newCopy = new CalculatedField(newField);
oldCopy.setId(null);
newCopy.setId(null);
oldCopy.setVersion(null);
newCopy.setVersion(null);
oldCopy.setCreatedTime(0);
newCopy.setCreatedTime(0);
return oldCopy.equals(newCopy);
}
protected void onEntitySaved(User user, E savedEntity, E oldEntity) throws ThingsboardException {
logEntityActionService.logEntityAction(user.getTenantId(), savedEntity.getId(), savedEntity, null,
oldEntity == null ? ActionType.ADDED : ActionType.UPDATED, user);
}
@SuppressWarnings("unchecked")
protected E findExistingEntity(EntitiesImportCtx ctx, E entity, IdProvider idProvider) {
return (E) Optional.ofNullable(entitiesService.findEntityByTenantIdAndExternalId(ctx.getTenantId(), entity.getId()))
@ -313,10 +364,10 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
.orElseThrow(() -> new MissingEntityException(externalId));
}
@SuppressWarnings("unchecked")
@RequiredArgsConstructor
protected class IdProvider {
private final EntitiesImportCtx ctx;
private final EntityImportResult<E> importResult;

9
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java

@ -64,13 +64,18 @@ public class DeviceImportService extends BaseEntityImportService<DeviceId, Devic
@Override
protected Device saveOrUpdate(EntitiesImportCtx ctx, Device device, DeviceExportData exportData, IdProvider idProvider) {
Device savedDevice;
if (exportData.getCredentials() != null && ctx.isSaveCredentials()) {
exportData.getCredentials().setId(null);
exportData.getCredentials().setDeviceId(null);
return deviceService.saveDeviceWithCredentials(device, exportData.getCredentials());
savedDevice = deviceService.saveDeviceWithCredentials(device, exportData.getCredentials());
} else {
return deviceService.saveDevice(device);
savedDevice = deviceService.saveDevice(device);
}
if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) {
importCalculatedFields(ctx, savedDevice, exportData, idProvider);
}
return savedDevice;
}
@Override

6
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java

@ -52,7 +52,11 @@ public class DeviceProfileImportService extends BaseEntityImportService<DevicePr
@Override
protected DeviceProfile saveOrUpdate(EntitiesImportCtx ctx, DeviceProfile deviceProfile, EntityExportData<DeviceProfile> exportData, IdProvider idProvider) {
return deviceProfileService.saveDeviceProfile(deviceProfile);
DeviceProfile saved = deviceProfileService.saveDeviceProfile(deviceProfile);
if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) {
importCalculatedFields(ctx, saved, exportData, idProvider);
}
return saved;
}
@Override

10
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java

@ -94,7 +94,6 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.function.Function;
import java.util.stream.Collectors;
import static com.google.common.util.concurrent.Futures.transform;
import static org.thingsboard.server.common.data.sync.vc.VcUtils.checkBranchName;
@ -304,6 +303,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
.updateRelations(config.isLoadRelations())
.saveAttributes(config.isLoadAttributes())
.saveCredentials(config.isLoadCredentials())
.saveCalculatedFields(config.isLoadCalculatedFields())
.findExistingByName(false)
.build());
ctx.setFinalImportAttempt(true);
@ -327,7 +327,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
var sw = TbStopWatch.create("before");
List<EntityType> entityTypes = request.getEntityTypes().keySet().stream()
.sorted(exportImportService.getEntityTypeComparatorForImport()).collect(Collectors.toList());
.sorted(exportImportService.getEntityTypeComparatorForImport()).toList();
for (EntityType entityType : entityTypes) {
log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType);
sw.startNew("Entities " + entityType.name());
@ -362,6 +362,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
.updateRelations(config.isLoadRelations())
.saveAttributes(config.isLoadAttributes())
.saveCredentials(config.isLoadCredentials())
.saveCalculatedFields(config.isLoadCalculatedFields())
.findExistingByName(config.isFindExistingEntityByName())
.build();
}
@ -471,7 +472,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
}
@Override
public ListenableFuture<EntityDataDiff> compareEntityDataToVersion(User user, EntityId entityId, String versionId) throws Exception {
public ListenableFuture<EntityDataDiff> compareEntityDataToVersion(User user, EntityId entityId, String versionId) {
HasId<EntityId> entity = exportableEntitiesService.findEntityByTenantIdAndId(user.getTenantId(), entityId);
if (!(entity instanceof ExportableEntity)) throw new IllegalArgumentException("Unsupported entity type");
@ -484,6 +485,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
.exportRelations(otherVersion.hasRelations())
.exportAttributes(otherVersion.hasAttributes())
.exportCredentials(otherVersion.hasCredentials())
.exportCalculatedFields(otherVersion.hasCalculatedFields())
.build());
EntityExportData<?> currentVersion;
try {
@ -503,7 +505,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
@Override
public ListenableFuture<List<BranchInfo>> listBranches(TenantId tenantId) throws Exception {
public ListenableFuture<List<BranchInfo>> listBranches(TenantId tenantId) {
return gitServiceQueue.listBranches(tenantId);
}

2
application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java

@ -69,6 +69,7 @@ public abstract class EntitiesExportCtx<R extends VersionCreateRequest> {
.exportRelations(config.isSaveRelations())
.exportAttributes(config.isSaveAttributes())
.exportCredentials(config.isSaveCredentials())
.exportCalculatedFields(config.isSaveCalculatedFields())
.build();
}
@ -85,4 +86,5 @@ public abstract class EntitiesExportCtx<R extends VersionCreateRequest> {
log.debug("[{}][{}] Local cache put: {}", internalId.getEntityType(), internalId.getId(), externalId);
externalIdMap.put(internalId, externalId != null ? externalId : internalId);
}
}

5
application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java

@ -91,6 +91,10 @@ public class EntitiesImportCtx {
return getSettings().isSaveCredentials();
}
public boolean isSaveCalculatedFields() {
return getSettings().isSaveCalculatedFields();
}
public EntityId getInternalId(EntityId externalId) {
var result = externalToInternalIdMap.get(externalId);
log.debug("[{}][{}] Local cache {} for id", externalId.getEntityType(), externalId.getId(), result != null ? "hit" : "miss");
@ -140,5 +144,4 @@ public class EntitiesImportCtx {
return notFoundIds.contains(externalId);
}
}

1
application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java

@ -39,6 +39,7 @@ public class EntityTypeExportCtx extends EntitiesExportCtx<VersionCreateRequest>
.exportRelations(config.isSaveRelations())
.exportAttributes(config.isSaveAttributes())
.exportCredentials(config.isSaveCredentials())
.exportCalculatedFields(config.isSaveCalculatedFields())
.build();
this.overwrite = ObjectUtils.defaultIfNull(config.getSyncStrategy(), defaultSyncStrategy) == SyncStrategy.OVERWRITE;
}

2
application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java

@ -370,7 +370,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
}
private <S> void addMainCallback(ListenableFuture<S> saveFuture, Consumer<S> onSuccess) {
DonAsynchron.withCallback(saveFuture, onSuccess, null, tsCallBackExecutor);
addMainCallback(saveFuture, onSuccess, null);
}
private <S> void addMainCallback(ListenableFuture<S> saveFuture, Consumer<S> onSuccess, Consumer<Throwable> onFailure) {

3
application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java

@ -93,7 +93,8 @@ public class TbCoreTransportApiService {
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
log.info("Received application ready event. Starting polling for events.");
transportApiTemplate.init(transportApiService);
transportApiTemplate.subscribe();
transportApiTemplate.launch(transportApiService);
}
@PreDestroy

1
application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java

@ -229,6 +229,7 @@ public class DefaultWebSocketService implements WebSocketService {
} catch (TbRateLimitsException e) {
log.debug("{} Failed to handle WS cmd: {}", sessionRef, cmd, e);
} catch (Exception e) {
sendError(sessionRef, cmd.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR, e.getMessage());
log.error("{} Failed to handle WS cmd: {}", sessionRef, cmd, e);
}
}

66
application/src/main/resources/thingsboard.yml

@ -512,6 +512,8 @@ actors:
enabled: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_ENABLED:true}"
# The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour
configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}"
# Time in seconds to receive calculation result.
calculation_timeout: "${ACTORS_CALCULATION_TIMEOUT_SEC:5}"
debug:
settings:
@ -1597,6 +1599,16 @@ queue:
- key: max.poll.records
# Amount of records to be returned in a single poll. For Housekeeper reprocessing topic, we should consume messages (tasks) one by one
value: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_MAX_POLL_RECORDS:1}"
edqs.events:
# Key-value properties for Kafka consumer for edqs.events topic
- key: max.poll.records
# Max poll records for edqs.events topic
value: "${TB_QUEUE_KAFKA_EDQS_EVENTS_MAX_POLL_RECORDS:512}"
edqs.state:
# Key-value properties for Kafka consumer for edqs.state topic
- key: max.poll.records
# Max poll records for edqs.state topic
value: "${TB_QUEUE_KAFKA_EDQS_STATE_MAX_POLL_RECORDS:512}"
other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000"
other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside
# - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms
@ -1630,6 +1642,12 @@ queue:
calculated-field: "${TB_QUEUE_KAFKA_CF_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}"
# Kafka properties for Calculated Field State topics
calculated-field-state: "${TB_QUEUE_KAFKA_CF_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:104857600000;partitions:1;min.insync.replicas:1;cleanup.policy:compact}"
# Kafka properties for EDQS events topics. Partitions number must be the same as queue.edqs.partitions
edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1}"
# Kafka properties for EDQS requests topic (default: 3 minutes retention). Partitions number must be the same as queue.edqs.partitions
edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}"
# Kafka properties for EDQS state topic (infinite retention, compaction). Partitions number must be the same as queue.edqs.partitions
edqs-state: "${TB_QUEUE_KAFKA_EDQS_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1;cleanup.policy:compact}"
consumer-stats:
# Prints lag between consumer group offset and last messages offset in Kafka topics
enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}"
@ -1659,6 +1677,8 @@ queue:
core:
# Default topic name
topic: "${TB_QUEUE_CORE_TOPIC:tb_core}"
# For high-priority notifications that require minimum latency and processing time
notifications_topic: "${TB_QUEUE_CORE_NOTIFICATIONS_TOPIC:tb_core.notifications}"
# Interval in milliseconds to poll messages by Core microservices
poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}"
# Amount of partitions used by Core microservices
@ -1703,6 +1723,44 @@ queue:
enabled: "${TB_HOUSEKEEPER_STATS_ENABLED:true}"
# Statistics printing interval for Housekeeper
print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}"
edqs:
sync:
# Enable/disable EDQS synchronization
enabled: "${TB_EDQS_SYNC_ENABLED:false}"
# Batch size of entities being synced with EDQS
entity_batch_size: "${TB_EDQS_SYNC_ENTITY_BATCH_SIZE:10000}"
# Batch size of timeseries data being synced with EDQS
ts_batch_size: "${TB_EDQS_SYNC_TS_BATCH_SIZE:10000}"
api:
# Whether to forward entity data query requests to EDQS (otherwise use PostgreSQL implementation)
supported: "${TB_EDQS_API_SUPPORTED:false}"
# Whether to auto-enable EDQS API (if queue.edqs.api.supported is true) when sync of data to Kafka is finished
auto_enable: "${TB_EDQS_API_AUTO_ENABLE:true}"
# Mode of EDQS: local (for monolith) or remote (with separate EDQS microservices)
mode: "${TB_EDQS_MODE:local}"
local:
# Path to RocksDB for EDQS backup when running in local mode
rocksdb_path: "${TB_EDQS_ROCKSDB_PATH:${user.home}/.rocksdb/edqs}"
# Number of partitions for EDQS topics
partitions: "${TB_EDQS_PARTITIONS:12}"
# EDQS partitioning strategy: tenant (partition is resolved by tenant id) or none (no specific strategy, resolving by message key)
partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}"
# EDQS requests topic
requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}"
# EDQS responses topic
responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}"
# Poll interval for EDQS topics
poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}"
# Maximum amount of pending requests to EDQS
max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}"
# Maximum timeout for requests to EDQS
max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:20000}"
stats:
# Enable/disable statistics for EDQS
enabled: "${TB_EDQS_STATS_ENABLED:true}"
# Statistics printing interval for EDQS
print-interval-ms: "${TB_EDQS_STATS_PRINT_INTERVAL_MS:300000}"
vc:
# Default topic name
topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}"
@ -1734,6 +1792,8 @@ queue:
rule-engine:
# Deprecated. It will be removed in the nearest releases
topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}"
# For high-priority notifications that require minimum latency and processing time
notifications_topic: "${TB_QUEUE_RULE_ENGINE_NOTIFICATIONS_TOPIC:tb_rule_engine.notifications}"
# Interval in milliseconds to poll messages by Rule Engine
poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}"
# Timeout for processing a message pack of Rule Engine
@ -1754,6 +1814,8 @@ queue:
event_topic: "${TB_QUEUE_CF_EVENT_TOPIC:tb_cf_event}"
# Topic name for Calculated Field (CF) compacted states
state_topic: "${TB_QUEUE_CF_STATE_TOPIC:tb_cf_state}"
# For high-priority notifications that require minimum latency and processing time
notifications_topic: "${TB_QUEUE_CF_NOTIFICATIONS_TOPIC:calculated_field.notifications}"
# Interval in milliseconds to poll messages by CF (Rule Engine) microservices
poll_interval: "${TB_QUEUE_CF_POLL_INTERVAL_MS:25}"
# Amount of partitions used by CF microservices
@ -1772,6 +1834,10 @@ queue:
edge:
# Default topic name
topic: "${TB_QUEUE_EDGE_TOPIC:tb_edge}"
# For high-priority notifications that require minimum latency and processing time
notifications_topic: "${TB_QUEUE_EDGE_NOTIFICATIONS_TOPIC:tb_edge.notifications}"
# For edge events messages
event_notifications_topic: "${TB_QUEUE_EDGE_EVENT_NOTIFICATIONS_TOPIC:tb_edge_event.notifications}"
# Amount of partitions used by Edge services
partitions: "${TB_QUEUE_EDGE_PARTITIONS:10}"
# Poll interval for topics related to Edge services

2
application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java

@ -402,7 +402,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
} catch (Exception e) {
throw new RuntimeException(e);
}
Awaitility.await("all tasks processed").atMost(60, TimeUnit.SECONDS).during(300, TimeUnit.MILLISECONDS)
Awaitility.await("all tasks processed").atMost(90, TimeUnit.SECONDS).during(300, TimeUnit.MILLISECONDS)
.until(() -> storage.getLag("tb_housekeeper") == 0);
}

72
application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java

@ -0,0 +1,72 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.controller;
import org.junit.Before;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.TestPropertySource;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.query.EntityCountQuery;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.msg.edqs.EdqsApiService;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.edqs.state.EdqsStateService;
import org.thingsboard.server.edqs.util.EdqsRocksDb;
import java.util.concurrent.TimeUnit;
import static org.awaitility.Awaitility.await;
@DaoSqlTest
@TestPropertySource(properties = {
// "queue.type=kafka", // uncomment to use Kafka
// "queue.kafka.bootstrap.servers=10.7.1.254:9092",
"queue.edqs.sync.enabled=true",
"queue.edqs.api.supported=true",
"queue.edqs.api.auto_enable=true",
"queue.edqs.mode=local"
})
public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest {
@Autowired
private EdqsApiService edqsApiService;
@Autowired
private EdqsStateService edqsStateService;
@MockBean // so that we don't do backup for tests
private EdqsRocksDb edqsRocksDb;
@Before
public void before() {
await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsApiService.isEnabled() && edqsStateService.isReady());
}
@Override
protected PageData<EntityData> findByQueryAndCheck(EntityDataQuery query, int expectedResultSize) {
return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findByQuery(query),
result -> result.getTotalElements() == expectedResultSize);
}
@Override
protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult) {
return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(query),
result -> result == expectedResult);
}
}

198
application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java

@ -18,12 +18,14 @@ package org.thingsboard.server.controller;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import org.awaitility.Awaitility;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.web.servlet.ResultActions;
import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.Dashboard;
@ -49,6 +51,7 @@ import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityDataSortOrder;
import org.thingsboard.server.common.data.query.EntityKey;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.EntityKeyValueType;
import org.thingsboard.server.common.data.query.EntityListFilter;
import org.thingsboard.server.common.data.query.EntityTypeFilter;
import org.thingsboard.server.common.data.query.FilterPredicateValue;
@ -70,9 +73,11 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.function.BiPredicate;
import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@DaoSqlTest
@ -130,36 +135,25 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
filter.setDeviceNameFilter("");
EntityCountQuery countQuery = new EntityCountQuery(filter);
Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
Assert.assertEquals(97, count.longValue());
countByQueryAndCheck(countQuery, 97);
filter.setDeviceTypes(List.of("unknown"));
count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
Assert.assertEquals(0, count.longValue());
countByQueryAndCheck(countQuery, 0);
filter.setDeviceTypes(List.of("default"));
filter.setDeviceNameFilter("Device1");
count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
Assert.assertEquals(11, count.longValue());
countByQueryAndCheck(countQuery, 11);
EntityListFilter entityListFilter = new EntityListFilter();
entityListFilter.setEntityType(EntityType.DEVICE);
entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList()));
countQuery = new EntityCountQuery(entityListFilter);
count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
Assert.assertEquals(97, count.longValue());
countByQueryAndCheck(countQuery, 97);
EntityTypeFilter filter2 = new EntityTypeFilter();
filter2.setEntityType(EntityType.DEVICE);
EntityCountQuery countQuery2 = new EntityCountQuery(filter2);
Long count2 = doPostWithResponse("/api/entitiesQuery/count", countQuery2, Long.class);
Assert.assertEquals(97, count2.longValue());
countQuery = new EntityCountQuery(filter2);
countByQueryAndCheck(countQuery, 97);
}
@Test
@ -169,51 +163,44 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
EntityTypeFilter allDeviceFilter = new EntityTypeFilter();
allDeviceFilter.setEntityType(EntityType.DEVICE);
EntityCountQuery query = new EntityCountQuery(allDeviceFilter);
Long initialCount = doPostWithResponse("/api/entitiesQuery/count", query, Long.class);
countByQueryAndCheck(query, 0);
loginTenantAdmin();
List<Device> devices = new ArrayList<>();
String devicePrefix = "Device" + RandomStringUtils.randomAlphabetic(5);
for (int i = 0; i < 97; i++) {
Device device = new Device();
device.setName("Device" + i);
device.setName(devicePrefix + i);
device.setType("default");
device.setLabel("testLabel" + (int) (Math.random() * 1000));
devices.add(doPost("/api/device", device, Device.class));
Thread.sleep(1);
}
DeviceTypeFilter filter = new DeviceTypeFilter();
filter.setDeviceType("default");
filter.setDeviceTypes(List.of("default"));
filter.setDeviceNameFilter("");
loginSysAdmin();
EntityCountQuery countQuery = new EntityCountQuery(filter);
countByQueryAndCheck(countQuery, 97);
Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
Assert.assertEquals(97, count.longValue());
filter.setDeviceType("unknown");
count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
Assert.assertEquals(0, count.longValue());
filter.setDeviceType("default");
filter.setDeviceNameFilter("Device1");
filter.setDeviceTypes(List.of("unknown"));
countByQueryAndCheck(countQuery, 0);
count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
Assert.assertEquals(11, count.longValue());
filter.setDeviceTypes(List.of("default"));
filter.setDeviceNameFilter(devicePrefix + "1");
countByQueryAndCheck(countQuery, 11);
EntityListFilter entityListFilter = new EntityListFilter();
entityListFilter.setEntityType(EntityType.DEVICE);
entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList()));
countQuery = new EntityCountQuery(entityListFilter);
countByQueryAndCheck(countQuery, 97);
count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
Assert.assertEquals(97, count.longValue());
Long count2 = doPostWithResponse("/api/entitiesQuery/count", query, Long.class);
Assert.assertEquals(initialCount + 97, count2.longValue());
countByQueryAndCheck(countQuery, 97);
}
@Test
@ -371,11 +358,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
PageData<EntityData> data =
doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
});
Assert.assertEquals(97, data.getTotalElements());
PageData<EntityData> data = findByQueryAndCheck(query, 97);
Assert.assertEquals(10, data.getTotalPages());
Assert.assertTrue(data.hasNext());
Assert.assertEquals(10, data.getData().size());
@ -383,8 +366,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
List<EntityData> loadedEntities = new ArrayList<>(data.getData());
while (data.hasNext()) {
query = query.next();
data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
});
data = findByQuery(query);
loadedEntities.addAll(data.getData());
}
Assert.assertEquals(97, loadedEntities.size());
@ -406,8 +388,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
pageLink = new EntityDataPageLink(10, 0, "device1", sortOrder);
query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
});
data = findByQuery(query);
Assert.assertEquals(11, data.getTotalElements());
Assert.assertEquals("Device19", data.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue());
@ -423,9 +404,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
EntityDataQuery query2 = new EntityDataQuery(filter2, pageLink2, entityFields2, null, null);
PageData<EntityData> data2 =
doPostWithTypedResponse("/api/entitiesQuery/find", query2, new TypeReference<PageData<EntityData>>() {
});
PageData<EntityData> data2 = findByQuery(query2);
Assert.assertEquals(97, data2.getTotalElements());
Assert.assertEquals(10, data2.getTotalPages());
@ -473,20 +452,15 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
List<EntityKey> entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"));
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
PageData<EntityData> data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {});
Assert.assertEquals(87, data.getTotalElements());
findByQueryAndCheck(query, 87);
filter.setFilters(List.of(new RelationEntityTypeFilter("NOT_CONTAINS", List.of(EntityType.DEVICE), false)));
query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {});
Assert.assertEquals(10, data.getTotalElements());
findByQueryAndCheck(query, 10);
filter.setFilters(List.of(new RelationEntityTypeFilter("NOT_CONTAINS", List.of(EntityType.DEVICE), true)));
query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {});
Assert.assertEquals(87, data.getTotalElements());
findByQueryAndCheck(query, 87);
}
private EntityRelation createFromRelation(Device mainDevice, Device device, String relationType) {
@ -531,14 +505,12 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
List<EntityKey> latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature"));
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null);
PageData<EntityData> data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
});
PageData<EntityData> data = findByQueryAndCheck(query, 67);
List<EntityData> loadedEntities = new ArrayList<>(data.getData());
while (data.hasNext()) {
query = query.next();
data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
});
data = findByQuery(query);
loadedEntities.addAll(data.getData());
}
Assert.assertEquals(67, loadedEntities.size());
@ -551,6 +523,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
KeyFilter highTemperatureFilter = new KeyFilter();
highTemperatureFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature"));
highTemperatureFilter.setValueType(EntityKeyValueType.NUMERIC);
NumericFilterPredicate predicate = new NumericFilterPredicate();
predicate.setValue(FilterPredicateValue.fromDouble(45));
predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER);
@ -559,13 +532,11 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters);
data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
});
data = findByQuery(query);
loadedEntities = new ArrayList<>(data.getData());
while (data.hasNext()) {
query = query.next();
data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
});
data = findByQuery(query);
loadedEntities.addAll(data.getData());
}
Assert.assertEquals(highTemperatures.size(), loadedEntities.size());
@ -604,6 +575,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
KeyFilter highTemperatureFilter = new KeyFilter();
highTemperatureFilter.setKey(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, "alarmActiveTime"));
highTemperatureFilter.setValueType(EntityKeyValueType.NUMERIC);
NumericFilterPredicate predicate = new NumericFilterPredicate();
DynamicValue<Double> dynamicValue =
@ -627,16 +599,16 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters);
Awaitility.await()
await()
.alias("data by query")
.atMost(TIMEOUT, TimeUnit.SECONDS)
.until(() -> {
var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {});
var data = findByQuery(query);
var loadedEntities = new ArrayList<>(data.getData());
return loadedEntities.size() == numOfDevices;
});
var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {});
var data = findByQuery(query);
var loadedEntities = new ArrayList<>(data.getData());
Assert.assertEquals(numOfDevices, loadedEntities.size());
@ -694,11 +666,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
EntityDataQuery query = new EntityDataQuery(entityTypeFilter, pageLink, entityFields, null, null);
PageData<EntityData> data =
doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
});
Assert.assertEquals(97, data.getTotalElements());
PageData<EntityData> data = findByQueryAndCheck(query, 97);
Assert.assertEquals(10, data.getTotalPages());
Assert.assertTrue(data.hasNext());
Assert.assertEquals(10, data.getData().size());
@ -712,9 +680,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
});
EntityCountQuery countQuery = new EntityCountQuery(entityTypeFilter);
Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
Assert.assertEquals(97, count.longValue());
countByQueryAndCheck(countQuery, 97);
}
@Test
@ -742,28 +708,29 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
KeyFilter activeAlarmTimeToLongFilter = getServerAttributeNumericGreaterThanKeyFilter("alarmActiveTime", 30);
KeyFilter tenantOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", TEST_TENANT_NAME);
KeyFilter wrongOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", "wrongName");
KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT");
KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT");
KeyFilter customerOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "CUSTOMER");
// all devices with ownerName = TEST TENANT
EntityCountQuery query = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerNameFilter));
checkEntitiesCount(query, numOfDevices);
EntityCountQuery query = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerNameFilter));
await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(query),
result -> result == numOfDevices);
// all devices with ownerName = TEST TENANT
EntityCountQuery activeAlarmTimeToLongQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeToLongFilter, tenantOwnerNameFilter));
checkEntitiesCount(activeAlarmTimeToLongQuery, 0);
EntityCountQuery activeAlarmTimeToLongQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeToLongFilter, tenantOwnerNameFilter));
countByQueryAndCheck(activeAlarmTimeToLongQuery, 0);
// all devices with wrong ownerName
EntityCountQuery wrongTenantNameQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, wrongOwnerNameFilter));
checkEntitiesCount(wrongTenantNameQuery, 0);
countByQueryAndCheck(wrongTenantNameQuery, 0);
// all devices with owner type = TENANT
EntityCountQuery tenantEntitiesQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerTypeFilter));
checkEntitiesCount(tenantEntitiesQuery, numOfDevices);
countByQueryAndCheck(tenantEntitiesQuery, numOfDevices);
// all devices with owner type = CUSTOMER
EntityCountQuery customerEntitiesQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, customerOwnerTypeFilter));
checkEntitiesCount(customerEntitiesQuery, 0);
countByQueryAndCheck(customerEntitiesQuery, 0);
}
@Test
@ -790,7 +757,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
KeyFilter activeAlarmTimeFilter = getServerAttributeNumericGreaterThanKeyFilter("alarmActiveTime", 5);
KeyFilter tenantOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", TEST_TENANT_NAME);
KeyFilter wrongOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", "wrongName");
KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT");
KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT");
KeyFilter customerOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "CUSTOMER");
EntityDataSortOrder sortOrder = new EntityDataSortOrder(
@ -851,41 +818,29 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
PageData<EntityData> data =
doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
});
Assert.assertEquals(1, data.getTotalElements());
Assert.assertEquals(1, data.getTotalPages());
Assert.assertEquals(1, data.getData().size());
findByQueryAndCheck(query, 1);
// unnassign dashboard
login(TENANT_EMAIL, TENANT_PASSWORD);
doDelete("/api/customer/" + savedCustomer.getId().getId().toString() + "/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class);
login(CUSTOMER_USER_EMAIL, CUSTOMER_USER_PASSWORD);
PageData<EntityData> dataAfterUnassign =
doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
});
Assert.assertEquals(0, dataAfterUnassign.getTotalElements());
Assert.assertEquals(0, dataAfterUnassign.getTotalPages());
Assert.assertEquals(0, dataAfterUnassign.getData().size());
findByQueryAndCheck(query, 0);
}
private void checkEntitiesByQuery(EntityDataQuery query, int expectedNumOfDevices, String expectedOwnerName, String expectedOwnerType) throws Exception {
Awaitility.await()
await()
.alias("data by query")
.atMost(30, TimeUnit.SECONDS)
.until(() -> {
var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {});
var data = findByQuery(query);
var loadedEntities = new ArrayList<>(data.getData());
return loadedEntities.size() == expectedNumOfDevices;
});
if (expectedNumOfDevices == 0) {
return;
}
var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {});
if (expectedNumOfDevices == 0) {
return;
}
var data = findByQuery(query);
var loadedEntities = new ArrayList<>(data.getData());
Assert.assertEquals(expectedNumOfDevices, loadedEntities.size());
@ -898,25 +853,37 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
String alarmActiveTime = entity.getLatest().get(EntityKeyType.ATTRIBUTE).getOrDefault("alarmActiveTime", new TsValue(0, "-1")).getValue();
Assert.assertEquals("Device" + i, name);
Assert.assertEquals( expectedOwnerName, ownerName);
Assert.assertEquals( expectedOwnerType, ownerType);
Assert.assertEquals(expectedOwnerName, ownerName);
Assert.assertEquals(expectedOwnerType, ownerType);
Assert.assertEquals("1" + i, alarmActiveTime);
}
}
private void checkEntitiesCount(EntityCountQuery query, int expectedNumOfDevices) {
Awaitility.await()
.alias("count by query")
.atMost(30, TimeUnit.SECONDS)
.until(() -> {
var count = doPost("/api/entitiesQuery/count", query, Integer.class);
return count == expectedNumOfDevices;
});
}
protected PageData<EntityData> findByQuery(EntityDataQuery query) throws Exception {
return doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {
});
}
protected PageData<EntityData> findByQueryAndCheck(EntityDataQuery query, int expectedResultSize) throws Exception {
PageData<EntityData> result = findByQuery(query);
assertThat(result.getTotalElements()).isEqualTo(expectedResultSize);
return result;
}
protected Long countByQuery(EntityCountQuery countQuery) throws Exception {
return doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
}
protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult) throws Exception {
Long result = countByQuery(query);
assertThat(result).isEqualTo(expectedResult);
return result;
}
private KeyFilter getEntityFieldStringEqualToKeyFilter(String keyName, String value) {
KeyFilter tenantOwnerNameFilter = new KeyFilter();
tenantOwnerNameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, keyName));
tenantOwnerNameFilter.setValueType(EntityKeyValueType.STRING);
StringFilterPredicate ownerNamePredicate = new StringFilterPredicate();
ownerNamePredicate.setValue(FilterPredicateValue.fromString(value));
ownerNamePredicate.setOperation(StringFilterPredicate.StringOperation.EQUAL);
@ -927,6 +894,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
private KeyFilter getServerAttributeNumericGreaterThanKeyFilter(String attribute, int value) {
KeyFilter numericFilter = new KeyFilter();
numericFilter.setKey(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, attribute));
numericFilter.setValueType(EntityKeyValueType.NUMERIC);
NumericFilterPredicate predicate = new NumericFilterPredicate();
predicate.setValue(FilterPredicateValue.fromDouble(value));
predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER);

85
application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java

@ -83,7 +83,8 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
@Slf4j
@DaoSqlTest
@TestPropertySource(properties = {
"server.ws.alarms_per_alarm_status_subscription_cache_size=5"
"server.ws.alarms_per_alarm_status_subscription_cache_size=5",
"server.ws.dynamic_page_link.refresh_interval=15"
})
public class WebsocketApiTest extends AbstractControllerTest {
@Autowired
@ -324,6 +325,83 @@ public class WebsocketApiTest extends AbstractControllerTest {
Assert.assertEquals(1, update.getCount());
}
@Test
public void testAlarmCountWsCmdWithSingleEntityFilter() throws Exception {
loginTenantAdmin();
SingleEntityFilter singleEntityFilter = new SingleEntityFilter();
singleEntityFilter.setSingleEntity(tenantId);
AlarmCountQuery alarmCountQuery = new AlarmCountQuery(singleEntityFilter);
AlarmCountCmd cmd1 = new AlarmCountCmd(1, alarmCountQuery);
getWsClient().send(cmd1);
AlarmCountUpdate update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply());
Assert.assertEquals(1, update.getCmdId());
Assert.assertEquals(0, update.getCount());
//create alarm, check count = 1
getWsClient().registerWaitForUpdate();
Alarm alarm = new Alarm();
alarm.setOriginator(tenantId);
alarm.setType("TEST ALARM");
alarm.setSeverity(AlarmSeverity.WARNING);
alarm = doPost("/api/alarm", alarm, Alarm.class);
update = getWsClient().parseAlarmCountReply(getWsClient().waitForUpdate());
Assert.assertEquals(1, update.getCmdId());
Assert.assertEquals(1, update.getCount());
// set wrong entity id in filter, check count = 0
singleEntityFilter.setSingleEntity(tenantAdminUserId);
AlarmCountCmd cmd3 = new AlarmCountCmd(2, alarmCountQuery);
getWsClient().send(cmd3);
update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply());
Assert.assertEquals(2, update.getCmdId());
Assert.assertEquals(0, update.getCount());
}
@Test
public void testAlarmCountWsCmdWithDeviceType() throws Exception {
loginTenantAdmin();
DeviceTypeFilter deviceTypeFilter = new DeviceTypeFilter();
deviceTypeFilter.setDeviceTypes(List.of("default"));
AlarmCountQuery alarmCountQuery = new AlarmCountQuery(deviceTypeFilter);
AlarmCountCmd cmd1 = new AlarmCountCmd(1, alarmCountQuery);
getWsClient().send(cmd1);
AlarmCountUpdate update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply());
Assert.assertEquals(1, update.getCmdId());
Assert.assertEquals(0, update.getCount());
getWsClient().registerWaitForUpdate();
Alarm alarm = new Alarm();
alarm.setOriginator(device.getId());
alarm.setType("TEST ALARM");
alarm.setSeverity(AlarmSeverity.WARNING);
alarm = doPost("/api/alarm", alarm, Alarm.class);
update = getWsClient().parseAlarmCountReply(getWsClient().waitForUpdate());
Assert.assertEquals(1, update.getCmdId());
Assert.assertEquals(1, update.getCount());
deviceTypeFilter.setDeviceTypes(List.of("non-existing"));
AlarmCountCmd cmd3 = new AlarmCountCmd(3, alarmCountQuery);
getWsClient().send(cmd3);
update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply());
Assert.assertEquals(3, update.getCmdId());
Assert.assertEquals(0, update.getCount());
}
@Test
public void testAlarmStatusWsCmd() throws Exception {
loginTenantAdmin();
@ -372,17 +450,18 @@ public class WebsocketApiTest extends AbstractControllerTest {
doPost("/api/alarm", alarm2, Alarm.class);
AlarmStatusUpdate alarmStatusUpdate3 = JacksonUtil.fromString(getWsClient().waitForReply(), AlarmStatusUpdate.class);
AlarmStatusUpdate alarmStatusUpdate3 = JacksonUtil.fromString(getWsClient().waitForUpdate(), AlarmStatusUpdate.class);
Assert.assertEquals(1, alarmStatusUpdate3.getCmdId());
Assert.assertTrue(alarmStatusUpdate3.isActive());
//change severity
getWsClient().registerWaitForUpdate();
alarm2.setSeverity(AlarmSeverity.MAJOR);
Alarm updatedAlarm = doPost("/api/alarm", alarm2, Alarm.class);
Assert.assertNotNull(updatedAlarm);
Assert.assertEquals(AlarmSeverity.MAJOR, updatedAlarm.getSeverity());
AlarmStatusUpdate alarmStatusUpdate4 = JacksonUtil.fromString(getWsClient().waitForReply(), AlarmStatusUpdate.class);
AlarmStatusUpdate alarmStatusUpdate4 = JacksonUtil.fromString(getWsClient().waitForUpdate(), AlarmStatusUpdate.class);
Assert.assertEquals(1, alarmStatusUpdate4.getCmdId());
Assert.assertFalse(alarmStatusUpdate4.isActive());

49
application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java

@ -90,14 +90,12 @@ import org.thingsboard.server.gen.edge.v1.EdgeConfiguration;
import org.thingsboard.server.gen.edge.v1.OAuth2ClientUpdateMsg;
import org.thingsboard.server.gen.edge.v1.OAuth2DomainUpdateMsg;
import org.thingsboard.server.gen.edge.v1.QueueUpdateMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataRequestMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg;
import org.thingsboard.server.gen.edge.v1.SyncCompletedMsg;
import org.thingsboard.server.gen.edge.v1.TenantProfileUpdateMsg;
import org.thingsboard.server.gen.edge.v1.TenantUpdateMsg;
import org.thingsboard.server.gen.edge.v1.UpdateMsgType;
import org.thingsboard.server.gen.edge.v1.UplinkMsg;
import org.thingsboard.server.gen.edge.v1.UserCredentialsUpdateMsg;
import org.thingsboard.server.gen.edge.v1.UserUpdateMsg;
@ -142,35 +140,14 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest {
installation();
edgeImitator = new EdgeImitator("localhost", 7070, edge.getRoutingKey(), edge.getSecret());
edgeImitator.expectMessageAmount(25);
edgeImitator.ignoreType(OAuth2ClientUpdateMsg.class);
edgeImitator.ignoreType(OAuth2DomainUpdateMsg.class);
edgeImitator.expectMessageAmount(26);
edgeImitator.connect();
requestEdgeRuleChainMetadata();
verifyEdgeConnectionAndInitialData();
}
private void requestEdgeRuleChainMetadata() throws Exception {
RuleChainId rootRuleChainId = getEdgeRootRuleChainId();
RuleChainMetadataRequestMsg.Builder builder = RuleChainMetadataRequestMsg.newBuilder()
.setRuleChainIdMSB(rootRuleChainId.getId().getMostSignificantBits())
.setRuleChainIdLSB(rootRuleChainId.getId().getLeastSignificantBits());
testAutoGeneratedCodeByProtobuf(builder);
UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder()
.addRuleChainMetadataRequestMsg(builder.build());
edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build());
}
private RuleChainId getEdgeRootRuleChainId() throws Exception {
return doGetTypedWithPageLink("/api/ruleChains?type={type}&", new TypeReference<PageData<RuleChain>>() {
},
new PageLink(100, 0, "Edge Root Rule Chain"),
"EDGE")
.getData().get(0).getId();
}
@After
public void teardownEdgeTest() {
try {
@ -213,6 +190,19 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest {
doPost("/api/ruleChain/metadata", rootRuleChainMetadata, RuleChainMetaData.class);
}
private RuleChainId getEdgeRootRuleChainId() throws Exception {
List<RuleChain> edgeRuleChains = doGetTypedWithPageLink("/api/ruleChains?type={type}&",
new TypeReference<PageData<RuleChain>>() {},
new PageLink(100, 0, "Edge Root Rule Chain"),
"EDGE").getData();
for (RuleChain edgeRuleChain : edgeRuleChains) {
if (edgeRuleChain.isRoot()) {
return edgeRuleChain.getId();
}
}
throw new RuntimeException("Root rule chain not found");
}
protected void extendDeviceProfileData(DeviceProfile deviceProfile) {
DeviceProfileData profileData = deviceProfile.getProfileData();
List<DeviceProfileAlarm> alarms = new ArrayList<>();
@ -255,8 +245,8 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest {
validateMsgsCnt(RuleChainUpdateMsg.class, 1);
UUID ruleChainUUID = validateRuleChains();
// 1 from request message
validateMsgsCnt(RuleChainMetadataUpdateMsg.class, 2);
// 1 from rule chain fetcher
validateMsgsCnt(RuleChainMetadataUpdateMsg.class, 1);
validateRuleChainMetadataUpdates(ruleChainUUID);
// 4 messages ('general', 'mail', 'connectivity', 'jwt')
@ -438,12 +428,11 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest {
}
private void validateRuleChainMetadataUpdates(UUID expectedRuleChainUUID) {
Optional<RuleChainMetadataUpdateMsg> ruleChainMetadataUpdateOpt = edgeImitator.findMessageByType(RuleChainMetadataUpdateMsg.class);
Assert.assertTrue(ruleChainMetadataUpdateOpt.isPresent());
RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = ruleChainMetadataUpdateOpt.get();
Optional<RuleChainMetadataUpdateMsg> ruleChainMetadataUpdateMsgOpt = edgeImitator.findMessageByType(RuleChainMetadataUpdateMsg.class);
Assert.assertTrue(ruleChainMetadataUpdateMsgOpt.isPresent());
RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = ruleChainMetadataUpdateMsgOpt.get();
Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, ruleChainMetadataUpdateMsg.getMsgType());
RuleChainMetaData ruleChainMetaData = JacksonUtil.fromString(ruleChainMetadataUpdateMsg.getEntity(), RuleChainMetaData.class, true);
Assert.assertNotNull(ruleChainMetaData);
Assert.assertEquals(expectedRuleChainUUID, ruleChainMetaData.getRuleChainId().getId());
}

27
application/src/test/java/org/thingsboard/server/edge/AssetEdgeTest.java

@ -169,6 +169,7 @@ public class AssetEdgeTest extends AbstractEdgeTest {
public void testSendAssetToCloud() throws Exception {
Asset asset = buildAssetForUplinkMsg("Asset Edge 2");
// created asset on edge
UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder();
AssetUpdateMsg.Builder assetUpdateMsgBuilder = AssetUpdateMsg.newBuilder();
assetUpdateMsgBuilder.setIdMSB(asset.getUuidId().getMostSignificantBits());
@ -191,6 +192,32 @@ public class AssetEdgeTest extends AbstractEdgeTest {
Asset foundAsset = doGet("/api/asset/" + asset.getUuidId(), Asset.class);
Assert.assertNotNull(foundAsset);
Assert.assertEquals("Asset Edge 2", foundAsset.getName());
// update asset on edge
asset.setName("Asset Edge 2 Updated");
uplinkMsgBuilder = UplinkMsg.newBuilder();
assetUpdateMsgBuilder = AssetUpdateMsg.newBuilder();
assetUpdateMsgBuilder.setIdMSB(asset.getUuidId().getMostSignificantBits());
assetUpdateMsgBuilder.setIdLSB(asset.getUuidId().getLeastSignificantBits());
assetUpdateMsgBuilder.setEntity(JacksonUtil.toString(asset));
assetUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE);
testAutoGeneratedCodeByProtobuf(assetUpdateMsgBuilder);
uplinkMsgBuilder.addAssetUpdateMsg(assetUpdateMsgBuilder.build());
testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder);
edgeImitator.expectResponsesAmount(1);
edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build());
Assert.assertTrue(edgeImitator.waitForResponses());
latestResponseMsg = edgeImitator.getLatestResponseMsg();
Assert.assertTrue(latestResponseMsg.getSuccess());
foundAsset = doGet("/api/asset/" + asset.getUuidId(), Asset.class);
Assert.assertNotNull(foundAsset);
Assert.assertEquals("Asset Edge 2 Updated", foundAsset.getName());
}
@Test

1
application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java

@ -184,6 +184,7 @@ public class DashboardEdgeTest extends AbstractEdgeTest {
Dashboard dashboard = buildDashboardForUplinkMsg(savedCustomer);
// create dashboard on edge
UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder();
DashboardUpdateMsg.Builder dashboardUpdateMsgBuilder = DashboardUpdateMsg.newBuilder();
dashboardUpdateMsgBuilder.setIdMSB(dashboard.getUuidId().getMostSignificantBits());

24
application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java

@ -593,8 +593,10 @@ public class DeviceEdgeTest extends AbstractEdgeTest {
@Test
public void testSendDeviceToCloud() throws Exception {
Device deviceMsg = buildDeviceForUplinkMsg("Edge Device 2", "test");
String deviceName = "Edge Device 2";
Device deviceMsg = buildDeviceForUplinkMsg(deviceName, "test");
// create device on edge
UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder();
DeviceUpdateMsg.Builder deviceUpdateMsgBuilder = DeviceUpdateMsg.newBuilder();
deviceUpdateMsgBuilder.setIdMSB(deviceMsg.getUuidId().getMostSignificantBits());
@ -609,7 +611,25 @@ public class DeviceEdgeTest extends AbstractEdgeTest {
Device device = doGet("/api/device/" + deviceMsg.getId().getId(), Device.class);
Assert.assertNotNull(device);
Assert.assertEquals("Edge Device 2", device.getName());
Assert.assertEquals(deviceName, device.getName());
// update device on edge
deviceMsg.setName(deviceName + " Updated");
uplinkMsgBuilder = UplinkMsg.newBuilder();
deviceUpdateMsgBuilder = DeviceUpdateMsg.newBuilder();
deviceUpdateMsgBuilder.setIdMSB(deviceMsg.getUuidId().getMostSignificantBits());
deviceUpdateMsgBuilder.setIdLSB(deviceMsg.getUuidId().getLeastSignificantBits());
deviceUpdateMsgBuilder.setEntity(JacksonUtil.toString(deviceMsg));
deviceUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE);
uplinkMsgBuilder.addDeviceUpdateMsg(deviceUpdateMsgBuilder.build());
edgeImitator.expectResponsesAmount(1);
edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build());
Assert.assertTrue(edgeImitator.waitForResponses());
device = doGet("/api/device/" + deviceMsg.getId().getId(), Device.class);
Assert.assertNotNull(device);
Assert.assertEquals(deviceName + " Updated", device.getName());
}
@Test

84
application/src/test/java/org/thingsboard/server/edge/RuleChainEdgeTest.java

@ -15,7 +15,7 @@
*/
package org.thingsboard.server.edge;
import com.google.protobuf.AbstractMessage;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import org.junit.Assert;
import org.junit.Test;
import org.thingsboard.common.util.JacksonUtil;
@ -29,16 +29,17 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataRequestMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg;
import org.thingsboard.server.gen.edge.v1.UpdateMsgType;
import org.thingsboard.server.gen.edge.v1.UplinkMsg;
import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@ -74,8 +75,9 @@ public class RuleChainEdgeTest extends AbstractEdgeTest {
RuleChainMetaData ruleChainMetaData = JacksonUtil.fromString(ruleChainMetadataUpdateMsg.getEntity(), RuleChainMetaData.class, true);
Assert.assertNotNull(ruleChainMetaData);
Assert.assertEquals(ruleChainMetaData.getRuleChainId(), savedRuleChain.getId());
testRuleChainMetadataRequestMsg(savedRuleChain.getId());
for (RuleNode ruleNode : ruleChainMetaData.getNodes()) {
Assert.assertEquals(CONFIGURATION_VERSION, ruleNode.getConfigurationVersion());
}
// unassign rule chain from edge
edgeImitator.expectMessageAmount(1);
@ -97,60 +99,62 @@ public class RuleChainEdgeTest extends AbstractEdgeTest {
}
@Test
public void testSendRuleChainMetadataRequestToCloud() throws Exception {
RuleChainId edgeRootRuleChainId = edge.getRootRuleChainId();
public void testRuleChainToCloud() throws Exception {
String ruleChainName = "Rule Chain Edge";
UUID uuid = Uuids.timeBased();
// create rule chain on edge
RuleChain edgeRuleChain = new RuleChain();
edgeRuleChain.setTenantId(tenantId);
edgeRuleChain.setId(new RuleChainId(uuid));
edgeRuleChain.setName(ruleChainName);
UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder();
RuleChainMetadataRequestMsg.Builder ruleChainMetadataRequestMsgBuilder = RuleChainMetadataRequestMsg.newBuilder();
ruleChainMetadataRequestMsgBuilder.setRuleChainIdMSB(edgeRootRuleChainId.getId().getMostSignificantBits());
ruleChainMetadataRequestMsgBuilder.setRuleChainIdLSB(edgeRootRuleChainId.getId().getLeastSignificantBits());
testAutoGeneratedCodeByProtobuf(ruleChainMetadataRequestMsgBuilder);
uplinkMsgBuilder.addRuleChainMetadataRequestMsg(ruleChainMetadataRequestMsgBuilder.build());
RuleChainUpdateMsg.Builder ruleChainUpdateMsgBuilder = RuleChainUpdateMsg.newBuilder();
ruleChainUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits());
ruleChainUpdateMsgBuilder.setIdLSB(uuid.getLeastSignificantBits());
ruleChainUpdateMsgBuilder.setEntity(JacksonUtil.toString(edgeRuleChain));
ruleChainUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE);
testAutoGeneratedCodeByProtobuf(ruleChainUpdateMsgBuilder);
uplinkMsgBuilder.addRuleChainUpdateMsg(ruleChainUpdateMsgBuilder.build());
testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder);
edgeImitator.expectResponsesAmount(1);
edgeImitator.expectMessageAmount(1);
edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build());
Assert.assertTrue(edgeImitator.waitForResponses());
Assert.assertTrue(edgeImitator.waitForMessages());
AbstractMessage latestMessage = edgeImitator.getLatestMessage();
Assert.assertTrue(latestMessage instanceof RuleChainMetadataUpdateMsg);
RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = (RuleChainMetadataUpdateMsg) latestMessage;
RuleChainMetaData ruleChainMetadataMsg = JacksonUtil.fromString(ruleChainMetadataUpdateMsg.getEntity(), RuleChainMetaData.class, true);
Assert.assertNotNull(ruleChainMetadataMsg);
Assert.assertEquals(edgeRootRuleChainId, ruleChainMetadataMsg.getRuleChainId());
UplinkResponseMsg latestResponseMsg = edgeImitator.getLatestResponseMsg();
Assert.assertTrue(latestResponseMsg.getSuccess());
testAutoGeneratedCodeByProtobuf(ruleChainMetadataUpdateMsg);
}
RuleChain ruleChain = doGet("/api/ruleChain/" + uuid, RuleChain.class);
Assert.assertNotNull(ruleChain);
Assert.assertEquals("Rule Chain Edge", ruleChain.getName());
private void testRuleChainMetadataRequestMsg(RuleChainId ruleChainId) throws Exception {
RuleChainMetadataRequestMsg.Builder ruleChainMetadataRequestMsgBuilder = RuleChainMetadataRequestMsg.newBuilder()
.setRuleChainIdMSB(ruleChainId.getId().getMostSignificantBits())
.setRuleChainIdLSB(ruleChainId.getId().getLeastSignificantBits());
testAutoGeneratedCodeByProtobuf(ruleChainMetadataRequestMsgBuilder);
// update rule chain on edge
edgeRuleChain.setName(ruleChainName + " Updated");
uplinkMsgBuilder = UplinkMsg.newBuilder();
ruleChainUpdateMsgBuilder = RuleChainUpdateMsg.newBuilder();
ruleChainUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits());
ruleChainUpdateMsgBuilder.setIdLSB(uuid.getLeastSignificantBits());
ruleChainUpdateMsgBuilder.setEntity(JacksonUtil.toString(edgeRuleChain));
ruleChainUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE);
testAutoGeneratedCodeByProtobuf(ruleChainUpdateMsgBuilder);
uplinkMsgBuilder.addRuleChainUpdateMsg(ruleChainUpdateMsgBuilder.build());
UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder()
.addRuleChainMetadataRequestMsg(ruleChainMetadataRequestMsgBuilder.build());
testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder);
edgeImitator.expectResponsesAmount(1);
edgeImitator.expectMessageAmount(1);
edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build());
Assert.assertTrue(edgeImitator.waitForResponses());
Assert.assertTrue(edgeImitator.waitForMessages());
AbstractMessage latestMessage = edgeImitator.getLatestMessage();
Assert.assertTrue(latestMessage instanceof RuleChainMetadataUpdateMsg);
RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = (RuleChainMetadataUpdateMsg) latestMessage;
RuleChainMetaData ruleChainMetadataMsg = JacksonUtil.fromString(ruleChainMetadataUpdateMsg.getEntity(), RuleChainMetaData.class, true);
Assert.assertNotNull(ruleChainMetadataMsg);
Assert.assertEquals(ruleChainId, ruleChainMetadataMsg.getRuleChainId());
latestResponseMsg = edgeImitator.getLatestResponseMsg();
Assert.assertTrue(latestResponseMsg.getSuccess());
for (RuleNode ruleNode : ruleChainMetadataMsg.getNodes()) {
Assert.assertEquals(CONFIGURATION_VERSION, ruleNode.getConfigurationVersion());
}
ruleChain = doGet("/api/ruleChain/" + uuid, RuleChain.class);
Assert.assertNotNull(ruleChain);
Assert.assertEquals(ruleChainName + " Updated", ruleChain.getName());
}
private RuleChainMetaData createRuleChainMetadata(RuleChain ruleChain) {

17
application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java

@ -153,7 +153,7 @@ public class HashPartitionServiceTest {
for (int queueIndex = 0; queueIndex < queueCount; queueIndex++) {
QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, "queue" + queueIndex, tenantId);
for (int partition = 0; partition < partitionCount; partition++) {
ServiceInfo serviceInfo = partitionService.resolveByPartitionIdx(services, queueKey, partition, Collections.emptyMap());
ServiceInfo serviceInfo = partitionService.resolveByPartitionIdx(services, queueKey, partition, Collections.emptyMap()).get(0);
String serviceId = serviceInfo.getServiceId();
map.put(serviceId, map.get(serviceId) + 1);
}
@ -308,7 +308,7 @@ public class HashPartitionServiceTest {
partitionService_common.recalculatePartitions(commonRuleEngine, List.of(dedicatedRuleEngine));
verifyPartitionChangeEvent(event -> {
QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, TenantId.SYS_TENANT_ID);
return event.getPartitionsMap().get(queueKey).size() == systemQueue.getPartitions();
return event.getNewPartitions().get(queueKey).size() == systemQueue.getPartitions();
});
Mockito.reset(applicationEventPublisher);
@ -336,14 +336,14 @@ public class HashPartitionServiceTest {
// expecting event about no partitions for isolated queue key
verifyPartitionChangeEvent(event -> {
QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId);
return event.getPartitionsMap().get(queueKey).isEmpty();
return event.getNewPartitions().get(queueKey).isEmpty();
});
partitionService_dedicated.updateQueues(List.of(queueUpdateMsg));
partitionService_dedicated.recalculatePartitions(dedicatedRuleEngine, List.of(commonRuleEngine));
verifyPartitionChangeEvent(event -> {
QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId);
return event.getPartitionsMap().get(queueKey).size() == isolatedQueue.getPartitions();
return event.getNewPartitions().get(queueKey).size() == isolatedQueue.getPartitions();
});
@ -361,7 +361,7 @@ public class HashPartitionServiceTest {
partitionService_dedicated.removeQueues(List.of(queueDeleteMsg));
verifyPartitionChangeEvent(event -> {
QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId);
return event.getPartitionsMap().get(queueKey).isEmpty();
return event.getNewPartitions().get(queueKey).isEmpty();
});
}
@ -381,12 +381,12 @@ public class HashPartitionServiceTest {
Stream.concat(Stream.of(TenantId.SYS_TENANT_ID), Stream.generate(UUID::randomUUID).map(TenantId::new).limit(10)).forEach(tenantId -> {
List<QueueKey> queues = Stream.generate(() -> RandomStringUtils.randomAlphabetic(10))
.map(queueName -> new QueueKey(ServiceType.TB_RULE_ENGINE, queueName, tenantId))
.limit(100).collect(Collectors.toList());
.limit(100).toList();
for (int partition = 0; partition < 10; partition++) {
ServiceInfo expectedAssignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, new QueueKey(ServiceType.TB_RULE_ENGINE, tenantId), partition, Collections.emptyMap());
ServiceInfo expectedAssignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, new QueueKey(ServiceType.TB_RULE_ENGINE, tenantId), partition, Collections.emptyMap()).get(0);
for (QueueKey queueKey : queues) {
ServiceInfo assignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, queueKey, partition, Collections.emptyMap());
ServiceInfo assignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, queueKey, partition, Collections.emptyMap()).get(0);
assertThat(assignedRuleEngine).as(queueKey + "[" + partition + "] should be assigned to " + expectedAssignedRuleEngine.getServiceId())
.isEqualTo(expectedAssignedRuleEngine);
}
@ -434,6 +434,7 @@ public class HashPartitionServiceTest {
ReflectionTestUtils.setField(partitionService, "hashFunctionName", hashFunctionName);
ReflectionTestUtils.setField(partitionService, "edgeTopic", "tb.edge");
ReflectionTestUtils.setField(partitionService, "edgePartitions", 10);
ReflectionTestUtils.setField(partitionService, "edqsPartitions", 12);
partitionService.init();
partitionService.partitionsInit();
return partitionService;

4
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java

@ -90,7 +90,7 @@ public class ScriptCalculatedFieldStateTest {
state.arguments = new HashMap<>(Map.of("assetHumidity", assetHumidityArgEntry));
Map<String, ArgumentEntry> newArgs = Map.of("deviceTemperature", deviceTemperatureArgEntry);
boolean stateUpdated = state.updateState(newArgs);
boolean stateUpdated = state.updateState(ctx, newArgs);
assertThat(stateUpdated).isTrue();
assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf(
@ -107,7 +107,7 @@ public class ScriptCalculatedFieldStateTest {
SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(ts, new LongDataEntry("assetHumidity", 41L), 349L);
Map<String, ArgumentEntry> newArgs = Map.of("assetHumidity", newArgEntry);
boolean stateUpdated = state.updateState(newArgs);
boolean stateUpdated = state.updateState(ctx, newArgs);
assertThat(stateUpdated).isTrue();
assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf(

30
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java

@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField
import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.DoubleDataEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.common.data.kv.StringDataEntry;
import org.thingsboard.server.dao.usagerecord.ApiLimitService;
@ -87,7 +88,7 @@ public class SimpleCalculatedFieldStateTest {
));
Map<String, ArgumentEntry> newArgs = Map.of("key3", key3ArgEntry);
boolean stateUpdated = state.updateState(newArgs);
boolean stateUpdated = state.updateState(ctx, newArgs);
assertThat(stateUpdated).isTrue();
assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf(
@ -105,7 +106,7 @@ public class SimpleCalculatedFieldStateTest {
SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis(), new LongDataEntry("key1", 18L), 190L);
Map<String, ArgumentEntry> newArgs = Map.of("key1", newArgEntry);
boolean stateUpdated = state.updateState(newArgs);
boolean stateUpdated = state.updateState(ctx, newArgs);
assertThat(stateUpdated).isTrue();
assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf(Map.of("key1", newArgEntry));
@ -119,7 +120,7 @@ public class SimpleCalculatedFieldStateTest {
));
Map<String, ArgumentEntry> newArgs = Map.of("key3", new TsRollingArgumentEntry(10, 30000L));
assertThatThrownBy(() -> state.updateState(newArgs))
assertThatThrownBy(() -> state.updateState(ctx, newArgs))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Rolling argument entry is not supported for simple calculated fields.");
}
@ -138,7 +139,7 @@ public class SimpleCalculatedFieldStateTest {
Output output = getCalculatedFieldConfig().getOutput();
assertThat(result.getType()).isEqualTo(output.getType());
assertThat(result.getScope()).isEqualTo(output.getScope());
assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49.0)));
assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49)));
}
@Test
@ -154,6 +155,26 @@ public class SimpleCalculatedFieldStateTest {
.hasMessage("Argument 'key2' is not a number.");
}
@Test
void testPerformCalculationWhenDecimalsByDefault() throws ExecutionException, InterruptedException {
state.arguments = new HashMap<>(Map.of(
"key1", new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new DoubleDataEntry("key1", 11.3456), 145L),
"key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 6, new DoubleDataEntry("key2", 15.1), 165L),
"key3", new SingleValueArgumentEntry(System.currentTimeMillis() - 3, new DoubleDataEntry("key3", 23.1), 184L)
));
Output output = getCalculatedFieldConfig().getOutput();
output.setDecimalsByDefault(3);
ctx.setOutput(output);
CalculatedFieldResult result = state.performCalculation(ctx).get();
assertThat(result).isNotNull();
assertThat(result.getType()).isEqualTo(output.getType());
assertThat(result.getScope()).isEqualTo(output.getScope());
assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49.546)));
}
@Test
void testIsReadyWhenNotAllArgPresent() {
assertThat(state.isReady()).isFalse();
@ -219,6 +240,7 @@ public class SimpleCalculatedFieldStateTest {
output.setName("output");
output.setType(OutputType.ATTRIBUTES);
output.setScope(AttributeScope.SERVER_SCOPE);
output.setDecimalsByDefault(0);
config.setOutput(output);

2
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java

@ -71,6 +71,6 @@ public class SingleValueArgumentEntryTest {
@Test
void testUpdateEntryWhenValueWasNotChanged() {
assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 11L), 237L))).isFalse();
assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 11L), 364L))).isTrue();
}
}

128
application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java

@ -0,0 +1,128 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.entitiy;
import com.google.common.collect.Lists;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.TestPropertySource;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.IdBased;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.query.EntityCountQuery;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.RelationsQueryFilter;
import org.thingsboard.server.common.data.relation.EntitySearchDirection;
import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter;
import org.thingsboard.server.common.msg.edqs.EdqsApiService;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.edqs.util.EdqsRocksDb;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static org.awaitility.Awaitility.await;
@DaoSqlTest
@TestPropertySource(properties = {
"queue.edqs.sync.enabled=true",
"queue.edqs.api.supported=true",
"queue.edqs.api.auto_enable=true",
"queue.edqs.mode=local"
})
public class EdqsEntityServiceTest extends EntityServiceTest {
@Autowired
private EdqsApiService edqsApiService;
@MockBean
private EdqsRocksDb edqsRocksDb;
@Before
public void beforeEach() {
await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsApiService.isEnabled());
}
// sql implementation has a bug with data duplication, edqs implementation returns correct value
@Override
@Test
public void testCountHierarchicalEntitiesByMultiRootQuery() throws InterruptedException {
List<Asset> buildings = new ArrayList<>();
List<Asset> apartments = new ArrayList<>();
Map<String, Map<UUID, String>> entityNameByTypeMap = new HashMap<>();
Map<UUID, UUID> childParentRelationMap = new HashMap<>();
createMultiRootHierarchy(buildings, apartments, entityNameByTypeMap, childParentRelationMap);
RelationsQueryFilter filter = new RelationsQueryFilter();
filter.setMultiRoot(true);
filter.setMultiRootEntitiesType(EntityType.ASSET);
filter.setMultiRootEntityIds(buildings.stream().map(IdBased::getId).map(d -> d.getId().toString()).collect(Collectors.toSet()));
filter.setDirection(EntitySearchDirection.FROM);
EntityCountQuery countQuery = new EntityCountQuery(filter);
countByQueryAndCheck(countQuery, 63);
filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("AptToHeat", Collections.singletonList(EntityType.DEVICE))));
countByQueryAndCheck(countQuery, 27);
filter.setMultiRootEntitiesType(EntityType.ASSET);
filter.setMultiRootEntityIds(apartments.stream().map(IdBased::getId).map(d -> d.getId().toString()).collect(Collectors.toSet()));
filter.setDirection(EntitySearchDirection.TO);
filter.setFilters(Lists.newArrayList(
new RelationEntityTypeFilter("buildingToApt", Collections.singletonList(EntityType.ASSET)),
new RelationEntityTypeFilter("AptToEnergy", Collections.singletonList(EntityType.DEVICE))));
countByQueryAndCheck(countQuery, 3);
deviceService.deleteDevicesByTenantId(tenantId);
assetService.deleteAssetsByTenantId(tenantId);
}
@Override
protected PageData<EntityData> findByQueryAndCheck(CustomerId customerId, EntityDataQuery query, long expectedResultSize) {
return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findByQuery(customerId, query),
result -> result.getTotalElements() == expectedResultSize);
}
@Override
protected List<EntityData> findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List<String> expectedTelemetries) {
return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetries),
loadedEntities -> loadedEntities.stream().map(entityData -> entityData.getLatest().get(entityKeyType).get(key).getValue()).toList().containsAll(expectedTelemetries));
}
@Override
protected long countByQueryAndCheck(EntityCountQuery countQuery, int expectedResult) {
return countByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), countQuery, expectedResult);
}
@Override
protected long countByQueryAndCheck(CustomerId customerId, EntityCountQuery query, int expectedResult) {
return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(customerId, query),
result -> result == expectedResult);
}
}

799
dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java → application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java

File diff suppressed because it is too large

93
application/src/test/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerServiceTest.java

@ -532,6 +532,98 @@ public class DefaultTbCoreConsumerServiceTest {
then(statsMock).should(never()).log(inactivityMsg);
}
@Test
public void givenProcessingSuccess_whenForwardingInactivityTimeoutUpdateMsgToStateService_thenOnSuccessCallbackIsCalled() {
// GIVEN
var inactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setInactivityTimeout(time)
.build();
doCallRealMethod().when(defaultTbCoreConsumerServiceMock).forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock);
// WHEN
defaultTbCoreConsumerServiceMock.forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock);
// THEN
then(stateServiceMock).should().onDeviceInactivityTimeoutUpdate(tenantId, deviceId, time);
then(tbCallbackMock).should().onSuccess();
then(tbCallbackMock).should(never()).onFailure(any());
}
@Test
public void givenProcessingFailure_whenForwardingInactivityTimeoutUpdateMsgToStateService_thenOnFailureCallbackIsCalled() {
// GIVEN
var inactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setInactivityTimeout(time)
.build();
doCallRealMethod().when(defaultTbCoreConsumerServiceMock).forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock);
var runtimeException = new RuntimeException("Something bad happened!");
doThrow(runtimeException).when(stateServiceMock).onDeviceInactivityTimeoutUpdate(tenantId, deviceId, time);
// WHEN
defaultTbCoreConsumerServiceMock.forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock);
// THEN
then(tbCallbackMock).should(never()).onSuccess();
then(tbCallbackMock).should().onFailure(runtimeException);
}
@Test
public void givenStatsEnabled_whenForwardingInactivityTimeoutUpdateMsgToStateService_thenStatsAreRecorded() {
// GIVEN
ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "stats", statsMock);
ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "statsEnabled", true);
var inactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setInactivityTimeout(time)
.build();
doCallRealMethod().when(defaultTbCoreConsumerServiceMock).forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock);
// WHEN
defaultTbCoreConsumerServiceMock.forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock);
// THEN
then(statsMock).should().log(inactivityTimeoutUpdateMsg);
}
@Test
public void givenStatsDisabled_whenForwardingInactivityTimeoutUpdateMsgToStateService_thenStatsAreNotRecorded() {
// GIVEN
ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "stats", statsMock);
ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "statsEnabled", false);
var inactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setInactivityTimeout(time)
.build();
doCallRealMethod().when(defaultTbCoreConsumerServiceMock).forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock);
// WHEN
defaultTbCoreConsumerServiceMock.forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock);
// THEN
then(statsMock).should(never()).log(inactivityTimeoutUpdateMsg);
}
@Test
public void givenRestApiCallResponseMsgProto_whenForwardToRuleEngineCallService_thenCallOnQueueMsg() {
// GIVEN
@ -545,4 +637,5 @@ public class DefaultTbCoreConsumerServiceTest {
// THEN
then(ruleEngineCallServiceMock).should().onQueueMsg(restApiCallResponseMsgProto, tbCallbackMock);
}
}

11
application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java

@ -626,8 +626,7 @@ public class TbRuleEngineQueueConsumerManagerTest {
.until(() -> consumer.subscribed && consumer.getPartitions().equals(expectedPartitions) && consumer.pollingStarted);
verify(consumer, times(1)).subscribe(any());
verify(consumer).subscribe(eq(expectedPartitions));
verify(consumer).doSubscribe(argThat(topics -> topics.containsAll(expectedPartitions.stream()
.map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()))));
verify(consumer).doSubscribe(argThat(topics -> topics.containsAll(expectedPartitions)));
verify(consumer, atLeastOnce()).poll(eq((long) queue.getPollInterval()));
verify(consumer, atLeastOnce()).doPoll(eq((long) queue.getPollInterval()));
verify(consumer, never()).unsubscribe();
@ -743,9 +742,11 @@ public class TbRuleEngineQueueConsumerManagerTest {
}
@Override
protected void doSubscribe(List<String> topicNames) {
log.debug("doSubscribe({})", topicNames);
this.topics = topicNames;
protected void doSubscribe(Set<TopicPartitionInfo> partitions) {
this.topics = partitions.stream()
.map(TopicPartitionInfo::getFullTopicName)
.collect(Collectors.toList());
log.debug("doSubscribe({})", topics);
subscribed = true;
}

202
application/src/test/java/org/thingsboard/server/service/script/TbelInvokeDocsIoTest.java

@ -21,6 +21,7 @@ import org.thingsboard.script.api.tbel.TbDate;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
import java.util.Comparator;
@ -146,6 +147,31 @@ class TbelInvokeDocsIoTest extends AbstractTbelInvokeTest {
assertEquals(expected, actual);
}
@Test
public void mapsImplicitIterationWithoutEntrySet() throws ExecutionException, InterruptedException {
msgStr = msgMapStr;
decoderStr = """
foreach(element : msg){
if(element.getKey() == null){
return raiseError("Bad getKey");
}
if(element.key == null){
return raiseError("Bad key");
}
if(element.getValue() == null){
return raiseError("Bad getValue");
}
if(element.value == null){
return raiseError("Bad value");
}
}
return msg;
""";
LinkedHashMap<String, Object> expected = expectedMap;
Object actual = invokeScript(evalScript(decoderStr), msgStr);
assertEquals(expected, actual);
}
@Test
public void mapsGetInfoSize_Test() throws ExecutionException, InterruptedException {
msgStr = msgMapStr;
@ -660,14 +686,13 @@ class TbelInvokeDocsIoTest extends AbstractTbelInvokeTest {
""";
decoderStr = """
var list = msg.list;
var listAdd = ["thigsboard", 4, 67];
return {
list: list.clone(),
length: list.length(),
memorySize: list.memorySize(),
indOf1: list.indexOf("B", 1),
indOf2: list.indexOf(2, 2),
sStr: list.validateClazzInArrayIsOnlyString()
sStr: list.validateClazzInArrayIsOnlyNumber()
}
""";
ArrayList list = new ArrayList<>(List.of(67, 2, 2, 2));
@ -677,7 +702,7 @@ class TbelInvokeDocsIoTest extends AbstractTbelInvokeTest {
expected.put("memorySize", 32L);
expected.put("indOf1", -1);
expected.put("indOf2", 2);
expected.put("sStr", false);
expected.put("sStr", true);
Object actual = invokeScript(evalScript(decoderStr), msgStr);
assertEquals(expected, actual);
}
@ -2148,6 +2173,177 @@ class TbelInvokeDocsIoTest extends AbstractTbelInvokeTest {
assertEquals(expected, actual);
}
@Test
public void toUnmodifiableExecutionArrayList_Test() throws ExecutionException, InterruptedException {
msgStr = "{}";
decoderStr = String.format("""
var original = [];
original.add(0x35);
var unmodifiable = original.toUnmodifiable();
msg.result = unmodifiable;
return {msg: msg};
""");
LinkedHashMap<String, Object> expected = new LinkedHashMap<>();
List expectedList = Arrays.asList(0x35);
LinkedHashMap<String, Object> expectedResult = new LinkedHashMap<>();
expectedResult.put("result", expectedList);
expected.put("msg", expectedResult);
Object actual = invokeScript(evalScript(decoderStr), msgStr);
assertEquals(expected, actual);
decoderStr = String.format("""
var original = [];
original.add(0x67);
var unmodifiable = original.toUnmodifiable();
unmodifiable.add(0x35);
msg.result = unmodifiable;
return {msg: msg};
""");
assertThatThrownBy(() -> {
invokeScript(evalScript(decoderStr), msgStr);
}).hasMessageContaining("Error: unmodifiable.add(0x35): List is unmodifiable");
}
@Test
public void toUnmodifiableExecutionHashMap_Test() throws ExecutionException, InterruptedException {
msgStr = "{}";
decoderStr = String.format("""
var original = {};
original.putIfAbsent("entry1", 73);
var unmodifiable = original.toUnmodifiable();
msg.result = unmodifiable;
return {msg: msg};
""");
LinkedHashMap<String, Object> expected = new LinkedHashMap<>();
LinkedHashMap<String, Object> expectedMap = new LinkedHashMap<>(Map.of("entry1", 73));
LinkedHashMap<String, Object> expectedResult = new LinkedHashMap<>();
expectedResult.put("result", expectedMap);
expected.put("msg", expectedResult);
Object actual = invokeScript(evalScript(decoderStr), msgStr);
assertEquals(expected, actual);
decoderStr = String.format("""
var original = {};
original.humidity = 73;
var unmodifiable = original.toUnmodifiable();
unmodifiable.put("temperature1", 96);
msg.result = unmodifiable;
return {msg: msg};
""");
assertThatThrownBy(() -> {
invokeScript(evalScript(decoderStr), msgStr);
}).hasMessageContaining("Error: unmodifiable.put(\"temperature1\", 96): Map is unmodifiable");
}
@Test
public void tbDateFunction_Test() throws ExecutionException, InterruptedException {
String stringDateUTC = "2024-01-01T10:00:00.00Z";
TbDate d = new TbDate(stringDateUTC);
msgStr = "{}";
decoderStr = String.format("""
var d = new Date("%s"); // TZ => "UTC"
var dIsoY1 = d.toISOString(); // return 2024-01-01T10:00:00Z
d.addYears(1);
var dIsoY2 = d.toISOString(); // return 2025-01-01T10:00:00Z
d.addYears(-2);
var dIsoY3 = d.toISOString(); // return 2023-01-01T10:00:00Z
d.addMonths(2);
var dIsoM1 = d.toISOString(); // return 2023-03-01T10:00:00Z
d.addMonths(10);
var dIsoM2 = d.toISOString(); // return 2024-01-01T10:00:00Z
d.addMonths(-13);
var dIsoM3 = d.toISOString(); // return 2022-12-01T10:00:00Z
d.addWeeks(4);
var dIsoW1 = d.toISOString(); // return 2022-12-29T10:00:00Z
d.addWeeks(-5);
var dIsoW2 = d.toISOString(); // return 2022-11-24T10:00:00Z
d.addDays(6);
var dIsoD1 = d.toISOString(); // return 2022-11-30T10:00:00Z
d.addDays(45);
var dIsoD2 = d.toISOString(); // return 2023-01-14T10:00:00Z
d.addDays(-50);
var dIsoD3 = d.toISOString(); // return 2022-11-25T10:00:00Z
d.addHours(23);
var dIsoH1 = d.toISOString(); // return 2022-11-26T09:00:00Z
d.addHours(-47);
var dIsoH2 = d.toISOString(); // return 2022-11-24T10:00:00Z
d.addMinutes(59);
var dIsoMin1 = d.toISOString(); // return 2022-11-24T10:59:00Z
d.addMinutes(-60);
var dIsoMin2 = d.toISOString(); // return 2022-11-24T09:59:00Z
d.addSeconds(59);
var dIsoS1 = d.toISOString(); // return 2022-11-24T09:59:59Z
d.addSeconds(-60);
var dIsoS2 = d.toISOString(); // return 2022-11-24T09:58:59Z
d.addNanos(999999);
var dIsoN1 = d.toISOString(); // return 2022-11-24T09:58:59.000999999Z
d.addNanos(-1000000);
var dIsoN2 = d.toISOString(); // return 2022-11-24T09:58:58.999999999Z
return {
"dIsoY1": dIsoY1,
"dIsoY2": dIsoY2,
"dIsoY3": dIsoY3,
"dIsoM1": dIsoM1,
"dIsoM2": dIsoM2,
"dIsoM3": dIsoM3,
"dIsoW1": dIsoW1,
"dIsoW2": dIsoW2,
"dIsoD1": dIsoD1,
"dIsoD2": dIsoD2,
"dIsoD3": dIsoD3,
"dIsoH1": dIsoH1,
"dIsoH2": dIsoH2,
"dIsoMin1": dIsoMin1,
"dIsoMin2": dIsoMin2,
"dIsoS1": dIsoS1,
"dIsoS2": dIsoS2,
"dIsoN1": dIsoN1,
"dIsoN2": dIsoN2
}
""", stringDateUTC);
LinkedHashMap<String, Object> expected = new LinkedHashMap<>();
expected.put("dIsoY1", d.toISOString());
d.addYears(1);
expected.put("dIsoY2", d.toISOString());
d.addYears(-2);
expected.put("dIsoY3", d.toISOString());
d.addMonths(2);
expected.put("dIsoM1", d.toISOString());
d.addMonths(10);
expected.put("dIsoM2", d.toISOString());
d.addMonths(-13);
expected.put("dIsoM3", d.toISOString());
d.addWeeks(4);
expected.put("dIsoW1", d.toISOString());
d.addWeeks(-5);
expected.put("dIsoW2", d.toISOString());
d.addDays(6);
expected.put("dIsoD1", d.toISOString());
d.addDays(45);
expected.put("dIsoD2", d.toISOString());
d.addDays(-50);
expected.put("dIsoD3", d.toISOString());
d.addHours(23);
expected.put("dIsoH1", d.toISOString());
d.addHours(-47);
expected.put("dIsoH2", d.toISOString());
d.addMinutes(59);
expected.put("dIsoMin1", d.toISOString());
d.addMinutes(-60);
expected.put("dIsoMin2", d.toISOString());
d.addSeconds(59);
expected.put("dIsoS1", d.toISOString());
d.addSeconds(-60);
expected.put("dIsoS2", d.toISOString());
d.addNanos(999999);
expected.put("dIsoN1", d.toISOString());
d.addNanos(-1000000);
expected.put("dIsoN2", d.toISOString());
Object actual = invokeScript(evalScript(decoderStr), msgStr);
assertEquals(expected, actual);
}
private List splice(List oldList, int start, int deleteCount, Object... values) {
start = initStartIndex(oldList, start);

61
application/src/test/java/org/thingsboard/server/service/state/DefaultRuleEngineDeviceStateManagerTest.java → application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateManagerTest.java

@ -52,7 +52,7 @@ import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.never;
@ExtendWith(MockitoExtension.class)
public class DefaultRuleEngineDeviceStateManagerTest {
public class DefaultDeviceStateManagerTest {
@Mock
private DeviceStateService deviceStateServiceMock;
@ -71,7 +71,7 @@ public class DefaultRuleEngineDeviceStateManagerTest {
@Captor
private ArgumentCaptor<TbQueueCallback> queueCallbackCaptor;
private DefaultRuleEngineDeviceStateManager deviceStateManager;
private DefaultDeviceStateManager deviceStateManager;
private static final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("57ab2e6c-bc4c-11ee-a506-0242ac120002"));
private static final DeviceId DEVICE_ID = DeviceId.fromString("74a9053e-bc4c-11ee-a506-0242ac120002");
@ -82,7 +82,7 @@ public class DefaultRuleEngineDeviceStateManagerTest {
@BeforeEach
public void setup() {
deviceStateManager = new DefaultRuleEngineDeviceStateManager(serviceInfoProviderMock, partitionServiceMock, Optional.of(deviceStateServiceMock), clusterServiceMock);
deviceStateManager = new DefaultDeviceStateManager(serviceInfoProviderMock, partitionServiceMock, Optional.of(deviceStateServiceMock), clusterServiceMock);
}
@ParameterizedTest
@ -90,7 +90,7 @@ public class DefaultRuleEngineDeviceStateManagerTest {
"when onDeviceX() is called, then should route event to local service and call onSuccess() callback.")
@MethodSource
public void givenRoutedToLocalAndProcessingSuccess_whenOnDeviceAction_thenShouldCallLocalServiceAndSuccessCallback(
BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback> onDeviceAction, Consumer<DeviceStateService> actionVerification
BiConsumer<DefaultDeviceStateManager, TbCallback> onDeviceAction, Consumer<DeviceStateService> actionVerification
) {
// GIVEN
given(serviceInfoProviderMock.isService(ServiceType.TB_CORE)).willReturn(true);
@ -109,20 +109,24 @@ public class DefaultRuleEngineDeviceStateManagerTest {
private static Stream<Arguments> givenRoutedToLocalAndProcessingSuccess_whenOnDeviceAction_thenShouldCallLocalServiceAndSuccessCallback() {
return Stream.of(
Arguments.of(
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS)
),
Arguments.of(
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS)
),
Arguments.of(
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS)
),
Arguments.of(
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS)
),
Arguments.of(
(BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS)
)
);
}
@ -132,7 +136,7 @@ public class DefaultRuleEngineDeviceStateManagerTest {
"when onDeviceX() is called, then should route event to local service and call onFailure() callback.")
@MethodSource
public void givenRoutedToLocalAndProcessingFailure_whenOnDeviceAction_thenShouldCallLocalServiceAndFailureCallback(
Consumer<DeviceStateService> exceptionThrowSetup, BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback> onDeviceAction, Consumer<DeviceStateService> actionVerification
Consumer<DeviceStateService> exceptionThrowSetup, BiConsumer<DefaultDeviceStateManager, TbCallback> onDeviceAction, Consumer<DeviceStateService> actionVerification
) {
// GIVEN
given(serviceInfoProviderMock.isService(ServiceType.TB_CORE)).willReturn(true);
@ -155,23 +159,28 @@ public class DefaultRuleEngineDeviceStateManagerTest {
return Stream.of(
Arguments.of(
(Consumer<DeviceStateService>) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS),
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS)
),
Arguments.of(
(Consumer<DeviceStateService>) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS),
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS)
),
Arguments.of(
(Consumer<DeviceStateService>) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS),
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS)
),
Arguments.of(
(Consumer<DeviceStateService>) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS),
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS)
),
Arguments.of(
(Consumer<DeviceStateService>) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS),
(BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(Consumer<DeviceStateService>) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS)
)
);
}
@ -181,7 +190,7 @@ public class DefaultRuleEngineDeviceStateManagerTest {
"when onDeviceX() is called, then should send correct queue message to external service with correct callback object.")
@MethodSource
public void givenRoutedToExternal_whenOnDeviceAction_thenShouldSendQueueMsgToExternalServiceWithCorrectCallback(
BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback> onDeviceAction, BiConsumer<TbClusterService, ArgumentCaptor<TbQueueCallback>> actionVerification
BiConsumer<DefaultDeviceStateManager, TbCallback> onDeviceAction, BiConsumer<TbClusterService, ArgumentCaptor<TbQueueCallback>> actionVerification
) {
// WHEN
ReflectionTestUtils.setField(deviceStateManager, "deviceStateService", Optional.empty());
@ -203,7 +212,7 @@ public class DefaultRuleEngineDeviceStateManagerTest {
private static Stream<Arguments> givenRoutedToExternal_whenOnDeviceAction_thenShouldSendQueueMsgToExternalServiceWithCorrectCallback() {
return Stream.of(
Arguments.of(
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<TbClusterService, ArgumentCaptor<TbQueueCallback>>) (clusterServiceMock, queueCallbackCaptor) -> {
var deviceConnectMsg = TransportProtos.DeviceConnectProto.newBuilder()
.setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits())
@ -219,7 +228,7 @@ public class DefaultRuleEngineDeviceStateManagerTest {
}
),
Arguments.of(
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<TbClusterService, ArgumentCaptor<TbQueueCallback>>) (clusterServiceMock, queueCallbackCaptor) -> {
var deviceActivityMsg = TransportProtos.DeviceActivityProto.newBuilder()
.setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits())
@ -235,7 +244,7 @@ public class DefaultRuleEngineDeviceStateManagerTest {
}
),
Arguments.of(
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<TbClusterService, ArgumentCaptor<TbQueueCallback>>) (clusterServiceMock, queueCallbackCaptor) -> {
var deviceDisconnectMsg = TransportProtos.DeviceDisconnectProto.newBuilder()
.setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits())
@ -251,7 +260,7 @@ public class DefaultRuleEngineDeviceStateManagerTest {
}
),
Arguments.of(
(BiConsumer<DefaultRuleEngineDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<TbClusterService, ArgumentCaptor<TbQueueCallback>>) (clusterServiceMock, queueCallbackCaptor) -> {
var deviceInactivityMsg = TransportProtos.DeviceInactivityProto.newBuilder()
.setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits())
@ -265,6 +274,22 @@ public class DefaultRuleEngineDeviceStateManagerTest {
.build();
then(clusterServiceMock).should().pushMsgToCore(eq(EXTERNAL_TPI), any(UUID.class), eq(toCoreMsg), queueCallbackCaptor.capture());
}
),
Arguments.of(
(BiConsumer<DefaultDeviceStateManager, TbCallback>) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock),
(BiConsumer<TbClusterService, ArgumentCaptor<TbQueueCallback>>) (clusterServiceMock, queueCallbackCaptor) -> {
var deviceInactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder()
.setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits())
.setTenantIdLSB(TENANT_ID.getId().getLeastSignificantBits())
.setDeviceIdMSB(DEVICE_ID.getId().getMostSignificantBits())
.setDeviceIdLSB(DEVICE_ID.getId().getLeastSignificantBits())
.setInactivityTimeout(EVENT_TS)
.build();
var toCoreMsg = TransportProtos.ToCoreMsg.newBuilder()
.setDeviceInactivityTimeoutUpdateMsg(deviceInactivityTimeoutUpdateMsg)
.build();
then(clusterServiceMock).should().pushMsgToCore(eq(EXTERNAL_TPI), any(UUID.class), eq(toCoreMsg), queueCallbackCaptor.capture());
}
)
);
}

60
application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java

@ -38,9 +38,6 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.data.notification.rule.trigger.DeviceActivityTrigger;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.TsValue;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor;
@ -88,7 +85,6 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.thingsboard.server.service.state.DefaultDeviceStateService.ACTIVITY_STATE;
import static org.thingsboard.server.service.state.DefaultDeviceStateService.INACTIVITY_ALARM_TIME;
import static org.thingsboard.server.service.state.DefaultDeviceStateService.INACTIVITY_TIMEOUT;
import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAST_ACTIVITY_TIME;
import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAST_CONNECT_TIME;
import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAST_DISCONNECT_TIME;
@ -211,7 +207,7 @@ public class DefaultDeviceStateServiceTest {
// THEN
then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) &&
request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(LAST_CONNECT_TIME) &&
request.getEntries().get(0).getValue().equals(lastConnectTime)
@ -298,7 +294,7 @@ public class DefaultDeviceStateServiceTest {
// THEN
then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) &&
request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(LAST_DISCONNECT_TIME) &&
request.getEntries().get(0).getValue().equals(lastDisconnectTime)
@ -421,13 +417,13 @@ public class DefaultDeviceStateServiceTest {
// THEN
then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) &&
request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) &&
request.getEntries().get(0).getValue().equals(lastInactivityTime)
));
then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) &&
request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) &&
request.getEntries().get(0).getValue().equals(false)
@ -465,12 +461,12 @@ public class DefaultDeviceStateServiceTest {
// THEN
then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) &&
request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME)
));
then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) &&
request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) &&
request.getEntries().get(0).getValue().equals(false)
@ -508,42 +504,6 @@ public class DefaultDeviceStateServiceTest {
verify(service).fetchDeviceStateDataUsingSeparateRequests(deviceId);
}
@Test
public void givenPersistToTelemetryAndDefaultInactivityTimeoutFetched_whenTransformingToDeviceStateData_thenTryGetInactivityFromAttribute() {
var defaultInactivityTimeoutInSec = 60L;
var latest =
Map.of(
EntityKeyType.TIME_SERIES, Map.of(INACTIVITY_TIMEOUT, new TsValue(0, Long.toString(defaultInactivityTimeoutInSec * 1000))),
EntityKeyType.SERVER_ATTRIBUTE, Map.of(INACTIVITY_TIMEOUT, new TsValue(0, Long.toString(5000L)))
);
process(latest, defaultInactivityTimeoutInSec);
}
@Test
public void givenPersistToTelemetryAndNoInactivityTimeoutFetchedFromTimeSeries_whenTransformingToDeviceStateData_thenTryGetInactivityFromAttribute() {
var defaultInactivityTimeoutInSec = 60L;
var latest =
Map.of(
EntityKeyType.SERVER_ATTRIBUTE, Map.of(INACTIVITY_TIMEOUT, new TsValue(0, Long.toString(5000L)))
);
process(latest, defaultInactivityTimeoutInSec);
}
private void process(Map<EntityKeyType, Map<String, TsValue>> latest, long defaultInactivityTimeoutInSec) {
service.setDefaultInactivityTimeoutInSec(defaultInactivityTimeoutInSec);
service.setDefaultInactivityTimeoutMs(defaultInactivityTimeoutInSec * 1000);
service.setPersistToTelemetry(true);
var deviceUuid = UUID.randomUUID();
var deviceId = new DeviceId(deviceUuid);
DeviceStateData deviceStateData = service.toDeviceStateData(new EntityData(deviceId, latest, Map.of()), new DeviceIdInfo(TenantId.SYS_TENANT_ID.getId(), UUID.randomUUID(), deviceUuid));
assertThat(deviceStateData.getState().getInactivityTimeout()).isEqualTo(5000L);
}
private void initStateService(long timeout) throws InterruptedException {
service.stop();
reset(service, telemetrySubscriptionService);
@ -556,7 +516,7 @@ public class DefaultDeviceStateServiceTest {
.thenReturn(new PageData<>(List.of(deviceIdInfo), 0, 1, false));
PartitionChangeEvent event = new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of(
new QueueKey(ServiceType.TB_CORE), Collections.singleton(tpi)
));
), Collections.emptyMap());
service.onApplicationEvent(event);
Thread.sleep(100);
}
@ -1002,7 +962,7 @@ public class DefaultDeviceStateServiceTest {
assertThat(actualNotification.isActive()).isFalse();
then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) &&
request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) &&
request.getEntries().get(0).getValue().equals(expectedLastInactivityAlarmTime)
@ -1170,7 +1130,7 @@ public class DefaultDeviceStateServiceTest {
assertThat(attributeRequestCaptor.getAllValues()).hasSize(2)
.anySatisfy(request -> {
assertThat(request.getTenantId()).isEqualTo(TenantId.SYS_TENANT_ID);
assertThat(request.getTenantId()).isEqualTo(tenantId);
assertThat(request.getEntityId()).isEqualTo(deviceId);
assertThat(request.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE);
assertThat(request.getEntries()).singleElement().satisfies(attributeKvEntry -> {
@ -1179,7 +1139,7 @@ public class DefaultDeviceStateServiceTest {
});
})
.anySatisfy(request -> {
assertThat(request.getTenantId()).isEqualTo(TenantId.SYS_TENANT_ID);
assertThat(request.getTenantId()).isEqualTo(tenantId);
assertThat(request.getEntityId()).isEqualTo(deviceId);
assertThat(request.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE);
assertThat(request.getEntries()).singleElement().satisfies(attributeKvEntry -> {

80
application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java

@ -43,6 +43,15 @@ import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.asset.AssetProfile;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Argument;
import org.thingsboard.server.common.data.cf.configuration.ArgumentType;
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration;
import org.thingsboard.server.common.data.cf.configuration.Output;
import org.thingsboard.server.common.data.cf.configuration.OutputType;
import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey;
import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration;
import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration;
import org.thingsboard.server.common.data.device.data.DeviceData;
import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration;
@ -70,6 +79,7 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.sync.ie.DeviceExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
@ -79,6 +89,7 @@ import org.thingsboard.server.common.data.util.ThrowingRunnable;
import org.thingsboard.server.controller.AbstractControllerTest;
import org.thingsboard.server.dao.asset.AssetProfileService;
import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.dao.cf.CalculatedFieldService;
import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.dao.dashboard.DashboardService;
import org.thingsboard.server.dao.device.DeviceProfileService;
@ -145,6 +156,8 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest {
protected TenantService tenantService;
@Autowired
protected EntityViewService entityViewService;
@Autowired
protected CalculatedFieldService calculatedFieldService;
protected TenantId tenantId1;
protected User tenantAdmin1;
@ -191,6 +204,7 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest {
Asset asset = createAsset(tenantId1, null, assetProfile.getId(), "Asset 1");
DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1");
Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device 1");
CalculatedField calculatedField = createCalculatedField(tenantId1, device.getId(), asset.getId());
Map<EntityType, EntityExportData> entitiesExportData = Stream.of(customer.getId(), asset.getId(), device.getId(),
ruleChain.getId(), dashboard.getId(), assetProfile.getId(), deviceProfile.getId())
@ -198,6 +212,7 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest {
try {
return exportEntity(tenantAdmin1, entityId, EntityExportSettings.builder()
.exportCredentials(false)
.exportCalculatedFields(true)
.build());
} catch (Exception e) {
throw new RuntimeException(e);
@ -245,7 +260,6 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest {
verify(entityActionService, Mockito.never()).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset),
any(), eq(ActionType.UPDATED), isNull());
EntityExportData<Asset> updatedAssetEntity = getAndClone(entitiesExportData, EntityType.ASSET);
updatedAssetEntity.getEntity().setLabel("t" + updatedAssetEntity.getEntity().getLabel());
Asset updatedAsset = importEntity(tenantAdmin2, updatedAssetEntity).getSavedEntity();
@ -268,10 +282,27 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest {
importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.DEVICE));
verify(tbClusterService, Mockito.never()).onDeviceUpdated(eq(importedDevice), eq(importedDevice));
// calculated field of imported device:
List<CalculatedField> calculatedFields = calculatedFieldService.findCalculatedFieldsByEntityId(tenantId2, importedDevice.getId());
assertThat(calculatedFields.size()).isOne();
var importedCalculatedField = calculatedFields.get(0);
assertThat(importedCalculatedField.getName()).isEqualTo(calculatedField.getName());
verify(tbClusterService).onCalculatedFieldUpdated(eq(importedCalculatedField), isNull(), any());
EntityExportData<Device> updatedDeviceEntity = getAndClone(entitiesExportData, EntityType.DEVICE);
updatedDeviceEntity.getEntity().setLabel("t" + updatedDeviceEntity.getEntity().getLabel());
Device updatedDevice = importEntity(tenantAdmin2, updatedDeviceEntity).getSavedEntity();
verify(tbClusterService).onDeviceUpdated(eq(updatedDevice), eq(importedDevice));
// update calculated field:
DeviceExportData deviceExportData = (DeviceExportData) getAndClone(entitiesExportData, EntityType.DEVICE);
deviceExportData.setCalculatedFields(deviceExportData.getCalculatedFields().stream().peek(field -> field.setName("t_" + field.getName())).toList());
importEntity(tenantAdmin2, deviceExportData).getSavedEntity();
calculatedFields = calculatedFieldService.findCalculatedFieldsByEntityId(tenantId2, importedDevice.getId());
assertThat(calculatedFields.size()).isOne();
importedCalculatedField = calculatedFields.get(0);
assertThat(importedCalculatedField.getName()).startsWith("t_");
}
@Test
@ -290,12 +321,15 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest {
Device device = createDevice(tenantId1, customer.getId(), deviceProfile.getId(), "Device 1");
EntityView entityView = createEntityView(tenantId1, customer.getId(), device.getId(), "Entity view 1");
CalculatedField calculatedField = createCalculatedField(tenantId1, device.getId(), device.getId());
Map<EntityId, EntityId> ids = new HashMap<>();
for (EntityId entityId : List.of(customer.getId(), ruleChain.getId(), dashboard.getId(), assetProfile.getId(), asset.getId(),
deviceProfile.getId(), device.getId(), entityView.getId(), ruleChain.getId(), dashboard.getId())) {
EntityExportData exportData = exportEntity(getSecurityUser(tenantAdmin1), entityId);
EntityImportResult importResult = importEntity(getSecurityUser(tenantAdmin2), exportData, EntityImportSettings.builder()
.saveCredentials(false)
.saveCalculatedFields(true)
.build());
ids.put(entityId, (EntityId) importResult.getSavedEntity().getId());
}
@ -325,10 +359,16 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest {
assertThat(exportedDeviceProfile.getDefaultRuleChainId()).isEqualTo(ruleChain.getId());
assertThat(exportedDeviceProfile.getDefaultDashboardId()).isEqualTo(dashboard.getId());
Device exportedDevice = (Device) exportEntity(tenantAdmin2, (DeviceId) ids.get(device.getId())).getEntity();
EntityExportData<Device> entityExportData = exportEntity(tenantAdmin2, (DeviceId) ids.get(device.getId()));
Device exportedDevice = entityExportData.getEntity();
assertThat(exportedDevice.getCustomerId()).isEqualTo(customer.getId());
assertThat(exportedDevice.getDeviceProfileId()).isEqualTo(deviceProfile.getId());
List<CalculatedField> calculatedFields = ((DeviceExportData) entityExportData).getCalculatedFields();
assertThat(calculatedFields.size()).isOne();
CalculatedField field = calculatedFields.get(0);
assertThat(field.getName()).isEqualTo(calculatedField.getName());
EntityView exportedEntityView = (EntityView) exportEntity(tenantAdmin2, (EntityViewId) ids.get(entityView.getId())).getEntity();
assertThat(exportedEntityView.getCustomerId()).isEqualTo(customer.getId());
assertThat(exportedEntityView.getEntityId()).isEqualTo(device.getId());
@ -340,7 +380,6 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest {
deviceProfileService.saveDeviceProfile(importedDeviceProfile);
}
protected Device createDevice(TenantId tenantId, CustomerId customerId, DeviceProfileId deviceProfileId, String name) {
Device device = new Device();
device.setTenantId(tenantId);
@ -549,9 +588,43 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest {
return relation;
}
private CalculatedField createCalculatedField(TenantId tenantId, EntityId entityId, EntityId referencedEntityId) {
CalculatedField calculatedField = new CalculatedField();
calculatedField.setTenantId(tenantId);
calculatedField.setEntityId(entityId);
calculatedField.setType(CalculatedFieldType.SIMPLE);
calculatedField.setName("Test Calculated Field");
calculatedField.setConfigurationVersion(1);
calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId));
calculatedField.setVersion(1L);
return calculatedFieldService.save(calculatedField);
}
private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) {
SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration();
Argument argument = new Argument();
argument.setRefEntityId(referencedEntityId);
ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null);
argument.setRefEntityKey(refEntityKey);
config.setArguments(Map.of("T", argument));
config.setExpression("T - (100 - H) / 5");
Output output = new Output();
output.setName("output");
output.setType(OutputType.TIME_SERIES);
config.setOutput(output);
return config;
}
protected <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> exportEntity(User user, I entityId) throws Exception {
return exportEntity(user, entityId, EntityExportSettings.builder()
.exportCredentials(true)
.exportCalculatedFields(true)
.build());
}
@ -562,6 +635,7 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest {
protected <E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> importEntity(User user, EntityExportData<E> exportData) throws Exception {
return importEntity(user, exportData, EntityImportSettings.builder()
.saveCredentials(true)
.saveCalculatedFields(true)
.build());
}

128
application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java

@ -44,6 +44,15 @@ import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.asset.AssetProfile;
import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Argument;
import org.thingsboard.server.common.data.cf.configuration.ArgumentType;
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration;
import org.thingsboard.server.common.data.cf.configuration.Output;
import org.thingsboard.server.common.data.cf.configuration.OutputType;
import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey;
import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration;
import org.thingsboard.server.common.data.debug.DebugSettings;
import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration;
import org.thingsboard.server.common.data.device.data.DeviceData;
@ -562,6 +571,81 @@ public class VersionControlTest extends AbstractControllerTest {
checkImportedEntity(tenantId1, defaultDeviceProfile, tenantId2, importedDeviceProfile);
}
@Test
public void testVcWithCalculatedFields_betweenTenants() throws Exception {
Asset asset = createAsset(null, null, "Asset 1");
Device device = createDevice(null, null, "Device 1", "test1");
CalculatedField calculatedField = createCalculatedField("CalculatedField1", device.getId(), asset.getId());
String versionId = createVersion("calculated fields of asset and device", EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE);
loginTenant2();
loadVersion(versionId, config -> {
config.setLoadCredentials(false);
}, EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE);
Asset importedAsset = findAsset(asset.getName());
Device importedDevice = findDevice(device.getName());
checkImportedEntity(tenantId1, device, tenantId2, importedDevice);
checkImportedEntity(tenantId1, asset, tenantId2, importedAsset);
List<CalculatedField> importedCalculatedFields = findCalculatedFieldsByEntityId(importedDevice.getId());
assertThat(importedCalculatedFields).size().isOne();
assertThat(importedCalculatedFields.get(0)).satisfies(importedField -> {
assertThat(importedField.getName()).isEqualTo(calculatedField.getName());
assertThat(importedField.getType()).isEqualTo(calculatedField.getType());
assertThat(importedField.getId()).isNotEqualTo(calculatedField.getId());
});
}
@Test
public void testVcWithReferencedCalculatedFields_betweenTenants() throws Exception {
Asset asset = createAsset(null, null, "Asset 1");
Device device = createDevice(null, null, "Device 1", "test1");
CalculatedField deviceCalculatedField = createCalculatedField("CalculatedField1", device.getId(), asset.getId());
CalculatedField assetCalculatedField = createCalculatedField("CalculatedField2", asset.getId(), device.getId());
String versionId = createVersion("calculated fields of asset and device", EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE);
loginTenant2();
loadVersion(versionId, config -> {
config.setLoadCredentials(false);
}, EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE);
Asset importedAsset = findAsset(asset.getName());
Device importedDevice = findDevice(device.getName());
checkImportedEntity(tenantId1, device, tenantId2, importedDevice);
checkImportedEntity(tenantId1, asset, tenantId2, importedAsset);
List<CalculatedField> importedDeviceCalculatedFields = findCalculatedFieldsByEntityId(importedDevice.getId());
assertThat(importedDeviceCalculatedFields).size().isOne();
assertThat(importedDeviceCalculatedFields.get(0)).satisfies(importedField -> {
assertThat(importedField.getName()).isEqualTo(deviceCalculatedField.getName());
assertThat(importedField.getType()).isEqualTo(deviceCalculatedField.getType());
assertThat(importedField.getId()).isNotEqualTo(deviceCalculatedField.getId());
});
List<CalculatedField> importedAssetCalculatedFields = findCalculatedFieldsByEntityId(importedAsset.getId());
assertThat(importedAssetCalculatedFields).size().isOne();
assertThat(importedAssetCalculatedFields.get(0)).satisfies(importedField -> {
assertThat(importedField.getName()).isEqualTo(assetCalculatedField.getName());
assertThat(importedField.getType()).isEqualTo(assetCalculatedField.getType());
assertThat(importedField.getId()).isNotEqualTo(assetCalculatedField.getId());
});
}
@Test
public void testVcWithCalculatedFields_sameTenant() throws Exception {
Asset asset = createAsset(null, null, "Asset 1");
CalculatedField calculatedField = createCalculatedField("CalculatedField", asset.getId(), asset.getId());
String versionId = createVersion("asset and field", EntityType.ASSET);
loadVersion(versionId, EntityType.ASSET);
CalculatedField importedCalculatedField = findCalculatedFieldByEntityId(asset.getId());
assertThat(importedCalculatedField.getId()).isEqualTo(calculatedField.getId());
assertThat(importedCalculatedField.getName()).isEqualTo(calculatedField.getName());
assertThat(importedCalculatedField.getConfiguration()).isEqualTo(calculatedField.getConfiguration());
assertThat(importedCalculatedField.getType()).isEqualTo(calculatedField.getType());
}
private <E extends ExportableEntity<?> & HasTenantId> void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, E importedEntity) {
assertThat(initialEntity.getTenantId()).isEqualTo(tenantId1);
assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2);
@ -626,10 +710,10 @@ public class VersionControlTest extends AbstractControllerTest {
request.setEntityTypes(Arrays.stream(entityTypes).collect(Collectors.toMap(t -> t, entityType -> {
EntityTypeVersionCreateConfig config = new EntityTypeVersionCreateConfig();
config.setAllEntities(true);
config.setSaveRelations(true);
config.setSaveAttributes(true);
config.setSaveCredentials(true);
config.setSaveCalculatedFields(true);
return config;
})));
@ -695,6 +779,7 @@ public class VersionControlTest extends AbstractControllerTest {
config.setLoadAttributes(true);
config.setLoadRelations(true);
config.setLoadCredentials(true);
config.setLoadCalculatedFields(true);
config.setRemoveOtherEntities(false);
config.setFindExistingEntityByName(true);
configModifier.accept(config);
@ -941,6 +1026,38 @@ public class VersionControlTest extends AbstractControllerTest {
return doPost("/api/v2/relation", relation, EntityRelation.class);
}
private CalculatedField createCalculatedField(String name, EntityId entityId, EntityId referencedEntityId) {
CalculatedField calculatedField = new CalculatedField();
calculatedField.setEntityId(entityId);
calculatedField.setType(CalculatedFieldType.SIMPLE);
calculatedField.setName(name);
calculatedField.setConfigurationVersion(1);
calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId));
calculatedField.setVersion(1L);
return doPost("/api/calculatedField", calculatedField, CalculatedField.class);
}
private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) {
SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration();
Argument argument = new Argument();
argument.setRefEntityId(referencedEntityId);
ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null);
argument.setRefEntityKey(refEntityKey);
config.setArguments(Map.of("T", argument));
config.setExpression("T - (100 - H) / 5");
Output output = new Output();
output.setName("output");
output.setType(OutputType.TIME_SERIES);
config.setOutput(output);
return config;
}
protected void checkImportedRuleChainData(RuleChain initialRuleChain, RuleChainMetaData initialMetaData, RuleChain importedRuleChain, RuleChainMetaData importedMetaData) {
assertThat(importedRuleChain.getType()).isEqualTo(initialRuleChain.getType());
assertThat(importedRuleChain.getName()).isEqualTo(initialRuleChain.getName());
@ -995,11 +1112,18 @@ public class VersionControlTest extends AbstractControllerTest {
private RuleChain findRuleChain(String name) throws Exception {
return doGetTypedWithPageLink("/api/ruleChains?", new TypeReference<PageData<RuleChain>>() {}, new PageLink(100, 0, name)).getData().get(0);
}
private RuleChainMetaData findRuleChainMetaData(RuleChainId ruleChainId) throws Exception {
return doGet("/api/ruleChain/" + ruleChainId + "/metadata", RuleChainMetaData.class);
}
private CalculatedField findCalculatedFieldByEntityId(EntityId entityId) throws Exception {
return doGetTypedWithPageLink("/api/" + entityId.getEntityType() + "/" + entityId.getId() + "/calculatedFields?", new TypeReference<PageData<CalculatedField>>() {}, new PageLink(100, 0)).getData().get(0);
}
private List<CalculatedField> findCalculatedFieldsByEntityId(EntityId entityId) throws Exception {
return doGetTypedWithPageLink("/api/" + entityId.getEntityType() + "/" + entityId.getId() + "/calculatedFields?", new TypeReference<PageData<CalculatedField>>() {}, new PageLink(100, 0)).getData();
}
}

26
application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java

@ -174,7 +174,7 @@ class DefaultTelemetrySubscriptionServiceTest {
}).when(calculatedFieldQueueService).pushRequestToQueue(any(TimeseriesSaveRequest.class), any(), any());
// send partition change event so currentPartitions set is populated
telemetryService.onTbApplicationEvent(new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of(new QueueKey(ServiceType.TB_CORE), Set.of(tpi))));
telemetryService.onTbApplicationEvent(new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of(new QueueKey(ServiceType.TB_CORE), Set.of(tpi)), Collections.emptyMap()));
}
@AfterEach
@ -183,6 +183,28 @@ class DefaultTelemetrySubscriptionServiceTest {
tsCallBackExecutor.shutdownNow();
}
/* --- Save time series API --- */
@Test
void shouldThrowErrorWhenTryingToSaveTimeseriesForApiUsageState() {
// GIVEN
var request = TimeseriesSaveRequest.builder()
.tenantId(tenantId)
.customerId(customerId)
.entityId(new ApiUsageStateId(UUID.randomUUID()))
.entries(sampleTimeseries)
.strategy(TimeseriesSaveRequest.Strategy.PROCESS_ALL)
.build();
// WHEN
assertThatThrownBy(() -> telemetryService.saveTimeseries(request))
.isInstanceOf(RuntimeException.class)
.hasMessage("Can't update API Usage State!");
// THEN
then(tsService).shouldHaveNoInteractions();
}
@Test
void shouldReportStorageDataPointsApiUsageWhenTimeSeriesIsSaved() {
// GIVEN
@ -392,6 +414,8 @@ class DefaultTelemetrySubscriptionServiceTest {
);
}
/* --- Save attributes API --- */
@ParameterizedTest
@MethodSource("allCombinationsOfThreeBooleans")
void shouldCallCorrectSaveAttributesApiBasedOnBooleanFlagsInTheSaveRequest(boolean saveAttributes, boolean sendWsUpdate, boolean processCalculatedFields) {

3
application/src/test/resources/application-test.properties

@ -58,3 +58,6 @@ sql.ttl.edge_events.edge_event_ttl=2592000
server.log_controller_error_stack_trace=false
transport.gateway.dashboard.sync.enabled=false
queue.edqs.sync.enabled=false
queue.edqs.api.supported=false

6
application/src/test/resources/update/330/device_profile_001_out.json

@ -64,7 +64,8 @@
"dynamicValue": {
"sourceType": null,
"sourceAttribute": null,
"inherit": false
"inherit": false,
"resolvedValue" : null
}
}
}
@ -103,7 +104,8 @@
"dynamicValue": {
"sourceType": null,
"sourceAttribute": null,
"inherit": false
"inherit": false,
"resolvedValue" : null
}
}
}

2
common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java

@ -26,6 +26,8 @@ public interface TbQueueRequestTemplate<Request extends TbQueueMsg, Response ext
ListenableFuture<Response> send(Request request, long timeoutNs);
ListenableFuture<Response> send(Request request, Integer partition);
void stop();
void setMessagesStats(MessagesStats messagesStats);

10
common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java

@ -15,9 +15,17 @@
*/
package org.thingsboard.server.queue;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import java.util.Set;
public interface TbQueueResponseTemplate<Request extends TbQueueMsg, Response extends TbQueueMsg> {
void init(TbQueueHandler<Request, Response> handler);
void subscribe();
void subscribe(Set<TopicPartitionInfo> partitions);
void launch(TbQueueHandler<Request, Response> handler);
void stop();
}

2
common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java

@ -118,6 +118,8 @@ public interface AlarmService extends EntityDaoService {
long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query);
long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query, Collection<EntityId> orderedEntityIds);
PageData<EntitySubtype> findAlarmTypesByTenantId(TenantId tenantId, PageLink pageLink);
List<UUID> findActiveOriginatorAlarms(TenantId tenantId, OriginatorAlarmFilter originatorAlarmFilter, int limit);

2
common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java

@ -46,6 +46,8 @@ public interface RuleChainService extends EntityDaoService {
RuleChain saveRuleChain(RuleChain ruleChain, boolean publishSaveEvent);
RuleChain saveRuleChain(RuleChain ruleChain, boolean publishSaveEvent, boolean doValidate);
boolean setRootRuleChain(TenantId tenantId, RuleChainId ruleChainId);
RuleChainUpdateResult saveRuleChainMetaData(TenantId tenantId, RuleChainMetaData ruleChainMetaData, Function<RuleNode, RuleNode> ruleNodeUpdater);

89
common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java

@ -0,0 +1,89 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
public enum ObjectType {
TENANT,
TENANT_PROFILE,
CUSTOMER,
QUEUE,
RPC,
RULE_CHAIN,
OTA_PACKAGE,
RESOURCE,
EVENT,
RULE_NODE,
USER,
EDGE,
WIDGETS_BUNDLE,
WIDGET_TYPE,
DASHBOARD,
DEVICE_PROFILE,
DEVICE,
DEVICE_CREDENTIALS,
ASSET_PROFILE,
ASSET,
ENTITY_VIEW,
ALARM,
ENTITY_ALARM,
OAUTH2_CLIENT,
OAUTH2_DOMAIN,
OAUTH2_MOBILE,
USER_SETTINGS,
NOTIFICATION_TARGET,
NOTIFICATION_TEMPLATE,
NOTIFICATION_RULE,
ALARM_COMMENT,
API_USAGE_STATE,
QUEUE_STATS,
AUDIT_LOG,
RELATION,
ATTRIBUTE_KV,
LATEST_TS_KV;
public static final Set<ObjectType> edqsTenantTypes = EnumSet.of(
TENANT, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD,
RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, API_USAGE_STATE, QUEUE_STATS
);
public static final Set<ObjectType> edqsTypes = EnumSet.copyOf(edqsTenantTypes);
public static final Set<ObjectType> edqsSystemTypes = EnumSet.of(TENANT, USER, DASHBOARD,
API_USAGE_STATE, ATTRIBUTE_KV, LATEST_TS_KV);
public static final Set<ObjectType> unversionedTypes = EnumSet.of(
QUEUE_STATS // created once, never updated
);
static {
edqsTypes.addAll(List.of(RELATION, ATTRIBUTE_KV, LATEST_TS_KV));
}
public EntityType toEntityType() {
return EntityType.valueOf(name());
}
public static ObjectType fromEntityType(EntityType entityType) {
try {
return ObjectType.valueOf(entityType.name());
} catch (Exception e) {
return null;
}
}
}

1
common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java

@ -19,7 +19,6 @@ import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.AssetProfileId;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.EntityId;

26
common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java

@ -23,7 +23,6 @@ import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import org.thingsboard.server.common.data.BaseData;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.HasDebugSettings;
import org.thingsboard.server.common.data.HasName;
import org.thingsboard.server.common.data.HasTenantId;
@ -37,11 +36,14 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.validation.Length;
import org.thingsboard.server.common.data.validation.NoXss;
import java.io.Serial;
@Schema
@Data
@EqualsAndHashCode(callSuper = true)
public class CalculatedField extends BaseData<CalculatedFieldId> implements HasName, HasTenantId, HasVersion, ExportableEntity<CalculatedFieldId>, HasDebugSettings {
public class CalculatedField extends BaseData<CalculatedFieldId> implements HasName, HasTenantId, HasVersion, HasDebugSettings {
@Serial
private static final long serialVersionUID = 4491966747773381420L;
private TenantId tenantId;
@ -66,9 +68,6 @@ public class CalculatedField extends BaseData<CalculatedFieldId> implements HasN
@Getter
@Setter
private Long version;
@Getter
@Setter
private CalculatedFieldId externalId;
public CalculatedField() {
super();
@ -78,7 +77,7 @@ public class CalculatedField extends BaseData<CalculatedFieldId> implements HasN
super(id);
}
public CalculatedField(TenantId tenantId, EntityId entityId, CalculatedFieldType type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version, CalculatedFieldId externalId) {
public CalculatedField(TenantId tenantId, EntityId entityId, CalculatedFieldType type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version) {
this.tenantId = tenantId;
this.entityId = entityId;
this.type = type;
@ -86,7 +85,19 @@ public class CalculatedField extends BaseData<CalculatedFieldId> implements HasN
this.configurationVersion = configurationVersion;
this.configuration = configuration;
this.version = version;
this.externalId = externalId;
}
public CalculatedField(CalculatedField calculatedField) {
super(calculatedField);
this.tenantId = calculatedField.tenantId;
this.entityId = calculatedField.entityId;
this.type = calculatedField.type;
this.name = calculatedField.name;
this.debugMode = calculatedField.debugMode;
this.debugSettings = calculatedField.debugSettings;
this.configurationVersion = calculatedField.configurationVersion;
this.configuration = calculatedField.configuration;
this.version = calculatedField.version;
}
@Schema(description = "JSON object with the Calculated Field Id. Referencing non-existing Calculated Field Id will cause error.")
@ -112,7 +123,6 @@ public class CalculatedField extends BaseData<CalculatedFieldId> implements HasN
.append(", configurationVersion=").append(configurationVersion)
.append(", configuration=").append(configuration)
.append(", version=").append(version)
.append(", externalId=").append(externalId)
.append(", createdTime=").append(createdTime)
.append(", id=").append(id).append(']')
.toString();

1
common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java

@ -26,5 +26,6 @@ public class Output {
private String name;
private OutputType type;
private AttributeScope scope;
private Integer decimalsByDefault;
}

75
common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java

@ -0,0 +1,75 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.edqs;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.ObjectType;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.KvEntry;
@Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class AttributeKv implements EdqsObject {
private EntityId entityId;
private AttributeScope scope;
private String key;
private Long version;
private DataPoint dataPoint; // optional (on deletion)
private Long lastUpdateTs; // only for serialization
private KvEntry value; // only for serialization
public AttributeKv(EntityId entityId, AttributeScope scope, AttributeKvEntry attributeKvEntry, long version) {
this.entityId = entityId;
this.scope = scope;
this.key = attributeKvEntry.getKey();
this.version = version;
this.lastUpdateTs = attributeKvEntry.getLastUpdateTs();
this.value = attributeKvEntry;
}
public AttributeKv(EntityId entityId, AttributeScope scope, String key, long version) {
this.entityId = entityId;
this.scope = scope;
this.key = key;
this.version = version;
}
@Override
public String key() {
return "a_" + entityId + "_" + scope + "_" + key;
}
@Override
public Long version() {
return version;
}
@Override
public ObjectType type() {
return ObjectType.ATTRIBUTE_KV;
}
}

40
common/data/src/main/java/org/thingsboard/server/common/data/edqs/DataPoint.java

@ -0,0 +1,40 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.edqs;
import org.thingsboard.server.common.data.kv.DataType;
public interface DataPoint {
String NOT_SUPPORTED = "Not supported!";
long getTs();
DataType getType();
String getStr();
long getLong();
double getDouble();
boolean getBool();
String getJson();
String valueToString();
}

34
common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEvent.java

@ -0,0 +1,34 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.edqs;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import org.thingsboard.server.common.data.ObjectType;
import org.thingsboard.server.common.data.id.TenantId;
@Data
@AllArgsConstructor
@Builder
public class EdqsEvent {
private final TenantId tenantId;
private final ObjectType objectType;
private final EdqsEventType eventType;
private final EdqsObject object;
}

21
common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEventType.java

@ -0,0 +1,21 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.edqs;
public enum EdqsEventType {
UPDATED,
DELETED
}

32
common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java

@ -0,0 +1,32 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.edqs;
import com.fasterxml.jackson.annotation.JsonIgnore;
import org.thingsboard.server.common.data.ObjectType;
public interface EdqsObject {
@JsonIgnore
String key();
@JsonIgnore
Long version();
@JsonIgnore
ObjectType type();
}

24
common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsSyncRequest.java

@ -0,0 +1,24 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.edqs;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;
@Data
@JsonIgnoreProperties
public class EdqsSyncRequest {
}

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save