Browse Source

Merge branch 'master' of github.com:thingsboard/thingsboard into aggregation-cf/improvements

pull/14565/head
IrynaMatveieva 10 months ago
parent
commit
d3e1d2123b
  1. 26
      application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java
  2. 7
      application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java
  3. 3
      application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java
  4. 24
      application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java
  5. 4
      application/src/main/java/org/thingsboard/server/controller/BaseController.java
  6. 2
      application/src/main/java/org/thingsboard/server/controller/EdgeController.java
  7. 1
      application/src/main/java/org/thingsboard/server/exception/ThingsboardErrorResponseHandler.java
  8. 24
      application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java
  9. 8
      application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java
  10. 4
      application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java
  11. 7
      application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java
  12. 39
      application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java
  13. 10
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java
  14. 51
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java
  15. 36
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java
  16. 15
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java
  17. 24
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java
  18. 4
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java
  19. 2
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java
  20. 19
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java
  21. 21
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/ScheduledRefreshSupported.java
  22. 46
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java
  23. 67
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java
  24. 3
      application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java
  25. 1
      application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java
  26. 12
      application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java
  27. 3
      application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java
  28. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/ai/AiModelEdgeProcessor.java
  29. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java
  30. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessor.java
  31. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/cf/CalculatedFieldEdgeProcessor.java
  32. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java
  33. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java
  34. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessor.java
  35. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java
  36. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java
  37. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java
  38. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java
  39. 13
      application/src/main/java/org/thingsboard/server/service/edge/stats/EdgeStatsService.java
  40. 2
      application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java
  41. 2
      application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java
  42. 2
      application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java
  43. 2
      application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/DefaultJwtSettingsValidator.java
  44. 2
      application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java
  45. 2
      application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java
  46. 2
      application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java
  47. 2
      application/src/main/java/org/thingsboard/server/service/security/system/SystemSecurityService.java
  48. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java
  49. 2
      application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java
  50. 12
      application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java
  51. 2
      application/src/main/java/org/thingsboard/server/utils/LwM2mObjectModelUtils.java
  52. 68
      application/src/main/resources/thingsboard.yml
  53. 57
      application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java
  54. 2
      application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java
  55. 2
      application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java
  56. 2
      application/src/test/java/org/thingsboard/server/controller/AssetProfileControllerTest.java
  57. 2
      application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java
  58. 2
      application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java
  59. 2
      application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java
  60. 2
      application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java
  61. 2
      application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java
  62. 2
      application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java
  63. 2
      application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java
  64. 2
      application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java
  65. 2
      application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java
  66. 2
      application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java
  67. 2
      application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java
  68. 148
      application/src/test/java/org/thingsboard/server/edge/EdgeStatsIntegrationTest.java
  69. 77
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationArgumentEntryTest.java
  70. 141
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java
  71. 133
      application/src/test/java/org/thingsboard/server/service/edge/EdgeStatsTest.java
  72. 2
      application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java
  73. 62
      application/src/test/java/org/thingsboard/server/service/security/auth/pat/ApiKeyAuthenticationProviderTest.java
  74. 5
      application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java
  75. 14
      common/data/src/main/java/org/thingsboard/server/common/data/device/profile/Lwm2mDeviceProfileTransportConfiguration.java
  76. 5
      common/data/src/main/java/org/thingsboard/server/exception/DataValidationException.java
  77. 4
      common/data/src/main/java/org/thingsboard/server/exception/EntitiesLimitExceededException.java
  78. 51
      common/data/src/main/java/org/thingsboard/server/exception/ThingsboardRuntimeException.java
  79. 1
      common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeRpcClient.java
  80. 1
      common/proto/src/main/proto/queue.proto
  81. 29
      common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaAdmin.java
  82. 1
      common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java
  83. 25
      common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClientContextImpl.java
  84. 58
      common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java
  85. 13
      common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisRegistrationStore.java
  86. 22
      common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2mUplinkMsgHandler.java
  87. 2
      dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java
  88. 2
      dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java
  89. 2
      dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java
  90. 2
      dao/src/main/java/org/thingsboard/server/dao/component/BaseComponentDescriptorService.java
  91. 2
      dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java
  92. 2
      dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java
  93. 2
      dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsServiceImpl.java
  94. 2
      dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java
  95. 2
      dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java
  96. 2
      dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java
  97. 3
      dao/src/main/java/org/thingsboard/server/dao/edge/PostgresEdgeEventService.java
  98. 15
      dao/src/main/java/org/thingsboard/server/dao/edge/stats/EdgeStatsCounterService.java
  99. 2
      dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java
  100. 2
      dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java

26
application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java

@ -132,6 +132,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
} else { } else {
removeState(cfId); removeState(cfId);
} }
msg.getCallback().onSuccess();
} }
public void process(CalculatedFieldStatePartitionRestoreMsg msg) { public void process(CalculatedFieldStatePartitionRestoreMsg msg) {
@ -242,7 +243,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
} }
if (state instanceof PropagationCalculatedFieldState propagationState) { if (state instanceof PropagationCalculatedFieldState propagationState) {
PropagationArgumentEntry entry = new PropagationArgumentEntry(); PropagationArgumentEntry entry = new PropagationArgumentEntry();
entry.setAdded(msg.getRelatedEntityId()); entry.setAdded(List.of(msg.getRelatedEntityId()));
updatedArgs = propagationState.update(Map.of(PROPAGATION_CONFIG_ARGUMENT, entry), ctx); updatedArgs = propagationState.update(Map.of(PROPAGATION_CONFIG_ARGUMENT, entry), ctx);
} }
if (CollectionsUtil.isEmpty(updatedArgs)) { if (CollectionsUtil.isEmpty(updatedArgs)) {
@ -421,19 +422,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
if (state == null) { if (state == null) {
state = createState(ctx); state = createState(ctx);
justRestored = true; justRestored = true;
} else if (ctx.shouldFetchRelationQueryDynamicArgumentsFromDb(state)) { } else if (ctx.shouldFetchRelatedEntities(state)) {
log.debug("[{}][{}] Going to update dynamic arguments for CF.", entityId, ctx.getCfId());
try {
Map<String, ArgumentEntry> dynamicArgsFromDb = cfService.fetchDynamicArgsFromDb(ctx, entityId);
dynamicArgsFromDb.forEach(newArgValues::putIfAbsent);
if (ctx.getCfType() == CalculatedFieldType.GEOFENCING) {
var geofencingState = (GeofencingCalculatedFieldState) state;
geofencingState.updateLastDynamicArgumentsRefreshTs();
}
} catch (Exception e) {
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build();
}
} else if (ctx.shouldFetchEntityRelations(state)) {
log.debug("[{}][{}] Going to update related entities for CF.", entityId, ctx.getCfId()); log.debug("[{}][{}] Going to update related entities for CF.", entityId, ctx.getCfId());
try { try {
if (state instanceof RelatedEntitiesAggregationCalculatedFieldState relatedEntitiesState) { if (state instanceof RelatedEntitiesAggregationCalculatedFieldState relatedEntitiesState) {
@ -447,6 +436,11 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
justRestored = true; justRestored = true;
} }
} }
if (state instanceof GeofencingCalculatedFieldState geofencingCalculatedFieldState) {
Map<String, ArgumentEntry> dynamicArgsFromDb = cfService.fetchDynamicArgsFromDb(ctx, entityId);
dynamicArgsFromDb.forEach(newArgValues::putIfAbsent);
geofencingCalculatedFieldState.updateScheduledRefreshTs();
}
} catch (Exception e) { } catch (Exception e) {
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build();
} }
@ -476,9 +470,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
state.setCtx(ctx, actorCtx); state.setCtx(ctx, actorCtx);
state.init(false); state.init(false);
if (ctx.getCfType() == CalculatedFieldType.GEOFENCING && ctx.isRelationQueryDynamicArguments()) { if (ctx.getCfType() == CalculatedFieldType.GEOFENCING && ctx.isCfHasRelationPathQuerySource()) {
GeofencingCalculatedFieldState geofencingState = (GeofencingCalculatedFieldState) state; GeofencingCalculatedFieldState geofencingState = (GeofencingCalculatedFieldState) state;
geofencingState.updateLastDynamicArgumentsRefreshTs(); geofencingState.updateScheduledRefreshTs();
} }
Map<String, ArgumentEntry> arguments = fetchArguments(ctx); Map<String, ArgumentEntry> arguments = fetchArguments(ctx);

7
application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java

@ -167,10 +167,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware
if (ctx != null) { if (ctx != null) {
msg.setCtx(ctx); msg.setCtx(ctx);
log.debug("Pushing CF state restore msg to specific actor [{}]", msg.getId().entityId()); log.debug("[{}] Pushing CF state restore msg to specific actor [{}]", tenantId, msg.getId().entityId());
getOrCreateActor(msg.getId().entityId()).tellWithHighPriority(msg); getOrCreateActor(msg.getId().entityId()).tellWithHighPriority(msg);
} else { } else if (msg.getState() != null) {
log.debug("[{}] Received CF state restore msg for non-existing CF [{}]. Removing state", tenantId, cfId);
cfStateService.deleteState(msg.getId(), msg.getCallback()); cfStateService.deleteState(msg.getId(), msg.getCallback());
} else {
msg.getCallback().onSuccess();
} }
} }

3
application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java

@ -19,6 +19,7 @@ import lombok.Data;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.MsgType;
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
@ -30,6 +31,7 @@ public class CalculatedFieldStateRestoreMsg implements ToCalculatedFieldSystemMs
private final CalculatedFieldEntityCtxId id; private final CalculatedFieldEntityCtxId id;
private final CalculatedFieldState state; private final CalculatedFieldState state;
private final TopicPartitionInfo partition; private final TopicPartitionInfo partition;
private final TbCallback callback;
private CalculatedFieldCtx ctx; private CalculatedFieldCtx ctx;
@Override @Override
@ -41,4 +43,5 @@ public class CalculatedFieldStateRestoreMsg implements ToCalculatedFieldSystemMs
public TenantId getTenantId() { public TenantId getTenantId() {
return id.tenantId(); return id.tenantId();
} }
} }

24
application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java

@ -43,7 +43,6 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.msg.MsgType;
import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.TbActorStopReason; import org.thingsboard.server.common.msg.TbActorStopReason;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
@ -139,13 +138,22 @@ public class TenantActor extends RuleChainManagerActor {
@Override @Override
protected boolean doProcess(TbActorMsg msg) { protected boolean doProcess(TbActorMsg msg) {
if (cantFindTenant) { if (cantFindTenant) {
log.info("[{}] Processing missing Tenant msg: {}", tenantId, msg); log.debug("[{}] Processing message for non-existing tenant: {}", tenantId, msg);
if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) { switch (msg.getMsgType()) {
QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg; case QUEUE_TO_RULE_ENGINE_MSG -> {
queueMsg.getMsg().getCallback().onSuccess(); ((QueueToRuleEngineMsg) msg).getMsg().getCallback().onSuccess();
} else if (msg.getMsgType().equals(MsgType.TRANSPORT_TO_DEVICE_ACTOR_MSG)) { }
TransportToDeviceActorMsgWrapper transportMsg = (TransportToDeviceActorMsgWrapper) msg; case TRANSPORT_TO_DEVICE_ACTOR_MSG -> {
transportMsg.getCallback().onSuccess(); ((TransportToDeviceActorMsgWrapper) msg).getCallback().onSuccess();
}
case CF_STATE_RESTORE_MSG -> {
((CalculatedFieldStateRestoreMsg) msg).getCallback().onSuccess();
}
default -> {
if (!log.isDebugEnabled()) {
log.info("[{}] Processing message for non-existing tenant: {}", tenantId, msg);
}
}
} }
return true; return true;
} }

4
application/src/main/java/org/thingsboard/server/controller/BaseController.java

@ -150,8 +150,6 @@ import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.domain.DomainService; import org.thingsboard.server.dao.domain.DomainService;
import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.exception.EntitiesLimitExceededException;
import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.job.JobService; import org.thingsboard.server.dao.job.JobService;
import org.thingsboard.server.dao.mobile.MobileAppBundleService; import org.thingsboard.server.dao.mobile.MobileAppBundleService;
@ -175,6 +173,8 @@ import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetTypeService;
import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.exception.EntitiesLimitExceededException;
import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler;
import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;

2
application/src/main/java/org/thingsboard/server/controller/EdgeController.java

@ -54,7 +54,7 @@ import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportReques
import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportResult;
import org.thingsboard.server.common.msg.edge.FromEdgeSyncResponse; import org.thingsboard.server.common.msg.edge.FromEdgeSyncResponse;
import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.config.annotations.ApiOperation;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.queue.util.TbCoreComponent;

1
application/src/main/java/org/thingsboard/server/exception/ThingsboardErrorResponseHandler.java

@ -52,7 +52,6 @@ import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.msg.tools.MaxPayloadSizeExceededException; import org.thingsboard.server.common.msg.tools.MaxPayloadSizeExceededException;
import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.msg.tools.TbRateLimitsException;
import org.thingsboard.server.dao.exception.EntitiesLimitExceededException;
import org.thingsboard.server.service.security.exception.AuthMethodNotSupportedException; import org.thingsboard.server.service.security.exception.AuthMethodNotSupportedException;
import org.thingsboard.server.service.security.exception.JwtExpiredTokenException; import org.thingsboard.server.service.security.exception.JwtExpiredTokenException;
import org.thingsboard.server.service.security.exception.UserPasswordExpiredException; import org.thingsboard.server.service.security.exception.UserPasswordExpiredException;

24
application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java

@ -22,17 +22,18 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.MoreExecutors;
import com.google.gson.JsonElement; import com.google.gson.JsonElement;
import com.google.gson.JsonParser;
import jakarta.annotation.PostConstruct; import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy; import jakarta.annotation.PreDestroy;
import lombok.Data; import lombok.Data;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.function.TriConsumer;
import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.DonAsynchron;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.AttributesSaveRequest;
import org.thingsboard.rule.engine.api.AttributesSaveRequest.Strategy; import org.thingsboard.rule.engine.api.AttributesSaveRequest.Strategy;
import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest;
import org.thingsboard.server.actors.calculatedField.MultipleTbCallback;
import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.common.adaptor.JsonConverter;
import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.AttributeScope;
@ -80,7 +81,6 @@ import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.function.Function; import java.util.function.Function;
@ -391,6 +391,24 @@ public abstract class AbstractCalculatedFieldProcessingService {
return new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, endTs, 0, limit, Aggregation.NONE); return new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, endTs, 0, limit, Aggregation.NONE);
} }
protected void handlePropagationResults(PropagationCalculatedFieldResult propagationResult, TbCallback callback,
TriConsumer<EntityId, TelemetryCalculatedFieldResult, TbCallback> telemetryResultHandler) {
List<EntityId> propagationEntityIds = propagationResult.getEntityIds();
if (propagationEntityIds.isEmpty()) {
callback.onSuccess();
return;
}
if (propagationEntityIds.size() == 1) {
EntityId propagationEntityId = propagationEntityIds.get(0);
telemetryResultHandler.accept(propagationEntityId, propagationResult.getResult(), callback);
return;
}
MultipleTbCallback multipleTbCallback = new MultipleTbCallback(propagationEntityIds.size(), callback);
for (var propagationEntityId : propagationEntityIds) {
telemetryResultHandler.accept(propagationEntityId, propagationResult.getResult(), multipleTbCallback);
}
}
protected void sendMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbCallback callback, TbMsg msg) { protected void sendMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbCallback callback, TbMsg msg) {
try { try {
clusterService.pushMsgToRuleEngine(tenantId, entityId, msg, new TbQueueCallback() { clusterService.pushMsgToRuleEngine(tenantId, entityId, msg, new TbQueueCallback() {
@ -413,7 +431,7 @@ public abstract class AbstractCalculatedFieldProcessingService {
protected void saveTelemetryResult(TenantId tenantId, EntityId entityId, String cfName, TelemetryCalculatedFieldResult cfResult, List<CalculatedFieldId> cfIds, TbCallback callback) { protected void saveTelemetryResult(TenantId tenantId, EntityId entityId, String cfName, TelemetryCalculatedFieldResult cfResult, List<CalculatedFieldId> cfIds, TbCallback callback) {
OutputType type = cfResult.getType(); OutputType type = cfResult.getType();
JsonElement jsonResult = JsonParser.parseString(Objects.requireNonNull(cfResult.stringValue())); JsonElement jsonResult = cfResult.toJsonElement();
log.trace("[{}][{}] Saving CF result: {}", tenantId, entityId, jsonResult); log.trace("[{}][{}] Saving CF result: {}", tenantId, entityId, jsonResult);
switch (type) { switch (type) {

8
application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java

@ -68,7 +68,7 @@ public abstract class AbstractCalculatedFieldStateService implements CalculatedF
protected abstract void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback); protected abstract void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback);
protected void processRestoredState(CalculatedFieldStateProto stateMsg, TopicPartitionInfo partition) { protected void processRestoredState(CalculatedFieldStateProto stateMsg, TopicPartitionInfo partition, TbCallback callback) {
var id = fromProto(stateMsg.getId()); var id = fromProto(stateMsg.getId());
if (partition == null) { if (partition == null) {
try { try {
@ -79,12 +79,12 @@ public abstract class AbstractCalculatedFieldStateService implements CalculatedF
} }
} }
var state = fromProto(id, stateMsg); var state = fromProto(id, stateMsg);
processRestoredState(id, state, partition); processRestoredState(id, state, partition, callback);
} }
protected void processRestoredState(CalculatedFieldEntityCtxId id, CalculatedFieldState state, TopicPartitionInfo partition) { protected void processRestoredState(CalculatedFieldEntityCtxId id, CalculatedFieldState state, TopicPartitionInfo partition, TbCallback callback) {
partition = partition.withTopic(DataConstants.CF_STATES_QUEUE_NAME); partition = partition.withTopic(DataConstants.CF_STATES_QUEUE_NAME);
actorSystemContext.tellWithHighPriority(new CalculatedFieldStateRestoreMsg(id, state, partition)); actorSystemContext.tellWithHighPriority(new CalculatedFieldStateRestoreMsg(id, state, partition, callback));
} }
@Override @Override

4
application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java

@ -27,9 +27,11 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntry;
import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional;
public interface CalculatedFieldProcessingService { public interface CalculatedFieldProcessingService {
@ -37,6 +39,8 @@ public interface CalculatedFieldProcessingService {
Map<String, ArgumentEntry> fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId); Map<String, ArgumentEntry> fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId);
Optional<PropagationArgumentEntry> fetchPropagationArgumentFromDb(CalculatedFieldCtx ctx, EntityId entityId);
List<EntityId> fetchRelatedEntities(CalculatedFieldCtx ctx, EntityId entityId); List<EntityId> fetchRelatedEntities(CalculatedFieldCtx ctx, EntityId entityId);
Map<String, ArgumentEntry> fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map<String, Argument> arguments); Map<String, ArgumentEntry> fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map<String, Argument> arguments);

7
application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java

@ -15,11 +15,14 @@
*/ */
package org.thingsboard.server.service.cf; package org.thingsboard.server.service.cf;
import com.google.gson.JsonElement;
import com.google.gson.JsonParser;
import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
import java.util.List; import java.util.List;
import java.util.Objects;
public interface CalculatedFieldResult { public interface CalculatedFieldResult {
@ -29,4 +32,8 @@ public interface CalculatedFieldResult {
boolean isEmpty(); boolean isEmpty();
default JsonElement toJsonElement() {
return JsonParser.parseString(Objects.requireNonNull(stringValue()));
}
} }

39
application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java

@ -17,13 +17,13 @@ package org.thingsboard.server.service.cf;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.function.TriConsumer;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg;
import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; import org.thingsboard.server.actors.calculatedField.MultipleTbCallback;
import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Argument;
import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric;
import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration;
@ -50,6 +50,7 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntry;
import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry;
import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService;
import java.util.ArrayList; import java.util.ArrayList;
@ -57,6 +58,7 @@ import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
@ -94,11 +96,18 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF
@Override @Override
public Map<String, ArgumentEntry> fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId) { public Map<String, ArgumentEntry> fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId) {
return switch (ctx.getCfType()) { return ctx.getCfType() == CalculatedFieldType.GEOFENCING ?
case GEOFENCING -> resolveArgumentFutures(fetchGeofencingCalculatedFieldArguments(ctx, entityId, true, System.currentTimeMillis())); resolveArgumentFutures(fetchGeofencingCalculatedFieldArguments(ctx, entityId, true, System.currentTimeMillis())) :
case PROPAGATION -> resolveArgumentFutures(Map.of(PROPAGATION_CONFIG_ARGUMENT, fetchPropagationCalculatedFieldArgument(ctx, entityId))); Collections.emptyMap();
default -> Collections.emptyMap(); }
};
@Override
public Optional<PropagationArgumentEntry> fetchPropagationArgumentFromDb(CalculatedFieldCtx ctx, EntityId entityId) {
if (ctx.getCfType() != CalculatedFieldType.PROPAGATION) {
return Optional.empty();
}
return Optional.of((PropagationArgumentEntry)
resolveArgumentValue(PROPAGATION_CONFIG_ARGUMENT, fetchPropagationCalculatedFieldArgument(ctx, entityId)));
} }
@Override @Override
@ -169,24 +178,6 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF
sendMsgToRuleEngine(tenantId, entityId, callback, result.toTbMsg(entityId, cfName, cfIds)); sendMsgToRuleEngine(tenantId, entityId, callback, result.toTbMsg(entityId, cfName, cfIds));
} }
private void handlePropagationResults(PropagationCalculatedFieldResult propagationResult, TbCallback callback,
TriConsumer<EntityId, TelemetryCalculatedFieldResult, TbCallback> telemetryResultHandler) {
List<EntityId> propagationEntityIds = propagationResult.getEntityIds();
if (propagationEntityIds.isEmpty()) {
callback.onSuccess();
return;
}
if (propagationEntityIds.size() == 1) {
EntityId propagationEntityId = propagationEntityIds.get(0);
telemetryResultHandler.accept(propagationEntityId, propagationResult.getResult(), callback);
return;
}
MultipleTbCallback multipleTbCallback = new MultipleTbCallback(propagationEntityIds.size(), callback);
for (var propagationEntityId : propagationEntityIds) {
telemetryResultHandler.accept(propagationEntityId, propagationResult.getResult(), multipleTbCallback);
}
}
@Override @Override
public void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List<CalculatedFieldEntityCtxId> linkedCalculatedFields, TbCallback callback) { public void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List<CalculatedFieldEntityCtxId> linkedCalculatedFields, TbCallback callback) {
Map<TopicPartitionInfo, List<CalculatedFieldEntityCtxId>> unicasts = new HashMap<>(); Map<TopicPartitionInfo, List<CalculatedFieldEntityCtxId>> unicasts = new HashMap<>();

10
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java

@ -63,7 +63,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState,
this.ctx = ctx; this.ctx = ctx;
this.actorCtx = actorCtx; this.actorCtx = actorCtx;
this.requiredArguments = ctx.getArgNames(); this.requiredArguments = ctx.getArgNames();
this.readinessStatus = checkReadiness(requiredArguments, arguments); this.readinessStatus = checkReadiness();
} }
@Override @Override
@ -108,7 +108,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState,
if (updatedArguments == null) { if (updatedArguments == null) {
return Collections.emptyMap(); return Collections.emptyMap();
} }
readinessStatus = checkReadiness(requiredArguments, arguments); readinessStatus = checkReadiness();
return updatedArguments; return updatedArguments;
} }
@ -183,13 +183,13 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState,
return latestTs; return latestTs;
} }
protected ReadinessStatus checkReadiness(List<String> requiredArguments, Map<String, ArgumentEntry> currentArguments) { protected ReadinessStatus checkReadiness() {
if (currentArguments == null) { if (arguments == null) {
return ReadinessStatus.from(requiredArguments); return ReadinessStatus.from(requiredArguments);
} }
List<String> emptyArguments = null; List<String> emptyArguments = null;
for (String requiredArgumentKey : requiredArguments) { for (String requiredArgumentKey : requiredArguments) {
ArgumentEntry argumentEntry = currentArguments.get(requiredArgumentKey); ArgumentEntry argumentEntry = arguments.get(requiredArgumentKey);
if (argumentEntry == null || argumentEntry.isEmpty()) { if (argumentEntry == null || argumentEntry.isEmpty()) {
if (emptyArguments == null) { if (emptyArguments == null) {
emptyArguments = new ArrayList<>(); emptyArguments = new ArrayList<>();

51
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java

@ -63,8 +63,7 @@ import org.thingsboard.server.dao.util.TimeUtils;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto;
import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService;
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.ScheduledRefreshSupported;
import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState;
import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService;
import java.io.Closeable; import java.io.Closeable;
@ -122,14 +121,14 @@ public class CalculatedFieldCtx implements Closeable {
private long maxSingleValueArgumentSize; private long maxSingleValueArgumentSize;
private long intermediateAggregationIntervalMillis; private long intermediateAggregationIntervalMillis;
private boolean relationQueryDynamicArguments; private boolean cfHasRelationPathQuerySource;
private List<String> mainEntityGeofencingArgumentNames; private List<String> mainEntityGeofencingArgumentNames;
private List<String> linkedEntityAndCurrentOwnerGeofencingArgumentNames; private List<String> linkedEntityAndCurrentOwnerGeofencingArgumentNames;
private List<String> relatedEntityArgumentNames; private List<String> relatedEntityArgumentNames;
private long scheduledUpdateIntervalMillis; private long scheduledUpdateIntervalMillis;
private long cfCheckReevaluationInterval; private long cfCheckReevaluationIntervalMillis;
private long alarmReevaluationInterval; private long alarmReevaluationIntervalMillis;
private Argument propagationArgument; private Argument propagationArgument;
private boolean applyExpressionForResolvedArguments; private boolean applyExpressionForResolvedArguments;
@ -161,10 +160,11 @@ public class CalculatedFieldCtx implements Closeable {
if (refId == null) { if (refId == null) {
if (CalculatedFieldType.RELATED_ENTITIES_AGGREGATION.equals(cfType)) { if (CalculatedFieldType.RELATED_ENTITIES_AGGREGATION.equals(cfType)) {
relatedEntityArguments.compute(refKey, (key, existingNames) -> CollectionsUtil.addToSet(existingNames, entry.getKey())); relatedEntityArguments.compute(refKey, (key, existingNames) -> CollectionsUtil.addToSet(existingNames, entry.getKey()));
cfHasRelationPathQuerySource = true;
continue; continue;
} }
if (entry.getValue().hasRelationQuerySource()) { if (entry.getValue().hasRelationQuerySource()) {
relationQueryDynamicArguments = true; cfHasRelationPathQuerySource = true;
continue; continue;
} }
if (entry.getValue().hasOwnerSource()) { if (entry.getValue().hasOwnerSource()) {
@ -201,7 +201,7 @@ public class CalculatedFieldCtx implements Closeable {
if (calculatedField.getConfiguration() instanceof PropagationCalculatedFieldConfiguration propagationConfig) { if (calculatedField.getConfiguration() instanceof PropagationCalculatedFieldConfiguration propagationConfig) {
propagationArgument = propagationConfig.toPropagationArgument(); propagationArgument = propagationConfig.toPropagationArgument();
applyExpressionForResolvedArguments = propagationConfig.isApplyExpressionToResolvedArguments(); applyExpressionForResolvedArguments = propagationConfig.isApplyExpressionToResolvedArguments();
relationQueryDynamicArguments = true; cfHasRelationPathQuerySource = true;
} }
} }
if (calculatedField.getConfiguration() instanceof ScheduledUpdateSupportedCalculatedFieldConfiguration scheduledConfig) { if (calculatedField.getConfiguration() instanceof ScheduledUpdateSupportedCalculatedFieldConfiguration scheduledConfig) {
@ -246,8 +246,7 @@ public class CalculatedFieldCtx implements Closeable {
boolean requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); boolean requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation();
if (calculatedField.getConfiguration() instanceof AlarmCalculatedFieldConfiguration) { if (calculatedField.getConfiguration() instanceof AlarmCalculatedFieldConfiguration) {
if (requiresScheduledReevaluation) { if (requiresScheduledReevaluation) {
long reevaluationIntervalMillis = TimeUnit.SECONDS.toMillis(alarmReevaluationInterval); if (now - lastReevaluationTs >= alarmReevaluationIntervalMillis) {
if (now - lastReevaluationTs >= reevaluationIntervalMillis) {
lastReevaluationTs = now; lastReevaluationTs = now;
return true; return true;
} }
@ -306,8 +305,8 @@ public class CalculatedFieldCtx implements Closeable {
this.maxStateSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes) * 1024; this.maxStateSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes) * 1024;
this.maxSingleValueArgumentSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxSingleValueArgumentSizeInKBytes) * 1024; this.maxSingleValueArgumentSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxSingleValueArgumentSizeInKBytes) * 1024;
this.intermediateAggregationIntervalMillis = TimeUnit.SECONDS.toMillis(apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getIntermediateAggregationIntervalInSecForCF)); this.intermediateAggregationIntervalMillis = TimeUnit.SECONDS.toMillis(apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getIntermediateAggregationIntervalInSecForCF));
this.cfCheckReevaluationInterval = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getCfReevaluationCheckInterval); this.cfCheckReevaluationIntervalMillis = TimeUnit.SECONDS.toMillis(apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getCfReevaluationCheckInterval));
this.alarmReevaluationInterval = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getAlarmsReevaluationInterval); this.alarmReevaluationIntervalMillis = TimeUnit.SECONDS.toMillis(apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getAlarmsReevaluationInterval));
} }
public double evaluateSimpleExpression(Expression expression, CalculatedFieldState state) { public double evaluateSimpleExpression(Expression expression, CalculatedFieldState state) {
@ -757,38 +756,20 @@ public class CalculatedFieldCtx implements Closeable {
return scheduledUpdateIntervalMillis == DISABLED_INTERVAL_VALUE; return scheduledUpdateIntervalMillis == DISABLED_INTERVAL_VALUE;
} }
public boolean shouldFetchRelationQueryDynamicArgumentsFromDb(CalculatedFieldState state) { public boolean shouldFetchRelatedEntities(CalculatedFieldState state) {
if (!relationQueryDynamicArguments) { if (!cfHasRelationPathQuerySource) {
return false; return false;
} }
return switch (cfType) { if (isScheduledUpdateDisabled()) {
case PROPAGATION -> true;
case GEOFENCING -> {
if (isScheduledUpdateDisabled()) {
yield false;
}
var geofencingState = (GeofencingCalculatedFieldState) state;
if (geofencingState.getLastDynamicArgumentsRefreshTs() == DEFAULT_LAST_UPDATE_TS) {
yield true;
}
yield geofencingState.getLastDynamicArgumentsRefreshTs() <
System.currentTimeMillis() - scheduledUpdateIntervalMillis;
}
default -> false;
};
}
public boolean shouldFetchEntityRelations(CalculatedFieldState state) {
if (!(state instanceof RelatedEntitiesAggregationCalculatedFieldState relatedEntitiesAggState)) {
return false; return false;
} }
if (isScheduledUpdateDisabled()) { if (!(state instanceof ScheduledRefreshSupported scheduledRefreshSupported)) {
return false; return false;
} }
if (relatedEntitiesAggState.getLastRelatedEntitiesRefreshTs() == DEFAULT_LAST_UPDATE_TS) { if (scheduledRefreshSupported.getLastScheduledRefreshTs() == DEFAULT_LAST_UPDATE_TS) {
return true; return true;
} }
return relatedEntitiesAggState.getLastRelatedEntitiesRefreshTs() < System.currentTimeMillis() - scheduledUpdateIntervalMillis; return scheduledRefreshSupported.getLastScheduledRefreshTs() < System.currentTimeMillis() - scheduledUpdateIntervalMillis;
} }
@Override @Override

36
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java

@ -43,6 +43,8 @@ import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory;
import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService; import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService;
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.bytesToString; import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.bytesToString;
@ -61,6 +63,8 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta
@Value("${queue.calculated_fields.poll_interval:25}") @Value("${queue.calculated_fields.poll_interval:25}")
private long pollInterval; private long pollInterval;
@Value("${queue.calculated_fields.pack_processing_timeout:60000}")
private long packProcessingTimeout;
private TbKafkaProducerTemplate<TbProtoQueueMsg<CalculatedFieldStateProto>> stateProducer; private TbKafkaProducerTemplate<TbProtoQueueMsg<CalculatedFieldStateProto>> stateProducer;
@ -74,21 +78,39 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta
.topic(partitionService.getTopic(queueKey)) .topic(partitionService.getTopic(queueKey))
.pollInterval(pollInterval) .pollInterval(pollInterval)
.msgPackProcessor((msgs, consumer, consumerKey, config) -> { .msgPackProcessor((msgs, consumer, consumerKey, config) -> {
CountDownLatch completionLatch = new CountDownLatch(msgs.size());
for (TbProtoQueueMsg<CalculatedFieldStateProto> msg : msgs) { for (TbProtoQueueMsg<CalculatedFieldStateProto> msg : msgs) {
TbCallback callback = new TbCallback() {
@Override
public void onSuccess() {
int processedMsgCount = counter.incrementAndGet();
if (processedMsgCount % 10000 == 0) {
log.info("Processed {} CF state messages", processedMsgCount);
}
completionLatch.countDown();
}
@Override
public void onFailure(Throwable t) {
log.error("Failed to process CF state message: {}", msg, t);
completionLatch.countDown();
}
};
try { try {
if (msg.getValue() != null) { if (msg.getValue() != null) {
processRestoredState(msg.getValue(), consumerKey.partition()); processRestoredState(msg.getValue(), consumerKey.partition(), callback);
} else { } else {
processRestoredState(getStateId(msg.getHeaders()), null, consumerKey.partition()); processRestoredState(getStateId(msg.getHeaders()), null, consumerKey.partition(), callback);
} }
} catch (Throwable t) { } catch (Throwable t) {
log.error("Failed to process state message: {}", msg, t); callback.onFailure(t);
} }
}
int processedMsgCount = counter.incrementAndGet(); boolean success = completionLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS);
if (processedMsgCount % 10000 == 0) { if (!success) {
log.info("Processed {} calculated field state msgs", processedMsgCount); log.error("Timeout to process CF state messages pack of size {}", msgs.size());
}
} }
}) })
.consumerCreator((queueConfig, tpi) -> queueFactory.createCalculatedFieldStateConsumer()) .consumerCreator((queueConfig, tpi) -> queueFactory.createCalculatedFieldStateConsumer())

15
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java

@ -62,11 +62,22 @@ public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldS
public void restore(QueueKey queueKey, Set<TopicPartitionInfo> partitions) { public void restore(QueueKey queueKey, Set<TopicPartitionInfo> partitions) {
if (stateService.getPartitions().isEmpty()) { if (stateService.getPartitions().isEmpty()) {
cfRocksDb.forEach((key, value) -> { cfRocksDb.forEach((key, value) -> {
CalculatedFieldStateProto stateMsg;
try { try {
processRestoredState(CalculatedFieldStateProto.parseFrom(value), null); stateMsg = CalculatedFieldStateProto.parseFrom(value);
} catch (Exception e) { } catch (Exception e) {
log.error("[{}] Failed to process restored state", key, e); log.error("Failed to parse CalculatedFieldStateProto for key {}", key, e);
return;
} }
processRestoredState(stateMsg, null, new TbCallback() {
@Override
public void onSuccess() {}
@Override
public void onFailure(Throwable t) {
log.error("Failed to process CF state message: {}", stateMsg, t);
}
});
}); });
} }
super.restore(queueKey, partitions); super.restore(queueKey, partitions);

24
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java

@ -41,6 +41,7 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType;
import org.thingsboard.server.service.cf.ctx.state.BaseCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.BaseCalculatedFieldState;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import org.thingsboard.server.service.cf.ctx.state.aggregation.function.AggEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.function.AggEntry;
import org.thingsboard.server.service.cf.ctx.state.geofencing.ScheduledRefreshSupported;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
@ -55,14 +56,14 @@ import static org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx.DIS
import static org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState.ReadinessStatus.MISSING_AGGREGATION_ENTITIES_ERROR; import static org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState.ReadinessStatus.MISSING_AGGREGATION_ENTITIES_ERROR;
@Slf4j @Slf4j
@Getter public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculatedFieldState implements ScheduledRefreshSupported {
public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculatedFieldState {
@Setter @Setter
@Getter
private long lastArgsRefreshTs = DEFAULT_LAST_UPDATE_TS; private long lastArgsRefreshTs = DEFAULT_LAST_UPDATE_TS;
@Setter @Setter
@Getter
private long lastMetricsEvalTs = DEFAULT_LAST_UPDATE_TS; private long lastMetricsEvalTs = DEFAULT_LAST_UPDATE_TS;
@Setter
private long lastRelatedEntitiesRefreshTs = DEFAULT_LAST_UPDATE_TS; private long lastRelatedEntitiesRefreshTs = DEFAULT_LAST_UPDATE_TS;
private long deduplicationIntervalMs = DISABLED_INTERVAL_VALUE; private long deduplicationIntervalMs = DISABLED_INTERVAL_VALUE;
private Map<String, AggMetric> metrics; private Map<String, AggMetric> metrics;
@ -104,13 +105,24 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat
@Override @Override
public void reset() { // must reset everything dependent on arguments public void reset() { // must reset everything dependent on arguments
super.reset(); super.reset();
resetScheduledRefreshTs();
lastArgsRefreshTs = DEFAULT_LAST_UPDATE_TS; lastArgsRefreshTs = DEFAULT_LAST_UPDATE_TS;
lastMetricsEvalTs = DEFAULT_LAST_UPDATE_TS; lastMetricsEvalTs = DEFAULT_LAST_UPDATE_TS;
lastRelatedEntitiesRefreshTs = DEFAULT_LAST_UPDATE_TS;
metrics = null; metrics = null;
} }
public void updateLastRelatedEntitiesRefreshTs() { @Override
public void resetScheduledRefreshTs() {
lastRelatedEntitiesRefreshTs = DEFAULT_LAST_UPDATE_TS;
}
@Override
public long getLastScheduledRefreshTs() {
return lastRelatedEntitiesRefreshTs;
}
@Override
public void updateScheduledRefreshTs() {
lastRelatedEntitiesRefreshTs = System.currentTimeMillis(); lastRelatedEntitiesRefreshTs = System.currentTimeMillis();
} }
@ -128,7 +140,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat
public List<EntityId> checkRelatedEntities(List<EntityId> relatedEntities) { public List<EntityId> checkRelatedEntities(List<EntityId> relatedEntities) {
Map<EntityId, Map<String, ArgumentEntry>> entityInputs = prepareInputs(); Map<EntityId, Map<String, ArgumentEntry>> entityInputs = prepareInputs();
findOutdatedEntities(entityInputs, relatedEntities).forEach(this::cleanupEntityData); findOutdatedEntities(entityInputs, relatedEntities).forEach(this::cleanupEntityData);
updateLastRelatedEntitiesRefreshTs(); updateScheduledRefreshTs();
return findMissingEntities(entityInputs, relatedEntities); return findMissingEntities(entityInputs, relatedEntities);
} }

4
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java

@ -205,7 +205,7 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt
handleExpiredInterval(intervalEntry, args, results); handleExpiredInterval(intervalEntry, args, results);
expiredIntervals.add(intervalEntry); expiredIntervals.add(intervalEntry);
} else if (now - startTs >= intervalEntry.getIntervalDuration()) { } else if (now - startTs >= intervalEntry.getIntervalDuration()) {
handleActiveInterval(ctx.getCfCheckReevaluationInterval(), intervalEntry, args, results); handleActiveInterval(ctx.getCfCheckReevaluationIntervalMillis(), intervalEntry, args, results);
if (watermarkDuration == 0) { if (watermarkDuration == 0) {
expiredIntervals.add(intervalEntry); expiredIntervals.add(intervalEntry);
} }
@ -288,7 +288,7 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt
} }
if (!metricsNode.isEmpty()) { if (!metricsNode.isEmpty()) {
ObjectNode resultNode = JacksonUtil.newObjectNode(); ObjectNode resultNode = JacksonUtil.newObjectNode();
resultNode.put("ts", interval.getEndTs() - 1); resultNode.put("ts", interval.getStartTs());
resultNode.set("values", metricsNode); resultNode.set("values", metricsNode);
result.add(resultNode); result.add(resultNode);

2
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java

@ -94,8 +94,6 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState {
private Alarm currentAlarm; private Alarm currentAlarm;
private boolean initialFetchDone; private boolean initialFetchDone;
// TODO: deprecate device profile node, describe the differences and improvements
public AlarmCalculatedFieldState(EntityId entityId) { public AlarmCalculatedFieldState(EntityId entityId) {
super(entityId); super(entityId);
} }

19
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java

@ -21,8 +21,6 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.MoreExecutors;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.geo.Coordinates; import org.thingsboard.common.util.geo.Coordinates;
@ -52,11 +50,9 @@ import static org.thingsboard.server.common.data.cf.configuration.geofencing.Ent
import static org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingPresenceStatus.INSIDE; import static org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingPresenceStatus.INSIDE;
import static org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingPresenceStatus.OUTSIDE; import static org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingPresenceStatus.OUTSIDE;
@Getter
@Setter
@Slf4j @Slf4j
@EqualsAndHashCode(callSuper = true) @EqualsAndHashCode(callSuper = true)
public class GeofencingCalculatedFieldState extends BaseCalculatedFieldState { public class GeofencingCalculatedFieldState extends BaseCalculatedFieldState implements ScheduledRefreshSupported {
private long lastDynamicArgumentsRefreshTs = DEFAULT_LAST_UPDATE_TS; private long lastDynamicArgumentsRefreshTs = DEFAULT_LAST_UPDATE_TS;
@ -147,10 +143,21 @@ public class GeofencingCalculatedFieldState extends BaseCalculatedFieldState {
@Override @Override
public void reset() { public void reset() {
super.reset(); super.reset();
resetScheduledRefreshTs();
}
@Override
public void resetScheduledRefreshTs() {
lastDynamicArgumentsRefreshTs = DEFAULT_LAST_UPDATE_TS; lastDynamicArgumentsRefreshTs = DEFAULT_LAST_UPDATE_TS;
} }
public void updateLastDynamicArgumentsRefreshTs() { @Override
public long getLastScheduledRefreshTs() {
return lastDynamicArgumentsRefreshTs;
}
@Override
public void updateScheduledRefreshTs() {
lastDynamicArgumentsRefreshTs = System.currentTimeMillis(); lastDynamicArgumentsRefreshTs = System.currentTimeMillis();
} }

21
ui-ngx/src/app/modules/login/pages/login/reset-password-request.component.scss → application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/ScheduledRefreshSupported.java

@ -13,17 +13,14 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
@import '../../../../../scss/constants'; package org.thingsboard.server.service.cf.ctx.state.geofencing;
public interface ScheduledRefreshSupported {
void resetScheduledRefreshTs();
long getLastScheduledRefreshTs();
void updateScheduledRefreshTs();
:host {
display: flex;
flex: 1 1 0;
.tb-request-password-reset-content {
background-color: #eee;
.tb-request-password-reset-card {
@media #{$mat-gt-xs} {
width: 450px !important;
}
}
}
} }

46
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java

@ -22,6 +22,8 @@ import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry;
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
@ -30,10 +32,11 @@ import java.util.Set;
public class PropagationArgumentEntry implements ArgumentEntry { public class PropagationArgumentEntry implements ArgumentEntry {
private Set<EntityId> entityIds; private Set<EntityId> entityIds;
private transient EntityId added; private transient List<EntityId> added;
private transient EntityId removed; private transient EntityId removed;
private boolean forceResetPrevious; private transient boolean forceResetPrevious;
private transient boolean ignoreRemovedEntities;
public PropagationArgumentEntry() { public PropagationArgumentEntry() {
this.entityIds = new HashSet<>(); this.entityIds = new HashSet<>();
@ -57,27 +60,44 @@ public class PropagationArgumentEntry implements ArgumentEntry {
@Override @Override
public boolean updateEntry(ArgumentEntry entry) { public boolean updateEntry(ArgumentEntry entry) {
if (!(entry instanceof PropagationArgumentEntry propagationArgumentEntry)) { if (!(entry instanceof PropagationArgumentEntry updated)) {
throw new IllegalArgumentException("Unsupported argument entry type for propagation argument entry: " + entry.getType()); throw new IllegalArgumentException("Unsupported argument entry type for propagation argument entry: " + entry.getType());
} }
if (propagationArgumentEntry.getAdded() != null) { if (updated.getAdded() != null) {
boolean updated = entityIds.add(propagationArgumentEntry.getAdded()); return checkAdded(updated.getAdded());
if (updated) { }
added = propagationArgumentEntry.getAdded(); if (updated.getRemoved() != null) {
} return entityIds.remove(updated.getRemoved());
return updated;
} }
if (propagationArgumentEntry.getRemoved() != null) { if (updated.isIgnoreRemovedEntities()) {
return entityIds.remove(propagationArgumentEntry.getRemoved()); Set<EntityId> updatedIds = updated.getEntityIds();
if (updatedIds.isEmpty()) {
entityIds.clear();
return false;
}
entityIds.retainAll(updatedIds);
return checkAdded(updatedIds);
} }
if (propagationArgumentEntry.isEmpty()) { if (updated.isEmpty()) {
entityIds.clear(); entityIds.clear();
return true; return true;
} }
entityIds = propagationArgumentEntry.getEntityIds(); entityIds = updated.getEntityIds();
return true; return true;
} }
private boolean checkAdded(Collection<EntityId> updatedIds) {
for (EntityId id : updatedIds) {
if (entityIds.add(id)) {
if (added == null) {
added = new ArrayList<>();
}
added.add(id);
}
}
return added != null && !added.isEmpty();
}
@Override @Override
public boolean isEmpty() { public boolean isEmpty() {
return entityIds.isEmpty(); return entityIds.isEmpty();

67
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java

@ -25,6 +25,8 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.Output;
import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.OutputType;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.util.CollectionsUtil;
import org.thingsboard.server.service.cf.CalculatedFieldProcessingService;
import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.CalculatedFieldResult;
import org.thingsboard.server.service.cf.PropagationCalculatedFieldResult; import org.thingsboard.server.service.cf.PropagationCalculatedFieldResult;
import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult;
@ -41,6 +43,8 @@ import static org.thingsboard.server.common.data.cf.configuration.PropagationCal
public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState { public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState {
private CalculatedFieldProcessingService cfProcessingService;
public PropagationCalculatedFieldState(EntityId entityId) { public PropagationCalculatedFieldState(EntityId entityId) {
super(entityId); super(entityId);
} }
@ -49,14 +53,30 @@ public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState
public void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx) { public void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx) {
this.ctx = ctx; this.ctx = ctx;
this.actorCtx = actorCtx; this.actorCtx = actorCtx;
this.cfProcessingService = ctx.getCfProcessingService();
this.requiredArguments = new ArrayList<>(ctx.getArgNames()); this.requiredArguments = new ArrayList<>(ctx.getArgNames());
requiredArguments.add(PROPAGATION_CONFIG_ARGUMENT); requiredArguments.add(PROPAGATION_CONFIG_ARGUMENT);
this.readinessStatus = checkReadiness(requiredArguments, arguments); this.readinessStatus = checkReadiness();
if (ctx.isApplyExpressionForResolvedArguments()) { if (ctx.isApplyExpressionForResolvedArguments()) {
this.tbelExpression = ctx.getTbelExpressions().get(ctx.getExpression()); this.tbelExpression = ctx.getTbelExpressions().get(ctx.getExpression());
} }
} }
@Override
public void init(boolean restored) {
super.init(restored);
if (restored) {
cfProcessingService.fetchPropagationArgumentFromDb(ctx, entityId).ifPresent(fromDb -> {
fromDb.setIgnoreRemovedEntities(true);
var updatedArgs = update(Map.of(PROPAGATION_CONFIG_ARGUMENT, fromDb), ctx);
if (updatedArgs.isEmpty()) {
return;
}
ctx.scheduleReevaluation(0L, actorCtx);
});
}
}
@Override @Override
public CalculatedFieldType getType() { public CalculatedFieldType getType() {
return CalculatedFieldType.PROPAGATION; return CalculatedFieldType.PROPAGATION;
@ -68,9 +88,10 @@ public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState
if (!(argumentEntry instanceof PropagationArgumentEntry propagationArgumentEntry)) { if (!(argumentEntry instanceof PropagationArgumentEntry propagationArgumentEntry)) {
return Futures.immediateFuture(PropagationCalculatedFieldResult.builder().build()); return Futures.immediateFuture(PropagationCalculatedFieldResult.builder().build());
} }
boolean newEntityAdded = propagationArgumentEntry.getAdded() != null;
List<EntityId> entityIds; List<EntityId> entityIds;
if (propagationArgumentEntry.getAdded() != null) { if (newEntityAdded) {
entityIds = List.of(propagationArgumentEntry.getAdded()); entityIds = propagationArgumentEntry.getAdded();
propagationArgumentEntry.setAdded(null); propagationArgumentEntry.setAdded(null);
} else { } else {
if (propagationArgumentEntry.getEntityIds().isEmpty()) { if (propagationArgumentEntry.getEntityIds().isEmpty()) {
@ -86,13 +107,43 @@ public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState
.build(), .build(),
MoreExecutors.directExecutor()); MoreExecutors.directExecutor());
} }
if (newEntityAdded || CollectionsUtil.isEmpty(updatedArgs)) {
updatedArgs = arguments;
}
return Futures.immediateFuture(PropagationCalculatedFieldResult.builder() return Futures.immediateFuture(PropagationCalculatedFieldResult.builder()
.entityIds(entityIds) .entityIds(entityIds)
.result(toTelemetryResult(ctx)) .result(toTelemetryResult(ctx, updatedArgs))
.build()); .build());
} }
private TelemetryCalculatedFieldResult toTelemetryResult(CalculatedFieldCtx ctx) { @Override
protected ReadinessStatus checkReadiness() {
if (ctx.isApplyExpressionForResolvedArguments() || arguments == null) {
return super.checkReadiness();
}
boolean propagationNotEmpty = false;
boolean hasOtherNonEmpty = false;
List<String> emptyArguments = null;
for (String requiredArgumentKey : requiredArguments) {
ArgumentEntry argumentEntry = arguments.get(requiredArgumentKey);
if (argumentEntry == null || argumentEntry.isEmpty()) {
if (emptyArguments == null) {
emptyArguments = new ArrayList<>();
}
emptyArguments.add(requiredArgumentKey);
} else if (PROPAGATION_CONFIG_ARGUMENT.equals(requiredArgumentKey)) {
propagationNotEmpty = true;
} else {
hasOtherNonEmpty = true;
}
}
if (propagationNotEmpty && hasOtherNonEmpty) {
return ReadinessStatus.READY;
}
return ReadinessStatus.from(emptyArguments);
}
private TelemetryCalculatedFieldResult toTelemetryResult(CalculatedFieldCtx ctx, Map<String, ArgumentEntry> updatedArgs) {
Output output = ctx.getOutput(); Output output = ctx.getOutput();
TelemetryCalculatedFieldResult.TelemetryCalculatedFieldResultBuilder telemetryCfBuilder = TelemetryCalculatedFieldResult.TelemetryCalculatedFieldResultBuilder telemetryCfBuilder =
TelemetryCalculatedFieldResult.builder() TelemetryCalculatedFieldResult.builder()
@ -100,12 +151,14 @@ public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState
.type(output.getType()) .type(output.getType())
.scope(output.getScope()); .scope(output.getScope());
ObjectNode valuesNode = JacksonUtil.newObjectNode(); ObjectNode valuesNode = JacksonUtil.newObjectNode();
arguments.forEach((outputKey, argumentEntry) -> { updatedArgs.forEach((outputKey, argumentEntry) -> {
if (argumentEntry instanceof PropagationArgumentEntry) { if (argumentEntry instanceof PropagationArgumentEntry) {
return; return;
} }
if (argumentEntry instanceof SingleValueArgumentEntry singleArgumentEntry) { if (argumentEntry instanceof SingleValueArgumentEntry singleArgumentEntry) {
JacksonUtil.addKvEntry(valuesNode, singleArgumentEntry.getKvEntryValue(), outputKey); if (!singleArgumentEntry.isEmpty()) {
JacksonUtil.addKvEntry(valuesNode, singleArgumentEntry.getKvEntryValue(), outputKey);
}
return; return;
} }
throw new IllegalArgumentException("Unsupported argument type: " + argumentEntry.getType() + " detected for argument: " + outputKey + ". " + throw new IllegalArgumentException("Unsupported argument type: " + argumentEntry.getType() + " detected for argument: " + outputKey + ". " +

3
application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java

@ -258,8 +258,7 @@ public class DeviceBulkImportService extends AbstractBulkImportService<Device> {
Lwm2mDeviceProfileTransportConfiguration transportConfiguration = new Lwm2mDeviceProfileTransportConfiguration(); Lwm2mDeviceProfileTransportConfiguration transportConfiguration = new Lwm2mDeviceProfileTransportConfiguration();
transportConfiguration.setBootstrap(Collections.emptyList()); transportConfiguration.setBootstrap(Collections.emptyList());
transportConfiguration.setClientLwM2mSettings(new OtherConfiguration(false,1, 1, 1, PowerMode.DRX, null, null, null, null, null, V1_0.toString())); transportConfiguration.setClientLwM2mSettings(new OtherConfiguration());
transportConfiguration.setObserveAttr(new TelemetryMappingConfiguration(Collections.emptyMap(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptyMap(), false, SINGLE));
DeviceProfileData deviceProfileData = new DeviceProfileData(); DeviceProfileData deviceProfileData = new DeviceProfileData();
DefaultDeviceProfileConfiguration configuration = new DefaultDeviceProfileConfiguration(); DefaultDeviceProfileConfiguration configuration = new DefaultDeviceProfileConfiguration();

1
application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java

@ -206,7 +206,6 @@ public class EdgeContextComponent {
@Autowired @Autowired
private Optional<EdgeStatsCounterService> statsCounterService; private Optional<EdgeStatsCounterService> statsCounterService;
// processors // processors
@Autowired @Autowired
private AlarmProcessor alarmProcessor; private AlarmProcessor alarmProcessor;

12
application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java

@ -306,7 +306,8 @@ public abstract class EdgeGrpcSession implements Closeable {
if (isConnected() && !pageData.getData().isEmpty()) { if (isConnected() && !pageData.getData().isEmpty()) {
if (fetcher instanceof GeneralEdgeEventFetcher) { if (fetcher instanceof GeneralEdgeEventFetcher) {
long queueSize = pageData.getTotalElements() - ((long) pageLink.getPageSize() * pageLink.getPage()); long queueSize = pageData.getTotalElements() - ((long) pageLink.getPageSize() * pageLink.getPage());
ctx.getStatsCounterService().ifPresent(statsCounterService -> statsCounterService.setDownlinkMsgsLag(edge.getTenantId(), edge.getId(), queueSize)); ctx.getStatsCounterService().ifPresent(statsCounterService ->
statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_LAG, tenantId, edge.getId(), queueSize));
} }
log.trace("[{}][{}][{}] event(s) are going to be processed.", tenantId, edge.getId(), pageData.getData().size()); log.trace("[{}][{}][{}] event(s) are going to be processed.", tenantId, edge.getId(), pageData.getData().size());
List<DownlinkMsg> downlinkMsgsPack = convertToDownlinkMsgsPack(pageData.getData()); List<DownlinkMsg> downlinkMsgsPack = convertToDownlinkMsgsPack(pageData.getData());
@ -504,7 +505,8 @@ public abstract class EdgeGrpcSession implements Closeable {
ctx.getRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId()) ctx.getRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId())
.customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg) .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg)
.error("Failed to deliver messages after " + MAX_DOWNLINK_ATTEMPTS + " attempts").build()); .error("Failed to deliver messages after " + MAX_DOWNLINK_ATTEMPTS + " attempts").build());
ctx.getStatsCounterService().ifPresent(statsCounterService -> statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_PERMANENTLY_FAILED, edge.getTenantId(), edge.getId(), copy.size())); ctx.getStatsCounterService().ifPresent(statsCounterService ->
statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_PERMANENTLY_FAILED, edge.getTenantId(), edge.getId(), copy.size()));
stopCurrentSendDownlinkMsgsTask(false); stopCurrentSendDownlinkMsgsTask(false);
} }
} else { } else {
@ -544,7 +546,8 @@ public abstract class EdgeGrpcSession implements Closeable {
try { try {
if (msg.getSuccess()) { if (msg.getSuccess()) {
sessionState.getPendingMsgsMap().remove(msg.getDownlinkMsgId()); sessionState.getPendingMsgsMap().remove(msg.getDownlinkMsgId());
ctx.getStatsCounterService().ifPresent(statsCounterService -> statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_PUSHED, edge.getTenantId(), edge.getId(), 1)); ctx.getStatsCounterService().ifPresent(statsCounterService ->
statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_PUSHED, edge.getTenantId(), edge.getId(), 1));
log.debug("[{}][{}][{}] Msg has been processed successfully! Msg Id: [{}], Msg: {}", tenantId, edge.getId(), sessionId, msg.getDownlinkMsgId(), msg); log.debug("[{}][{}][{}] Msg has been processed successfully! Msg Id: [{}], Msg: {}", tenantId, edge.getId(), sessionId, msg.getDownlinkMsgId(), msg);
} else { } else {
log.debug("[{}][{}][{}] Msg processing failed! Msg Id: [{}], Error msg: {}", tenantId, edge.getId(), sessionId, msg.getDownlinkMsgId(), msg.getErrorMsg()); log.debug("[{}][{}][{}] Msg processing failed! Msg Id: [{}], Error msg: {}", tenantId, edge.getId(), sessionId, msg.getDownlinkMsgId(), msg.getErrorMsg());
@ -813,7 +816,8 @@ public abstract class EdgeGrpcSession implements Closeable {
} }
} }
highPriorityQueue.add(edgeEvent); highPriorityQueue.add(edgeEvent);
ctx.getStatsCounterService().ifPresent(statsCounterService -> statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edge.getTenantId(), edgeEvent.getEdgeId(), 1)); ctx.getStatsCounterService().ifPresent(statsCounterService ->
statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edge.getTenantId(), edgeEvent.getEdgeId(), 1));
} }
protected ListenableFuture<List<Void>> processUplinkMsg(UplinkMsg uplinkMsg) { protected ListenableFuture<List<Void>> processUplinkMsg(UplinkMsg uplinkMsg) {

3
application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java

@ -52,7 +52,8 @@ public class KafkaEdgeEventService extends BaseEdgeEventService {
TopicPartitionInfo tpi = topicService.getEdgeEventNotificationsTopic(edgeEvent.getTenantId(), edgeEvent.getEdgeId()); TopicPartitionInfo tpi = topicService.getEdgeEventNotificationsTopic(edgeEvent.getTenantId(), edgeEvent.getEdgeId());
ToEdgeEventNotificationMsg msg = ToEdgeEventNotificationMsg.newBuilder().setEdgeEventMsg(ProtoUtils.toProto(edgeEvent)).build(); ToEdgeEventNotificationMsg msg = ToEdgeEventNotificationMsg.newBuilder().setEdgeEventMsg(ProtoUtils.toProto(edgeEvent)).build();
producerProvider.getTbEdgeEventsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); producerProvider.getTbEdgeEventsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null);
statsCounterService.ifPresent(statsCounterService -> statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edgeEvent.getTenantId(), edgeEvent.getEdgeId(), 1)); statsCounterService.ifPresent(statsCounterService ->
statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edgeEvent.getTenantId(), edgeEvent.getEdgeId(), 1));
return Futures.immediateFuture(null); return Futures.immediateFuture(null);
} }

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/ai/AiModelEdgeProcessor.java

@ -29,7 +29,7 @@ import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.data.id.AiModelId; import org.thingsboard.server.common.data.id.AiModelId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.AiModelUpdateMsg; import org.thingsboard.server.gen.edge.v1.AiModelUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.EdgeVersion;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java

@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.EdgeVersion;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessor.java

@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.EdgeVersion;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/cf/CalculatedFieldEdgeProcessor.java

@ -36,7 +36,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.CalculatedFieldUpdateMsg; import org.thingsboard.server.gen.edge.v1.CalculatedFieldUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.EdgeVersion;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java

@ -29,7 +29,7 @@ import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DashboardId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.EdgeVersion;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java

@ -44,7 +44,7 @@ import org.thingsboard.server.common.msg.TbMsgDataType;
import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponseActorMsg; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponseActorMsg;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg; import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DeviceRpcCallMsg; import org.thingsboard.server.gen.edge.v1.DeviceRpcCallMsg;
import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg; import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessor.java

@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.EdgeVersion;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java

@ -32,7 +32,7 @@ import org.thingsboard.server.common.data.id.EntityViewId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.EdgeVersion;
import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java

@ -27,7 +27,7 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.data.id.TbResourceId; import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.EdgeVersion;
import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java

@ -31,7 +31,7 @@ import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.EdgeVersion;
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java

@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.EdgeVersion;
import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.edge.v1.UpdateMsgType;

13
application/src/main/java/org/thingsboard/server/service/edge/stats/EdgeStatsService.java

@ -54,7 +54,7 @@ import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_P
import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_TMP_FAILED; import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_TMP_FAILED;
@TbCoreComponent @TbCoreComponent
@ConditionalOnProperty(prefix = "edges.stats", name = "enabled", havingValue = "true", matchIfMissing = false) @ConditionalOnProperty(prefix = "edges.stats", name = "enabled", havingValue = "true")
@RequiredArgsConstructor @RequiredArgsConstructor
@Service @Service
@Slf4j @Slf4j
@ -70,7 +70,6 @@ public class EdgeStatsService {
@Value("${edges.stats.report-interval-millis:600000}") @Value("${edges.stats.report-interval-millis:600000}")
private long reportIntervalMillis; private long reportIntervalMillis;
@Scheduled( @Scheduled(
fixedDelayString = "${edges.stats.report-interval-millis:600000}", fixedDelayString = "${edges.stats.report-interval-millis:600000}",
initialDelayString = "${edges.stats.report-interval-millis:600000}" initialDelayString = "${edges.stats.report-interval-millis:600000}"
@ -80,13 +79,13 @@ public class EdgeStatsService {
long now = System.currentTimeMillis(); long now = System.currentTimeMillis();
long ts = now - (now % reportIntervalMillis); long ts = now - (now % reportIntervalMillis);
Map<EdgeId, MsgCounters> countersByEdge = statsCounterService.getCounterByEdge(); Map<EdgeId, MsgCounters> countersByEdgeSnapshot = new HashMap<>(statsCounterService.getMsgCountersByEdge());
Map<EdgeId, Long> lagByEdgeId = kafkaAdmin.isPresent() ? getEdgeLagByEdgeId(countersByEdge) : Collections.emptyMap(); boolean isKafkaStats = kafkaAdmin.isPresent();
Map<EdgeId, MsgCounters> countersByEdgeSnapshot = new HashMap<>(statsCounterService.getCounterByEdge()); Map<EdgeId, Long> lagByEdgeId = isKafkaStats ? getLagByEdgeId(countersByEdgeSnapshot) : Collections.emptyMap();
countersByEdgeSnapshot.forEach((edgeId, counters) -> { countersByEdgeSnapshot.forEach((edgeId, counters) -> {
TenantId tenantId = counters.getTenantId(); TenantId tenantId = counters.getTenantId();
if (kafkaAdmin.isPresent()) { if (isKafkaStats) {
counters.getMsgsLag().set(lagByEdgeId.getOrDefault(edgeId, 0L)); counters.getMsgsLag().set(lagByEdgeId.getOrDefault(edgeId, 0L));
} }
List<TsKvEntry> statsEntries = List.of( List<TsKvEntry> statsEntries = List.of(
@ -102,7 +101,7 @@ public class EdgeStatsService {
}); });
} }
private Map<EdgeId, Long> getEdgeLagByEdgeId(Map<EdgeId, MsgCounters> countersByEdge) { private Map<EdgeId, Long> getLagByEdgeId(Map<EdgeId, MsgCounters> countersByEdge) {
Map<EdgeId, String> edgeToTopicMap = countersByEdge.entrySet().stream() Map<EdgeId, String> edgeToTopicMap = countersByEdge.entrySet().stream()
.collect(Collectors.toMap( .collect(Collectors.toMap(
Map.Entry::getKey, Map.Entry::getKey,

2
application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java

@ -103,7 +103,7 @@ import org.thingsboard.server.dao.device.DeviceConnectivityConfiguration;
import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceCredentialsService;
import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.mobile.MobileAppDao; import org.thingsboard.server.dao.mobile.MobileAppDao;
import org.thingsboard.server.dao.notification.NotificationSettingsService; import org.thingsboard.server.dao.notification.NotificationSettingsService;
import org.thingsboard.server.dao.notification.NotificationTargetService; import org.thingsboard.server.dao.notification.NotificationTargetService;

2
application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java

@ -39,7 +39,7 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails; import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.dashboard.DashboardService;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService; import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService;
import org.thingsboard.server.dao.resource.ImageService; import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.resource.ResourceService;

2
application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java

@ -69,7 +69,7 @@ import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.ota.OtaPackageService;
import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.resource.ResourceService;

2
application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/DefaultJwtSettingsValidator.java

@ -21,7 +21,7 @@ import org.apache.commons.lang3.StringUtils;
import org.bouncycastle.util.Arrays; import org.bouncycastle.util.Arrays;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.security.model.JwtSettings; import org.thingsboard.server.common.data.security.model.JwtSettings;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import java.util.Base64; import java.util.Base64;
import java.util.Optional; import java.util.Optional;

2
application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java

@ -29,11 +29,11 @@ import org.thingsboard.server.common.data.security.model.mfa.account.AccountTwoF
import org.thingsboard.server.common.data.security.model.mfa.account.TwoFaAccountConfig; import org.thingsboard.server.common.data.security.model.mfa.account.TwoFaAccountConfig;
import org.thingsboard.server.common.data.security.model.mfa.provider.TwoFaProviderConfig; import org.thingsboard.server.common.data.security.model.mfa.provider.TwoFaProviderConfig;
import org.thingsboard.server.common.data.security.model.mfa.provider.TwoFaProviderType; import org.thingsboard.server.common.data.security.model.mfa.provider.TwoFaProviderType;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.service.ConstraintValidator; import org.thingsboard.server.dao.service.ConstraintValidator;
import org.thingsboard.server.dao.settings.AdminSettingsDao; import org.thingsboard.server.dao.settings.AdminSettingsDao;
import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.dao.user.UserAuthSettingsDao; import org.thingsboard.server.dao.user.UserAuthSettingsDao;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.service.security.auth.mfa.TwoFactorAuthService; import org.thingsboard.server.service.security.auth.mfa.TwoFactorAuthService;
import java.util.Comparator; import java.util.Comparator;

2
application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java

@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.security.model.SecuritySettings;
import org.thingsboard.server.common.data.security.model.UserPasswordPolicy; import org.thingsboard.server.common.data.security.model.UserPasswordPolicy;
import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.settings.SecuritySettingsService; import org.thingsboard.server.dao.settings.SecuritySettingsService;
import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.queue.util.TbCoreComponent;

2
application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java

@ -50,7 +50,7 @@ import org.thingsboard.server.common.data.security.model.SecuritySettings;
import org.thingsboard.server.common.data.security.model.UserPasswordPolicy; import org.thingsboard.server.common.data.security.model.UserPasswordPolicy;
import org.thingsboard.server.common.data.security.model.mfa.PlatformTwoFaSettings; import org.thingsboard.server.common.data.security.model.mfa.PlatformTwoFaSettings;
import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.audit.AuditLogService;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.dao.settings.SecuritySettingsService; import org.thingsboard.server.dao.settings.SecuritySettingsService;
import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.user.UserService;

2
application/src/main/java/org/thingsboard/server/service/security/system/SystemSecurityService.java

@ -24,7 +24,7 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.security.model.UserPasswordPolicy; import org.thingsboard.server.common.data.security.model.UserPasswordPolicy;
import org.thingsboard.server.common.data.security.model.mfa.PlatformTwoFaSettings; import org.thingsboard.server.common.data.security.model.mfa.PlatformTwoFaSettings;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.SecurityUser;
public interface SystemSecurityService { public interface SystemSecurityService {

2
application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java

@ -31,7 +31,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.common.data.util.ThrowingRunnable; import org.thingsboard.server.common.data.util.ThrowingRunnable;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.TbLogEntityActionService; import org.thingsboard.server.service.entitiy.TbLogEntityActionService;

2
application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java

@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.MoreExecutors;
import com.google.protobuf.ByteString; import com.google.protobuf.ByteString;
import org.thingsboard.server.exception.EntitiesLimitExceededException;
import jakarta.annotation.PostConstruct; import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy; import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@ -76,7 +77,6 @@ import org.thingsboard.server.dao.device.provision.ProvisionFailedException;
import org.thingsboard.server.dao.device.provision.ProvisionRequest; import org.thingsboard.server.dao.device.provision.ProvisionRequest;
import org.thingsboard.server.dao.device.provision.ProvisionResponse; import org.thingsboard.server.dao.device.provision.ProvisionResponse;
import org.thingsboard.server.dao.device.provision.ProvisionResponseStatus; import org.thingsboard.server.dao.device.provision.ProvisionResponseStatus;
import org.thingsboard.server.dao.exception.EntitiesLimitExceededException;
import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.ota.OtaPackageService;
import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.queue.QueueService;
import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.relation.RelationService;

12
application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java

@ -33,6 +33,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ArgumentIntervalProt
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto;
import org.thingsboard.server.gen.transport.TransportProtos.EntityIdProto;
import org.thingsboard.server.gen.transport.TransportProtos.GeofencingArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingArgumentProto;
import org.thingsboard.server.gen.transport.TransportProtos.GeofencingZoneProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingZoneProto;
import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto;
@ -61,6 +62,7 @@ import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgume
import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalculatedFieldState;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.TreeMap; import java.util.TreeMap;
@ -108,6 +110,7 @@ public class CalculatedFieldUtils {
case SINGLE_VALUE -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry)); case SINGLE_VALUE -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry));
case TS_ROLLING -> builder.addRollingValueArguments(toRollingArgumentProto(argName, (TsRollingArgumentEntry) argEntry)); case TS_ROLLING -> builder.addRollingValueArguments(toRollingArgumentProto(argName, (TsRollingArgumentEntry) argEntry));
case GEOFENCING -> builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry)); case GEOFENCING -> builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry));
case PROPAGATION -> builder.addAllPropagationEntityIds(toPropagationEntityIdsProto((PropagationArgumentEntry) argEntry));
case RELATED_ENTITIES -> { case RELATED_ENTITIES -> {
RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry; RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry;
Map<EntityId, ArgumentEntry> entityInputs = relatedEntitiesArgumentEntry.getEntityInputs(); Map<EntityId, ArgumentEntry> entityInputs = relatedEntitiesArgumentEntry.getEntityInputs();
@ -140,6 +143,10 @@ public class CalculatedFieldUtils {
return builder.build(); return builder.build();
} }
private static List<EntityIdProto> toPropagationEntityIdsProto(PropagationArgumentEntry argEntry) {
return argEntry.getEntityIds().stream().map(ProtoUtils::toProto).collect(Collectors.toList());
}
private static AlarmRuleStateProto toAlarmRuleStateProto(AlarmRuleState ruleState) { private static AlarmRuleStateProto toAlarmRuleStateProto(AlarmRuleState ruleState) {
return AlarmRuleStateProto.newBuilder() return AlarmRuleStateProto.newBuilder()
.setSeverity(Optional.ofNullable(ruleState.getSeverity()).map(Enum::name).orElse("")) .setSeverity(Optional.ofNullable(ruleState.getSeverity()).map(Enum::name).orElse(""))
@ -275,7 +282,10 @@ public class CalculatedFieldUtils {
state.getArguments().put(argProto.getKey(), fromRollingArgumentProto(argProto))); state.getArguments().put(argProto.getKey(), fromRollingArgumentProto(argProto)));
case GEOFENCING -> proto.getGeofencingArgumentsList().forEach(argProto -> case GEOFENCING -> proto.getGeofencingArgumentsList().forEach(argProto ->
state.getArguments().put(argProto.getArgName(), fromGeofencingArgumentProto(argProto))); state.getArguments().put(argProto.getArgName(), fromGeofencingArgumentProto(argProto)));
case PROPAGATION -> state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, new PropagationArgumentEntry()); case PROPAGATION -> {
List<EntityId> propagationEntityIds = proto.getPropagationEntityIdsList().stream().map(ProtoUtils::fromProto).toList();
state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, new PropagationArgumentEntry(propagationEntityIds));
}
case ALARM -> { case ALARM -> {
AlarmCalculatedFieldState alarmState = (AlarmCalculatedFieldState) state; AlarmCalculatedFieldState alarmState = (AlarmCalculatedFieldState) state;
AlarmStateProto alarmStateProto = proto.getAlarmState(); AlarmStateProto alarmStateProto = proto.getAlarmState();

2
application/src/main/java/org/thingsboard/server/utils/LwM2mObjectModelUtils.java

@ -26,7 +26,7 @@ import org.thingsboard.server.common.data.lwm2m.LwM2mInstance;
import org.thingsboard.server.common.data.lwm2m.LwM2mObject; import org.thingsboard.server.common.data.lwm2m.LwM2mObject;
import org.thingsboard.server.common.data.lwm2m.LwM2mResourceObserve; import org.thingsboard.server.common.data.lwm2m.LwM2mResourceObserve;
import org.thingsboard.server.common.data.util.TbDDFFileParser; import org.thingsboard.server.common.data.util.TbDDFFileParser;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;

68
application/src/main/resources/thingsboard.yml

@ -516,14 +516,14 @@ actors:
response_timeout_ms: "${ACTORS_RPC_RESPONSE_TIMEOUT_MS:30000}" response_timeout_ms: "${ACTORS_RPC_RESPONSE_TIMEOUT_MS:30000}"
# Close transport session if RPC delivery timed out. If enabled, RPC will be reverted to the queued state. # Close transport session if RPC delivery timed out. If enabled, RPC will be reverted to the queued state.
# Note: # Note:
# - For MQTT transport: # <ul><li> For MQTT transport:
# - QoS level 0: This feature does not apply, as no acknowledgment is expected, and therefore no timeout is triggered. # <ul><li> QoS level 0: This feature does not apply, as no acknowledgment is expected, and therefore no timeout is triggered.</li>
# - QoS level 1: This feature applies, as an acknowledgment is expected. # <li> QoS level 1: This feature applies, as an acknowledgment is expected.</li>
# - QoS level 2: Unsupported. # <li> QoS level 2: Unsupported.</li></ul></li>
# - For CoAP transport: # <li> For CoAP transport:
# - Confirmable requests: This feature applies, as delivery confirmation is expected. # <ul><li> Confirmable requests: This feature applies, as delivery confirmation is expected.</li>
# - Non-confirmable requests: This feature does not apply, as no delivery acknowledgment is expected. # <li> Non-confirmable requests: This feature does not apply, as no delivery acknowledgment is expected.</li></ul></li>
# - For HTTP and SNPM transports: RPC is considered delivered immediately, and there is no logic to await acknowledgment. # <li> For HTTP and SNPM transports: RPC is considered delivered immediately, and there is no logic to await acknowledgment.</li></ul>
close_session_on_rpc_delivery_timeout: "${ACTORS_RPC_CLOSE_SESSION_ON_RPC_DELIVERY_TIMEOUT:false}" close_session_on_rpc_delivery_timeout: "${ACTORS_RPC_CLOSE_SESSION_ON_RPC_DELIVERY_TIMEOUT:false}"
statistics: statistics:
# Enable/disable actor statistics # Enable/disable actor statistics
@ -733,13 +733,13 @@ redis:
# if set false will be used pool config build from values of the pool config section # if set false will be used pool config build from values of the pool config section
useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}"
sentinel: sentinel:
# name of the master node # Name of the master node
master: "${REDIS_MASTER:}" master: "${REDIS_MASTER:}"
# comma-separated list of "host:port" pairs of sentinels # Comma-separated list of "host:port" pairs of sentinels
sentinels: "${REDIS_SENTINELS:}" sentinels: "${REDIS_SENTINELS:}"
# password to authenticate with sentinel # Password to authenticate with sentinel
password: "${REDIS_SENTINEL_PASSWORD:}" password: "${REDIS_SENTINEL_PASSWORD:}"
# if set false will be used pool config build from values of the pool config section # If set false will be used pool config build from values of the pool config section
useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}"
# db index # db index
db: "${REDIS_DB:0}" db: "${REDIS_DB:0}"
@ -1039,10 +1039,10 @@ transport:
activity: activity:
# This property specifies the strategy for reporting activity events within each reporting period. # This property specifies the strategy for reporting activity events within each reporting period.
# The accepted values are 'FIRST', 'LAST', 'FIRST_AND_LAST' and 'ALL'. # The accepted values are 'FIRST', 'LAST', 'FIRST_AND_LAST' and 'ALL'.
# - 'FIRST': Only the first activity event in each reporting period is reported. # <ul><li> 'FIRST': Only the first activity event in each reporting period is reported.</li>
# - 'LAST': Only the last activity event in the reporting period is reported. # <li> 'LAST': Only the last activity event in the reporting period is reported.</li>
# - 'FIRST_AND_LAST': Both the first and last activity events in the reporting period are reported. # <li> 'FIRST_AND_LAST': Both the first and last activity events in the reporting period are reported.</li>
# - 'ALL': All activity events in the reporting period are reported. # <li> 'ALL': All activity events in the reporting period are reported.</li></ul>
reporting_strategy: "${TB_TRANSPORT_ACTIVITY_REPORTING_STRATEGY:LAST}" reporting_strategy: "${TB_TRANSPORT_ACTIVITY_REPORTING_STRATEGY:LAST}"
json: json:
# Cast String data types to Numeric if possible when processing Telemetry/Attributes JSON # Cast String data types to Numeric if possible when processing Telemetry/Attributes JSON
@ -1160,15 +1160,15 @@ transport:
dtls: dtls:
# RFC7925_RETRANSMISSION_TIMEOUT_IN_MILLISECONDS = 9000 # RFC7925_RETRANSMISSION_TIMEOUT_IN_MILLISECONDS = 9000
retransmission_timeout: "${LWM2M_DTLS_RETRANSMISSION_TIMEOUT_MS:9000}" retransmission_timeout: "${LWM2M_DTLS_RETRANSMISSION_TIMEOUT_MS:9000}"
# CoAP DTLS connection ID length for LWM2M. RFC 9146, Connection Identifier for DTLS 1.2 # LWM2M DTLS connection ID length for LWM2M. RFC 9146, Connection Identifier for DTLS 1.2
# Default: off # Default: off. <br>
# Control usage of DTLS connection ID length (CID). # Control usage of DTLS connection ID length (CID).
# - 'off' to deactivate it. # <ul> <li> 'off' to deactivate it. </li>
# - 'on' to activate Connection ID support (same as CID 0 or more 0). # <li> 'on' to activate Connection ID support (same as CID 0 or more 0). </li>
# - A positive value defines generated CID size in bytes. # <li> A positive value defines generated CID size in bytes.</li>
# - A value of 0 means we accept using CID but will not generate one for foreign peer (enables support but not for incoming traffic). # <li> A value of 0 means we accept using CID but will not generate one for foreign peer (enables support but not for incoming traffic). </li>
# - A value between 0 and <= 4: SingleNodeConnectionIdGenerator is used # <li> A value between 0 and <= 4: SingleNodeConnectionIdGenerator is used </li>
# - A value that are > 4: MultiNodeConnectionIdGenerator is used # <li> A value that are > 4: MultiNodeConnectionIdGenerator is used </li> </ul>
connection_id_length: "${LWM2M_DTLS_CONNECTION_ID_LENGTH:8}" connection_id_length: "${LWM2M_DTLS_CONNECTION_ID_LENGTH:8}"
server: server:
# LwM2M Server ID # LwM2M Server ID
@ -1351,14 +1351,14 @@ coap:
# CoAP DTLS bind port # CoAP DTLS bind port
bind_port: "${COAP_DTLS_BIND_PORT:5684}" bind_port: "${COAP_DTLS_BIND_PORT:5684}"
# CoAP DTLS connection ID length. RFC 9146, Connection Identifier for DTLS 1.2 # CoAP DTLS connection ID length. RFC 9146, Connection Identifier for DTLS 1.2
# Default: off # Default: off. <br>
# Control usage of DTLS connection ID length (CID). # Control usage of DTLS connection ID length (CID).
# - 'off' to deactivate it. # <ul> <li> 'off' to deactivate it. </li>
# - 'on' to activate Connection ID support (same as CID 0 or more 0). # <li> 'on' to activate Connection ID support (same as CID 0 or more 0). </li>
# - A positive value defines generated CID size in bytes. # <li> A positive value defines generated CID size in bytes.</li>
# - A value of 0 means we accept using CID but will not generate one for foreign peer (enables support but not for incoming traffic). # <li> A value of 0 means we accept using CID but will not generate one for foreign peer (enables support but not for incoming traffic).</li>
# - A value between 0 and <= 4: SingleNodeConnectionIdGenerator is used # <li> A value between 0 and <= 4: SingleNodeConnectionIdGenerator is used</li>
# - A value that are > 4: MultiNodeConnectionIdGenerator is used # <li> A value that are > 4: MultiNodeConnectionIdGenerator is used</li> </ul>
connection_id_length: "${COAP_DTLS_CONNECTION_ID_LENGTH:8}" connection_id_length: "${COAP_DTLS_CONNECTION_ID_LENGTH:8}"
# Specify the MTU (Maximum Transmission Unit). # Specify the MTU (Maximum Transmission Unit).
# Should be used if LAN MTU is not used, e.g. if IP tunnels are used or if the client uses a smaller value than the LAN MTU. # Should be used if LAN MTU is not used, e.g. if IP tunnels are used or if the client uses a smaller value than the LAN MTU.
@ -1375,13 +1375,13 @@ coap:
# In order to negotiate smaller maximum fragment lengths, # In order to negotiate smaller maximum fragment lengths,
# clients MAY include an extension of type "max_fragment_length" in the (extended) client hello. # clients MAY include an extension of type "max_fragment_length" in the (extended) client hello.
# The "extension_data" field of this extension SHALL contain: # The "extension_data" field of this extension SHALL contain:
# enum { # <pre> enum {
# 2^9(1) == 512, # 2^9(1) == 512,
# 2^10(2) == 1024, # 2^10(2) == 1024,
# 2^11(3) == 2048, # 2^11(3) == 2048,
# 2^12(4) == 4096, # 2^12(4) == 4096,
# (255) # (255)
# } MaxFragmentLength; # } MaxFragmentLength; </pre>
# TLS already requires clients and servers to support fragmentation of handshake messages. # TLS already requires clients and servers to support fragmentation of handshake messages.
max_fragment_length: "${COAP_DTLS_MAX_FRAGMENT_LENGTH:1024}" max_fragment_length: "${COAP_DTLS_MAX_FRAGMENT_LENGTH:1024}"
# Server DTLS credentials # Server DTLS credentials
@ -1745,6 +1745,8 @@ queue:
print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}"
# Time to wait for the stats-loading requests to Kafka to finish # Time to wait for the stats-loading requests to Kafka to finish
kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}" kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}"
# Topics cache TTL in milliseconds. 5 minutes by default
topics_cache_ttl_ms: "${TB_QUEUE_KAFKA_TOPICS_CACHE_TTL_MS:300000}"
partitions: partitions:
hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256
transport_api: transport_api:

57
application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java

@ -1089,7 +1089,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
// Telemetry on device // Telemetry on device
doPost("/api/plugins/telemetry/DEVICE/" + device.getUuidId() + "/timeseries/unusedScope", doPost("/api/plugins/telemetry/DEVICE/" + device.getUuidId() + "/timeseries/unusedScope",
JacksonUtil.toJsonNode("{\"temperature\":12.5}")).andExpect(status().isOk()); JacksonUtil.toJsonNode("{\"temperature\":12.5, \"humidity\":85}")).andExpect(status().isOk());
// --- Build CF: PROPAGATION with expression --- // --- Build CF: PROPAGATION with expression ---
CalculatedField cf = new CalculatedField(); CalculatedField cf = new CalculatedField();
@ -1102,11 +1102,14 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
cfg.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE)); cfg.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE));
cfg.setApplyExpressionToResolvedArguments(true); cfg.setApplyExpressionToResolvedArguments(true);
Argument arg = new Argument(); Argument arg1 = new Argument();
arg.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); arg1.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null));
cfg.setArguments(Map.of("t", arg));
cfg.setExpression("{\"testResult\": t * 2}"); Argument arg2 = new Argument();
arg2.setRefEntityKey(new ReferencedEntityKey("humidity", ArgumentType.TS_LATEST, null));
cfg.setArguments(Map.of("t", arg1, "h", arg2));
cfg.setExpression("return { testResult: (t + h) / 2};");
AttributesOutput output = new AttributesOutput(); AttributesOutput output = new AttributesOutput();
output.setScope(AttributeScope.SERVER_SCOPE); output.setScope(AttributeScope.SERVER_SCOPE);
@ -1125,8 +1128,8 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
ArrayNode attrs2 = getServerAttributes(asset2.getId(), "testResult"); ArrayNode attrs2 = getServerAttributes(asset2.getId(), "testResult");
assertThat(attrs1).isNotNull(); assertThat(attrs1).isNotNull();
assertThat(attrs2).isNotNull(); assertThat(attrs2).isNotNull();
assertThat(attrs1.get(0).get("value").asDouble()).isEqualTo(25.0); assertThat(attrs1.get(0).get("value").asDouble()).isEqualTo(48.75);
assertThat(attrs2.get(0).get("value").asDouble()).isEqualTo(25.0); assertThat(attrs2.get(0).get("value").asDouble()).isEqualTo(48.75);
}); });
String deleteUrl = String.format("/api/v2/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", String deleteUrl = String.format("/api/v2/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s",
@ -1148,7 +1151,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
ArrayNode attrs2 = getServerAttributes(asset2.getId(), "testResult"); ArrayNode attrs2 = getServerAttributes(asset2.getId(), "testResult");
assertThat(attrs1).isNullOrEmpty(); assertThat(attrs1).isNullOrEmpty();
assertThat(attrs2).isNotNull(); assertThat(attrs2).isNotNull();
assertThat(attrs2.get(0).get("value").asDouble()).isEqualTo(50); assertThat(attrs2.get(0).get("value").asDouble()).isEqualTo(55);
}); });
} }
@ -1167,7 +1170,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
// Telemetry on device // Telemetry on device
long ts = System.currentTimeMillis() - 300000L; long ts = System.currentTimeMillis() - 300000L;
postTelemetry(device.getId(), String.format("{\"ts\": %s, \"values\": {\"temperature\":12.5}}", ts)); postTelemetry(device.getId(), String.format("{\"ts\": %s, \"values\": {\"temperature\":12.5, \"humidity\":85}}", ts));
// --- Build CF: PROPAGATION without expression --- // --- Build CF: PROPAGATION without expression ---
CalculatedField cf = new CalculatedField(); CalculatedField cf = new CalculatedField();
@ -1180,9 +1183,12 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
cfg.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE)); cfg.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE));
cfg.setApplyExpressionToResolvedArguments(false); // arguments-only mode cfg.setApplyExpressionToResolvedArguments(false); // arguments-only mode
Argument arg = new Argument(); Argument arg1 = new Argument();
arg.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); arg1.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null));
cfg.setArguments(Map.of("temperatureComputed", arg)); Argument arg2 = new Argument();
arg2.setRefEntityKey(new ReferencedEntityKey("humidity", ArgumentType.TS_LATEST, null));
cfg.setArguments(Map.of("temperatureComputed", arg1, "humidityComputed", arg2));
TimeSeriesOutput output = new TimeSeriesOutput(); TimeSeriesOutput output = new TimeSeriesOutput();
output.setStrategy(new TimeSeriesImmediateOutputStrategy(0, true, true, true, true)); output.setStrategy(new TimeSeriesImmediateOutputStrategy(0, true, true, true, true));
@ -1197,14 +1203,18 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
.atMost(TIMEOUT, TimeUnit.SECONDS) .atMost(TIMEOUT, TimeUnit.SECONDS)
.pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS)
.untilAsserted(() -> { .untilAsserted(() -> {
ObjectNode telemetry1 = getLatestTelemetry(asset1.getId(), "temperatureComputed"); ObjectNode telemetry1 = getLatestTelemetry(asset1.getId(), "temperatureComputed,humidityComputed");
ObjectNode telemetry2 = getLatestTelemetry(asset2.getId(), "temperatureComputed"); ObjectNode telemetry2 = getLatestTelemetry(asset2.getId(), "temperatureComputed,humidityComputed");
assertThat(telemetry1).isNotNull(); assertThat(telemetry1).isNotNull();
assertThat(telemetry2).isNotNull(); assertThat(telemetry2).isNotNull();
assertThat(telemetry1.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts)); assertThat(telemetry1.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts));
assertThat(telemetry1.get("temperatureComputed").get(0).get("value").asDouble()).isEqualTo(12.5); assertThat(telemetry1.get("temperatureComputed").get(0).get("value").asDouble()).isEqualTo(12.5);
assertThat(telemetry1.get("humidityComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts));
assertThat(telemetry1.get("humidityComputed").get(0).get("value").asDouble()).isEqualTo(85);
assertThat(telemetry2.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts)); assertThat(telemetry2.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts));
assertThat(telemetry2.get("temperatureComputed").get(0).get("value").asDouble()).isEqualTo(12.5); assertThat(telemetry2.get("temperatureComputed").get(0).get("value").asDouble()).isEqualTo(12.5);
assertThat(telemetry2.get("humidityComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts));
assertThat(telemetry2.get("humidityComputed").get(0).get("value").asDouble()).isEqualTo(85);
}); });
String deleteUrl = String.format("/api/v2/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", String deleteUrl = String.format("/api/v2/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s",
@ -1212,10 +1222,10 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
EntityRelation.CONTAINS_TYPE, device.getId().getId(), EntityType.DEVICE EntityRelation.CONTAINS_TYPE, device.getId().getId(), EntityType.DEVICE
); );
doDelete(deleteUrl).andExpect(status().isOk()); doDelete(deleteUrl).andExpect(status().isOk());
doDelete("/api/plugins/telemetry/ASSET/" + asset1.getId() + "/timeseries/delete?keys=temperatureComputed&deleteAllDataForKeys=true").andExpect(status().isOk()); doDelete("/api/plugins/telemetry/ASSET/" + asset1.getId() + "/timeseries/delete?keys=temperatureComputed,humidityComputed&deleteAllDataForKeys=true").andExpect(status().isOk());
// Update telemetry on device // Update telemetry on the device
long newTs = System.currentTimeMillis() - 300000L; long newTs = ts + 300000L;
postTelemetry(device.getId(), String.format("{\"ts\": %s, \"values\": {\"temperature\":25}}", newTs)); postTelemetry(device.getId(), String.format("{\"ts\": %s, \"values\": {\"temperature\":25}}", newTs));
// --- Assert propagated calculation (arguments-only mode after update) --- // --- Assert propagated calculation (arguments-only mode after update) ---
@ -1223,13 +1233,18 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
.atMost(TIMEOUT, TimeUnit.SECONDS) .atMost(TIMEOUT, TimeUnit.SECONDS)
.pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS)
.untilAsserted(() -> { .untilAsserted(() -> {
ObjectNode telemetry1 = getLatestTelemetry(asset1.getId(), "temperatureComputed"); ObjectNode telemetry1 = getLatestTelemetry(asset1.getId(), "temperatureComputed,humidityComputed");
ObjectNode telemetry2 = getLatestTelemetry(asset2.getId(), "temperatureComputed"); ObjectNode telemetry2 = getLatestTelemetry(asset2.getId(), "temperatureComputed,humidityComputed");
assertThat(telemetry1).isNotNull(); assertThat(telemetry1).isNotNull();
assertThat(telemetry2).isNotNull(); assertThat(telemetry2).isNotNull();
assertThat(telemetry1.get("temperatureComputed").get(0).get("value")).isEqualTo(NullNode.instance); assertThat(telemetry1.get("temperatureComputed").get(0).get("value")).isEqualTo(NullNode.instance);
assertThat(telemetry1.get("humidityComputed").get(0).get("value")).isEqualTo(NullNode.instance);
assertThat(telemetry2.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(newTs)); assertThat(telemetry2.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(newTs));
assertThat(telemetry2.get("temperatureComputed").get(0).get("value").asDouble()).isEqualTo(25); assertThat(telemetry2.get("temperatureComputed").get(0).get("value").asDouble()).isEqualTo(25);
// TS for humidity is not updated -> expected
assertThat(telemetry2.get("humidityComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts));
assertThat(telemetry2.get("humidityComputed").get(0).get("value").asDouble()).isEqualTo(85);
}); });
Asset asset3 = createAsset("Propagated Asset 3", null); Asset asset3 = createAsset("Propagated Asset 3", null);
@ -1241,10 +1256,12 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
.atMost(TIMEOUT, TimeUnit.SECONDS) .atMost(TIMEOUT, TimeUnit.SECONDS)
.pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS)
.untilAsserted(() -> { .untilAsserted(() -> {
ObjectNode telemetry = getLatestTelemetry(asset3.getId(), "temperatureComputed"); ObjectNode telemetry = getLatestTelemetry(asset3.getId(), "temperatureComputed,humidityComputed");
assertThat(telemetry).isNotNull(); assertThat(telemetry).isNotNull();
assertThat(telemetry.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(newTs)); assertThat(telemetry.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(newTs));
assertThat(telemetry.get("temperatureComputed").get(0).get("value").asDouble()).isEqualTo(25); assertThat(telemetry.get("temperatureComputed").get(0).get("value").asDouble()).isEqualTo(25);
assertThat(telemetry.get("humidityComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(newTs));
assertThat(telemetry.get("humidityComputed").get(0).get("value").asDouble()).isEqualTo(85);
}); });
} }

2
application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java

@ -1209,7 +1209,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
Map<CalculatedFieldId, CalculatedFieldState> statesMap = (Map<CalculatedFieldId, CalculatedFieldState>) ReflectionTestUtils.getField(processor, "states"); Map<CalculatedFieldId, CalculatedFieldState> statesMap = (Map<CalculatedFieldId, CalculatedFieldState>) ReflectionTestUtils.getField(processor, "states");
Awaitility.await("CF state for entity actor ready to refresh dynamic arguments").atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> { Awaitility.await("CF state for entity actor ready to refresh dynamic arguments").atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> {
CalculatedFieldState calculatedFieldState = statesMap.get(cfId); CalculatedFieldState calculatedFieldState = statesMap.get(cfId);
boolean isReady = calculatedFieldState != null && ((GeofencingCalculatedFieldState) calculatedFieldState).getLastDynamicArgumentsRefreshTs() < boolean isReady = calculatedFieldState != null && ((GeofencingCalculatedFieldState) calculatedFieldState).getLastScheduledRefreshTs() <
System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(scheduledUpdateInterval); System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(scheduledUpdateInterval);
log.warn("entityId {}, cfId {}, state ready to refresh == {}", entityId, cfId, isReady); log.warn("entityId {}, cfId {}, state ready to refresh == {}", entityId, cfId, isReady);
return isReady; return isReady;

2
application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java

@ -50,7 +50,7 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.asset.AssetDao;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;

2
application/src/test/java/org/thingsboard/server/controller/AssetProfileControllerTest.java

@ -42,7 +42,7 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.asset.AssetProfileDao; import org.thingsboard.server.dao.asset.AssetProfileDao;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.ArrayList; import java.util.ArrayList;

2
application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java

@ -44,7 +44,7 @@ import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.customer.CustomerDao; import org.thingsboard.server.dao.customer.CustomerDao;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.ArrayList; import java.util.ArrayList;

2
application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java

@ -51,7 +51,7 @@ import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.dashboard.DashboardDao; import org.thingsboard.server.dao.dashboard.DashboardDao;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.ArrayList; import java.util.ArrayList;

2
application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java

@ -72,7 +72,7 @@ import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportColumn
import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportRequest; import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportRequest;
import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportResult;
import org.thingsboard.server.dao.device.DeviceDao; import org.thingsboard.server.dao.device.DeviceDao;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException;
import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;

2
application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java

@ -52,7 +52,7 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.device.DeviceProfileDao; import org.thingsboard.server.dao.device.DeviceProfileDao;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.ArrayList; import java.util.ArrayList;

2
application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java

@ -72,7 +72,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.security.model.JwtSettings; import org.thingsboard.server.common.data.security.model.JwtSettings;
import org.thingsboard.server.dao.edge.EdgeDao; import org.thingsboard.server.dao.edge.EdgeDao;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.edge.imitator.EdgeImitator; import org.thingsboard.server.edge.imitator.EdgeImitator;

2
application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java

@ -63,7 +63,7 @@ import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.dao.entityview.EntityViewDao; import org.thingsboard.server.dao.entityview.EntityViewDao;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;

2
application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java

@ -37,7 +37,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;

2
application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java

@ -47,7 +47,7 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.rule.RuleChainDao; import org.thingsboard.server.dao.rule.RuleChainDao;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;

2
application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java

@ -48,7 +48,7 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.common.data.widget.WidgetType;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails; import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.ArrayList; import java.util.ArrayList;

2
application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java

@ -50,7 +50,7 @@ import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.settings.StarredDashboardInfo; import org.thingsboard.server.common.data.settings.StarredDashboardInfo;
import org.thingsboard.server.common.data.settings.UserDashboardsInfo; import org.thingsboard.server.common.data.settings.UserDashboardsInfo;
import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.dao.user.UserDao; import org.thingsboard.server.dao.user.UserDao;

2
application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java

@ -29,7 +29,7 @@ import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.ArrayList; import java.util.ArrayList;

148
application/src/test/java/org/thingsboard/server/edge/EdgeStatsIntegrationTest.java

@ -0,0 +1,148 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.edge;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.extern.slf4j.Slf4j;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.edge.EdgeEvent;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.dao.edge.stats.EdgeStatsCounterService;
import org.thingsboard.server.dao.edge.stats.EdgeStatsKey;
import org.thingsboard.server.dao.edge.stats.MsgCounters;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.service.edge.stats.EdgeStatsService;
import java.time.Duration;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import static org.awaitility.Awaitility.await;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_ADDED;
import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_PERMANENTLY_FAILED;
import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_PUSHED;
import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_TMP_FAILED;
@DaoSqlTest
@Slf4j
public class EdgeStatsIntegrationTest extends AbstractEdgeTest {
private static final String STATISTICS_DEVICE_PROFILE = "STATISTICS";
private static final long EXPECTED_MSGS_ADDED = 6L;
private static final long EXPECTED_MSGS_PUSHED = 6L;
private static final long EXPECTED_MSGS_PERMANENTLY_FAILED = 0L;
private static final long EXPECTED_MSGS_TMP_FAILED = 0L;
@Autowired
private EdgeStatsService edgeStatsService;
@Autowired
private EdgeStatsCounterService statsCounterService;
@Test
public void testReportStats() throws Exception {
// GIVEN
simulateEdgeEventsAddedDownlinkPushed();
// Await Edge Counters Updated
await().atMost(10, TimeUnit.SECONDS).pollInterval(Duration.ofMillis(200)).untilAsserted(() -> {
MsgCounters counters = statsCounterService.getMsgCountersByEdge().get(edge.getId());
assertEquals(EXPECTED_MSGS_ADDED, counters.getMsgsAdded().get());
assertEquals(EXPECTED_MSGS_PUSHED, counters.getMsgsPushed().get());
assertEquals(EXPECTED_MSGS_PERMANENTLY_FAILED, counters.getMsgsPermanentlyFailed().get());
assertEquals(EXPECTED_MSGS_TMP_FAILED, counters.getMsgsTmpFailed().get());
});
Thread.sleep(1000);
// WHEN
edgeStatsService.reportStats();
// THEN
await().atMost(10, TimeUnit.SECONDS).pollInterval(Duration.ofMillis(200)).untilAsserted(() -> {
List<TsKvEntry> actualStats = fetchLatestStats();
assertEquals(EXPECTED_MSGS_ADDED, getStatsLongValue(actualStats, DOWNLINK_MSGS_ADDED));
assertEquals(EXPECTED_MSGS_PUSHED, getStatsLongValue(actualStats, DOWNLINK_MSGS_PUSHED));
assertEquals(EXPECTED_MSGS_PERMANENTLY_FAILED, getStatsLongValue(actualStats, DOWNLINK_MSGS_PERMANENTLY_FAILED));
assertEquals(EXPECTED_MSGS_TMP_FAILED, getStatsLongValue(actualStats, DOWNLINK_MSGS_TMP_FAILED));
});
}
private long getStatsLongValue(List<TsKvEntry> stats, EdgeStatsKey key) {
return stats.stream().filter(e -> e.getKey().equals(key.getKey())).findFirst().get().getLongValue().orElse(0L);
}
private List<TsKvEntry> fetchLatestStats() throws ExecutionException, InterruptedException {
return tsService.findLatest(
tenantId,
edge.getId(),
Arrays.stream(EdgeStatsKey.values()).map(EdgeStatsKey::getKey).toList()).get();
}
private void simulateEdgeEventsAddedDownlinkPushed() throws InterruptedException, ExecutionException {
statsCounterService.clear(edge.getId());
// Save device and assign to edge
// 2 DOWNLINK_MSGS_ADDED, EdgeEvents: [{DEVICE_PROFILE: ADDED}, {DEVICE: ASSIGNED_TO_EDGE}]
// 2 DOWNLINK_MSGS_PUSHED, Downlinks: [{deviceProfileUpdateMsg}, {deviceUpdateMsg, deviceProfileUpdateMsg, deviceCredentialsUpdateMsg}]
edgeImitator.expectMessageAmount(4);
Device savedDevice = saveDevice("StatisticDevice", STATISTICS_DEVICE_PROFILE);
doPost("/api/edge/" + edge.getUuidId() + "/device/" + savedDevice.getUuidId(), Device.class);
Assert.assertTrue(edgeImitator.waitForMessages());
// Save asset and assign to edge
// 1 DOWNLINK_MSGS_ADDED, EdgeEvents: [{ASSET: ASSIGNED_TO_EDGE}]
// 1 DOWNLINK_MSGS_PUSHED, Downlinks: [{assetUpdateMsg, assetProfileUpdateMsg}]
edgeImitator.expectMessageAmount(2);
Asset savedAsset = saveAsset("Edge Asset");
doPost("/api/edge/" + edge.getUuidId()
+ "/asset/" + savedAsset.getUuidId(), Asset.class);
Assert.assertTrue(edgeImitator.waitForMessages());
// Create customer and assign edge to the customer
// 2 DOWNLINK_MSGS_ADDED, EdgeEvents: [{CUSTOMER: ADDED}, {EDGE: ASSIGNED_TO_CUSTOMER}]
// 2 DOWNLINK_MSGS_PUSHED, Downlinks: [{customerUpdateMsg}, {edgeConfiguration}]
edgeImitator.expectMessageAmount(2);
Customer customer = new Customer();
customer.setTitle("Edge Customer");
Customer savedCustomer = doPost("/api/customer", customer, Customer.class);
doPost("/api/customer/" + savedCustomer.getUuidId()
+ "/edge/" + edge.getUuidId(), Edge.class);
Assert.assertTrue(edgeImitator.waitForMessages());
// Send device telemetry downlink for the device
// 1 DOWNLINK_MSGS_ADDED, EdgeEvents: [{DEVICE: TIMESERIES_UPDATED}]
// 1 DOWNLINK_MSGS_PUSHED, Downlinks: [{entityData}]
edgeImitator.expectMessageAmount(1);
String timeseriesData = "{\"data\":{\"temperature\":25},\"ts\":" + System.currentTimeMillis() + "}";
JsonNode timeseriesEntityData = JacksonUtil.toJsonNode(timeseriesData);
EdgeEvent edgeEvent = constructEdgeEvent(tenantId, edge.getId(), EdgeEventActionType.TIMESERIES_UPDATED, savedDevice.getId().getId(), EdgeEventType.DEVICE, timeseriesEntityData);
edgeEventService.saveAsync(edgeEvent).get();
Assert.assertTrue(edgeImitator.waitForMessages());
}
}

77
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationArgumentEntryTest.java

@ -104,19 +104,19 @@ public class PropagationArgumentEntryTest {
@Test @Test
void testUpdateEntryWhenAdded() { void testUpdateEntryWhenAdded() {
var added = new PropagationArgumentEntry(); var added = new PropagationArgumentEntry();
added.setAdded(ENTITY_3_ID); added.setAdded(List.of(ENTITY_3_ID));
boolean changed = entry.updateEntry(added); boolean changed = entry.updateEntry(added);
assertThat(changed).isTrue(); assertThat(changed).isTrue();
assertThat(entry.getEntityIds()).containsExactlyInAnyOrder(ENTITY_1_ID, ENTITY_2_ID, ENTITY_3_ID); assertThat(entry.getEntityIds()).containsExactlyInAnyOrder(ENTITY_1_ID, ENTITY_2_ID, ENTITY_3_ID);
assertThat(entry.getAdded()).isEqualTo(ENTITY_3_ID); assertThat(entry.getAdded()).isEqualTo(List.of(ENTITY_3_ID));
} }
@Test @Test
void testUpdateEntryWhenAddedExistingEntity() { void testUpdateEntryWhenAddedExistingEntity() {
var added = new PropagationArgumentEntry(); var added = new PropagationArgumentEntry();
added.setAdded(ENTITY_2_ID); added.setAdded(List.of(ENTITY_2_ID));
boolean changed = entry.updateEntry(added); boolean changed = entry.updateEntry(added);
@ -149,6 +149,77 @@ public class PropagationArgumentEntryTest {
assertThat(entry.getRemoved()).isNull(); assertThat(entry.getRemoved()).isNull();
} }
@Test
void testUpdateEntryWhenPartitionStateRestoreAddsMissingIds() {
var restore = new PropagationArgumentEntry(List.of(ENTITY_1_ID, ENTITY_2_ID, ENTITY_3_ID));
restore.setIgnoreRemovedEntities(true);
boolean changed = entry.updateEntry(restore);
assertThat(changed).isTrue();
assertThat(entry.getEntityIds()).containsExactlyInAnyOrder(ENTITY_1_ID, ENTITY_2_ID, ENTITY_3_ID);
assertThat(entry.getAdded()).containsExactly(ENTITY_3_ID);
assertThat(entry.getRemoved()).isNull();
assertThat(entry.isIgnoreRemovedEntities()).isFalse();
}
@Test
void testUpdateEntryWhenPartitionStateRestoreRemovesStaleIds() {
var restore = new PropagationArgumentEntry(List.of(ENTITY_1_ID));
restore.setIgnoreRemovedEntities(true);
boolean changed = entry.updateEntry(restore);
assertThat(changed).isFalse(); // expected no change, since we consider the removal of stale ids as no-op
assertThat(entry.getEntityIds()).containsExactlyInAnyOrder(ENTITY_1_ID);
assertThat(entry.getAdded()).isNull();
assertThat(entry.getRemoved()).isNull();
assertThat(entry.isIgnoreRemovedEntities()).isFalse();
}
@Test
void testUpdateEntryWhenPartitionStateRestoreAddsAndRemoves() {
var restore = new PropagationArgumentEntry(List.of(ENTITY_1_ID, ENTITY_3_ID));
restore.setIgnoreRemovedEntities(true);
boolean changed = entry.updateEntry(restore);
assertThat(changed).isTrue();
assertThat(entry.getEntityIds()).containsExactlyInAnyOrder(ENTITY_1_ID, ENTITY_3_ID);
assertThat(entry.getAdded()).containsExactly(ENTITY_3_ID);
assertThat(entry.getRemoved()).isNull();
assertThat(entry.isIgnoreRemovedEntities()).isFalse();
}
@Test
void testUpdateEntryWhenPartitionStateRestoreNoChanges() {
var restore = new PropagationArgumentEntry(List.of(ENTITY_1_ID, ENTITY_2_ID));
restore.setIgnoreRemovedEntities(true);
boolean changed = entry.updateEntry(restore);
assertThat(changed).isFalse();
assertThat(entry.getEntityIds()).containsExactlyInAnyOrder(ENTITY_1_ID, ENTITY_2_ID);
assertThat(entry.getAdded()).isNull();
assertThat(entry.getRemoved()).isNull();
assertThat(entry.isIgnoreRemovedEntities()).isFalse();
}
@Test
void testUpdateEntryWhenPartitionStateRestoreEmptySet() {
var restore = new PropagationArgumentEntry(List.of());
restore.setIgnoreRemovedEntities(true);
boolean changed = entry.updateEntry(restore);
assertThat(changed).isFalse(); // expected no change, since we consider the removal of stale ids as no-op
assertThat(entry.getEntityIds()).isEmpty();
assertThat(entry.getAdded()).isNull();
assertThat(entry.getRemoved()).isNull();
assertThat(entry.isIgnoreRemovedEntities()).isFalse();
}
@Test @Test
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
void testToTbelCfArgWithValues() { void testToTbelCfArgWithValues() {

141
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java

@ -47,6 +47,7 @@ import org.thingsboard.server.common.data.relation.EntitySearchDirection;
import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.relation.RelationPathLevel;
import org.thingsboard.server.common.stats.DefaultStatsFactory; import org.thingsboard.server.common.stats.DefaultStatsFactory;
import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.dao.usagerecord.ApiLimitService;
import org.thingsboard.server.service.cf.CalculatedFieldProcessingService;
import org.thingsboard.server.service.cf.PropagationCalculatedFieldResult; import org.thingsboard.server.service.cf.PropagationCalculatedFieldResult;
import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult;
import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry;
@ -57,12 +58,17 @@ import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.stream.Stream; import java.util.stream.Stream;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import static org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration.PROPAGATION_CONFIG_ARGUMENT; import static org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration.PROPAGATION_CONFIG_ARGUMENT;
@ -70,17 +76,26 @@ import static org.thingsboard.server.common.data.cf.configuration.PropagationCal
public class PropagationCalculatedFieldStateTest { public class PropagationCalculatedFieldStateTest {
private static final String TEMPERATURE_ARGUMENT_NAME = "t"; private static final String TEMPERATURE_ARGUMENT_NAME = "t";
private static final String HUMIDITY_ARGUMENT_NAME = "h";
private static final String TEST_RESULT_EXPRESSION_KEY = "testResult"; private static final String TEST_RESULT_EXPRESSION_KEY = "testResult";
private static final double TEMPERATURE_VALUE = 12.5; private static final double TEMPERATURE_VALUE = 12.5;
private static final double HUMIDITY_VALUE = 85;
private static final PropagationArgumentEntry EMPTY_PROPAGATION_ARGUMENT = new PropagationArgumentEntry(Collections.emptyList());
private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("6c3513cb-85e7-4510-8746-1ba01859a8ce")); private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("6c3513cb-85e7-4510-8746-1ba01859a8ce"));
private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("be960a50-c029-4698-b2ec-c56a543c561c")); private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("be960a50-c029-4698-b2ec-c56a543c561c"));
private final AssetId ASSET_ID_1 = new AssetId(UUID.fromString("d26f0e5b-7d7d-4a61-9f5e-08ab97b30734")); private final AssetId ASSET_ID_1 = new AssetId(UUID.fromString("d26f0e5b-7d7d-4a61-9f5e-08ab97b30734"));
private final AssetId ASSET_ID_2 = new AssetId(UUID.fromString("1933a317-4df5-4d36-9800-68aded74579b")); private final AssetId ASSET_ID_2 = new AssetId(UUID.fromString("1933a317-4df5-4d36-9800-68aded74579b"));
private final SingleValueArgumentEntry singleValueArgEntry = private final SingleValueArgumentEntry EMPTY_SINGLE_VALUE_ARGUMENT = new SingleValueArgumentEntry();
private final SingleValueArgumentEntry temperatureArgumentEntry =
new SingleValueArgumentEntry(System.currentTimeMillis(), new DoubleDataEntry("temperature", TEMPERATURE_VALUE), 99L); new SingleValueArgumentEntry(System.currentTimeMillis(), new DoubleDataEntry("temperature", TEMPERATURE_VALUE), 99L);
private final SingleValueArgumentEntry humidityArgumentEntry =
new SingleValueArgumentEntry(System.currentTimeMillis(), new DoubleDataEntry("humidity", HUMIDITY_VALUE), 99L);
private final PropagationArgumentEntry propagationArgEntry = private final PropagationArgumentEntry propagationArgEntry =
new PropagationArgumentEntry(new ArrayList<>(List.of(ASSET_ID_2, ASSET_ID_1))); new PropagationArgumentEntry(new ArrayList<>(List.of(ASSET_ID_2, ASSET_ID_1)));
@ -96,15 +111,19 @@ public class PropagationCalculatedFieldStateTest {
@MockitoBean @MockitoBean
private ActorSystemContext actorSystemContext; private ActorSystemContext actorSystemContext;
@MockitoBean
private CalculatedFieldProcessingService cfProcessingService;
@BeforeEach @BeforeEach
void setUp() { void setUp() {
when(actorSystemContext.getTbelInvokeService()).thenReturn(tbelInvokeService); when(actorSystemContext.getTbelInvokeService()).thenReturn(tbelInvokeService);
when(actorSystemContext.getApiLimitService()).thenReturn(apiLimitService); when(actorSystemContext.getApiLimitService()).thenReturn(apiLimitService);
when(actorSystemContext.getCalculatedFieldProcessingService()).thenReturn(cfProcessingService);
when(apiLimitService.getLimit(any(), any())).thenReturn(1000L); when(apiLimitService.getLimit(any(), any())).thenReturn(1000L);
} }
void initCtxAndState(boolean applyExpressionToResolvedArguments) { void initCtxAndState(boolean applyExpressionToResolvedArguments) {
ctx = new CalculatedFieldCtx(getCalculatedField(applyExpressionToResolvedArguments), actorSystemContext); ctx = spy(new CalculatedFieldCtx(getCalculatedField(applyExpressionToResolvedArguments), actorSystemContext));
ctx.init(); ctx.init();
state = new PropagationCalculatedFieldState(ctx.getEntityId()); state = new PropagationCalculatedFieldState(ctx.getEntityId());
@ -121,7 +140,7 @@ public class PropagationCalculatedFieldStateTest {
@Test @Test
void testInitAddsRequiredArgument() { void testInitAddsRequiredArgument() {
initCtxAndState(false); initCtxAndState(false);
assertThat(state.getRequiredArguments()).containsExactlyInAnyOrder(TEMPERATURE_ARGUMENT_NAME, PROPAGATION_CONFIG_ARGUMENT); assertThat(state.getRequiredArguments()).containsExactlyInAnyOrder(TEMPERATURE_ARGUMENT_NAME, HUMIDITY_ARGUMENT_NAME, PROPAGATION_CONFIG_ARGUMENT);
} }
@Test @Test
@ -133,7 +152,7 @@ public class PropagationCalculatedFieldStateTest {
private static Stream<ArgumentEntry> provideInvalidPropagationArgs() { private static Stream<ArgumentEntry> provideInvalidPropagationArgs() {
return Stream.of( return Stream.of(
null, null,
new PropagationArgumentEntry(Collections.emptyList()) EMPTY_PROPAGATION_ARGUMENT
); );
} }
@ -143,7 +162,8 @@ public class PropagationCalculatedFieldStateTest {
initCtxAndState(false); initCtxAndState(false);
Map<String, ArgumentEntry> args = new HashMap<>(); Map<String, ArgumentEntry> args = new HashMap<>();
args.put(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry); // Valid user arg args.put(TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry); // Valid user arg
args.put(HUMIDITY_ARGUMENT_NAME, humidityArgumentEntry); // Valid user arg
if (propagationEntry != null) { if (propagationEntry != null) {
args.put(PROPAGATION_CONFIG_ARGUMENT, propagationEntry); args.put(PROPAGATION_CONFIG_ARGUMENT, propagationEntry);
@ -155,19 +175,54 @@ public class PropagationCalculatedFieldStateTest {
} }
@Test @Test
void testIsReadyWhenPropagationArgHasEntities() { void testIsReadyWithoutExpressionWhenAllArgumentsAreNotEmpty() {
initCtxAndState(false); initCtxAndState(false);
state.update(Map.of(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry, PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry), ctx); Map<String, ArgumentEntry> updatedArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, humidityArgumentEntry,
PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry
);
state.update(updatedArgs, ctx);
assertThat(state.isReady()).isTrue(); assertThat(state.isReady()).isTrue();
assertThat(state.getReadinessStatus().errorMsg()).isNull(); assertThat(state.getReadinessStatus().errorMsg()).isNull();
} }
@Test
void testIsReadyWithoutExpressionWhenAtLeastOneArgumentIsNotEmpty() {
initCtxAndState(false);
Map<String, ArgumentEntry> updatedArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, EMPTY_SINGLE_VALUE_ARGUMENT,
PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry);
state.update(updatedArgs, ctx);
assertThat(state.isReady()).isTrue();
assertThat(state.getReadinessStatus().errorMsg()).isNull();
}
@Test
void testIsNotReadyWithExpressionWhenAtLeastOneArgumentIsEmpty() {
initCtxAndState(true);
Map<String, ArgumentEntry> updatedArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, EMPTY_SINGLE_VALUE_ARGUMENT,
PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry);
state.update(updatedArgs, ctx);
assertThat(state.isReady()).isFalse();
assertThat(state.getReadinessStatus().errorMsg()).isEqualTo("Required arguments are missing: h");
}
@Test @Test
void testPerformCalculationWithEmptyPropagationArg() throws Exception { void testPerformCalculationWithEmptyPropagationArg() throws Exception {
initCtxAndState(false); initCtxAndState(false);
state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, new PropagationArgumentEntry(Collections.emptyList())); Map<String, ArgumentEntry> initArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, humidityArgumentEntry,
PROPAGATION_CONFIG_ARGUMENT, EMPTY_PROPAGATION_ARGUMENT);
state.update(initArgs, ctx);
assertThat(state.isReady()).isFalse();
// test empty propagation argument calculation
PropagationCalculatedFieldResult result = performCalculation(); PropagationCalculatedFieldResult result = performCalculation();
assertThat(result).isNotNull(); assertThat(result).isNotNull();
@ -178,8 +233,12 @@ public class PropagationCalculatedFieldStateTest {
@Test @Test
void testPerformCalculationWithArgumentsOnlyMode() throws Exception { void testPerformCalculationWithArgumentsOnlyMode() throws Exception {
initCtxAndState(false); initCtxAndState(false);
state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry); Map<String, ArgumentEntry> initArgs = Map.of(
state.getArguments().put(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry); TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, EMPTY_SINGLE_VALUE_ARGUMENT,
PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry);
state.update(initArgs, ctx);
assertThat(state.isReady()).isTrue();
PropagationCalculatedFieldResult propagationResult = performCalculation(); PropagationCalculatedFieldResult propagationResult = performCalculation();
@ -193,7 +252,7 @@ public class PropagationCalculatedFieldStateTest {
assertThat(result.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE); assertThat(result.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE);
ObjectNode expectedNode = JacksonUtil.newObjectNode(); ObjectNode expectedNode = JacksonUtil.newObjectNode();
JacksonUtil.addKvEntry(expectedNode, singleValueArgEntry.getKvEntryValue(), TEMPERATURE_ARGUMENT_NAME); JacksonUtil.addKvEntry(expectedNode, temperatureArgumentEntry.getKvEntryValue(), TEMPERATURE_ARGUMENT_NAME);
assertThat(result.getResult()).isEqualTo(expectedNode); assertThat(result.getResult()).isEqualTo(expectedNode);
} }
@ -201,9 +260,13 @@ public class PropagationCalculatedFieldStateTest {
@Test @Test
void testPerformCalculationWithExpressionResultMode() throws Exception { void testPerformCalculationWithExpressionResultMode() throws Exception {
initCtxAndState(true); initCtxAndState(true);
state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry); Map<String, ArgumentEntry> initArgs = Map.of(
state.getArguments().put(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry); TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, humidityArgumentEntry,
PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry
);
state.update(initArgs, ctx);
assertThat(state.isReady()).isTrue();
PropagationCalculatedFieldResult propagationResult = performCalculation(); PropagationCalculatedFieldResult propagationResult = performCalculation();
assertThat(propagationResult).isNotNull(); assertThat(propagationResult).isNotNull();
@ -216,7 +279,7 @@ public class PropagationCalculatedFieldStateTest {
assertThat(result.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE); assertThat(result.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE);
ObjectNode expectedNode = JacksonUtil.newObjectNode(); ObjectNode expectedNode = JacksonUtil.newObjectNode();
expectedNode.put(TEST_RESULT_EXPRESSION_KEY, TEMPERATURE_VALUE * 2); expectedNode.put(TEST_RESULT_EXPRESSION_KEY, (TEMPERATURE_VALUE + HUMIDITY_VALUE) / 2);
assertThat(result.getResult()).isEqualTo(expectedNode); assertThat(result.getResult()).isEqualTo(expectedNode);
} }
@ -224,18 +287,52 @@ public class PropagationCalculatedFieldStateTest {
@Test @Test
void testPropagationWithUpdatedPropagationArgument() throws ExecutionException, InterruptedException { void testPropagationWithUpdatedPropagationArgument() throws ExecutionException, InterruptedException {
initCtxAndState(false); initCtxAndState(false);
state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry); Map<String, ArgumentEntry> initArgs = Map.of(
state.getArguments().put(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry); TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, EMPTY_SINGLE_VALUE_ARGUMENT,
PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry
);
state.update(initArgs, ctx);
assertThat(state.isReady()).isTrue();
AssetId newEntityId = new AssetId(UUID.fromString("83e2c962-eeae-4708-984e-e6a24760f9c3")); AssetId newEntityId = new AssetId(UUID.fromString("83e2c962-eeae-4708-984e-e6a24760f9c3"));
PropagationArgumentEntry propagationArgumentEntry = new PropagationArgumentEntry(); PropagationArgumentEntry propagationArgumentEntry = new PropagationArgumentEntry();
propagationArgumentEntry.setAdded(newEntityId); propagationArgumentEntry.setAdded(List.of(newEntityId));
Map<String, ArgumentEntry> updated = state.update(Map.of(PROPAGATION_CONFIG_ARGUMENT, propagationArgumentEntry), ctx); Map<String, ArgumentEntry> updated = state.update(Map.of(PROPAGATION_CONFIG_ARGUMENT, propagationArgumentEntry), ctx);
assertThat(updated).isNotNull().containsEntry(PROPAGATION_CONFIG_ARGUMENT, propagationArgumentEntry); assertThat(updated).isNotNull().containsEntry(PROPAGATION_CONFIG_ARGUMENT, propagationArgumentEntry);
PropagationCalculatedFieldResult propagationCalculatedFieldResult = performCalculation(updated); PropagationCalculatedFieldResult propagationCalculatedFieldResult = performCalculation(updated);
assertThat(propagationCalculatedFieldResult).isNotNull(); assertThat(propagationCalculatedFieldResult).isNotNull();
assertThat(propagationCalculatedFieldResult.getEntityIds()).isNotNull().containsExactly(newEntityId); assertThat(propagationCalculatedFieldResult.getEntityIds()).isNotNull().containsExactly(newEntityId);
assertThat(propagationCalculatedFieldResult.getResult()).isNotNull();
assertThat(propagationCalculatedFieldResult.getResult().getResult()).isNotNull();
assertThat(propagationCalculatedFieldResult.getResult().getResult()).isEqualTo(JacksonUtil.newObjectNode().put(TEMPERATURE_ARGUMENT_NAME, TEMPERATURE_VALUE));
}
@Test
void testPropapagationStateInitWithRestoredSetToFalse() {
initCtxAndState(false);
verify(cfProcessingService, never()).fetchPropagationArgumentFromDb(any(), any());
verify(ctx, never()).scheduleReevaluation(anyLong(), any());
}
@Test
void testPropapagationStateInitWithRestoredSetToTrue() {
initCtxAndState(false);
Map<String, ArgumentEntry> initArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, humidityArgumentEntry,
PROPAGATION_CONFIG_ARGUMENT, new PropagationArgumentEntry(Collections.emptyList())
);
state.update(initArgs, ctx);
assertThat(state.isReady()).isFalse();
when(cfProcessingService.fetchPropagationArgumentFromDb(any(), any())).thenReturn(Optional.of(propagationArgEntry));
state.init(true);
verify(cfProcessingService).fetchPropagationArgumentFromDb(ctx, state.getEntityId());
verify(ctx).scheduleReevaluation(0L, state.getActorCtx());
} }
private CalculatedField getCalculatedField(boolean applyExpressionToResolvedArguments) { private CalculatedField getCalculatedField(boolean applyExpressionToResolvedArguments) {
@ -260,8 +357,12 @@ public class PropagationCalculatedFieldStateTest {
ReferencedEntityKey tempKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); ReferencedEntityKey tempKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null);
temperatureArg.setRefEntityKey(tempKey); temperatureArg.setRefEntityKey(tempKey);
config.setArguments(Map.of(TEMPERATURE_ARGUMENT_NAME, temperatureArg)); Argument humidityArg = new Argument();
config.setExpression("{" + TEST_RESULT_EXPRESSION_KEY + ": " + TEMPERATURE_ARGUMENT_NAME + " * 2}"); ReferencedEntityKey humidityKey = new ReferencedEntityKey("humidity", ArgumentType.TS_LATEST, null);
humidityArg.setRefEntityKey(humidityKey);
config.setArguments(Map.of(TEMPERATURE_ARGUMENT_NAME, temperatureArg, HUMIDITY_ARGUMENT_NAME, humidityArg));
config.setExpression("return { " + TEST_RESULT_EXPRESSION_KEY + ": (" + TEMPERATURE_ARGUMENT_NAME + " + " + HUMIDITY_ARGUMENT_NAME + ") / 2};");
AttributesOutput output = new AttributesOutput(); AttributesOutput output = new AttributesOutput();
output.setScope(AttributeScope.SERVER_SCOPE); output.setScope(AttributeScope.SERVER_SCOPE);

133
application/src/test/java/org/thingsboard/server/service/edge/EdgeStatsTest.java

@ -21,6 +21,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor; import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.test.util.ReflectionTestUtils; import org.springframework.test.util.ReflectionTestUtils;
@ -44,9 +45,11 @@ import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_ADDED; import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_ADDED;
@ -58,6 +61,16 @@ import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_T
@ExtendWith(MockitoExtension.class) @ExtendWith(MockitoExtension.class)
public class EdgeStatsTest { public class EdgeStatsTest {
private static final int TTL_DAYS = 30;
private static final long REPORT_INTERVAL_MILLIS = 600_000L;
private static final long EXPECTED_MSGS_ADDED = 5L;
private static final long EXPECTED_MSGS_PUSHED = 3L;
private static final long EXPECTED_MSGS_PERMANENTLY_FAILED = 1L;
private static final long EXPECTED_MSGS_TMP_FAILED = 0L;
private static final long EXPECTED_MSGS_LAG = 10L;
private static final long EXPECTED_MSGS_KAFKA_LAG = 15L;
@Mock @Mock
private TimeseriesService tsService; private TimeseriesService tsService;
@Mock @Mock
@ -66,108 +79,100 @@ public class EdgeStatsTest {
private EdgeStatsCounterService statsCounterService; private EdgeStatsCounterService statsCounterService;
private EdgeStatsService edgeStatsService; private EdgeStatsService edgeStatsService;
@Captor
private ArgumentCaptor<List<TsKvEntry>> captor;
private final TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); private final TenantId tenantId = TenantId.fromUUID(UUID.randomUUID());
private final EdgeId edgeId = new EdgeId(UUID.randomUUID()); private final EdgeId edgeId = new EdgeId(UUID.randomUUID());
@BeforeEach @BeforeEach
void setUp() { void setUp() {
edgeStatsService = new EdgeStatsService( edgeStatsService = createEdgeStatsService(Optional.empty());
}
private EdgeStatsService createEdgeStatsService(Optional<KafkaAdmin> kafkaAdmin) {
EdgeStatsService service = new EdgeStatsService(
tsService, tsService,
statsCounterService, statsCounterService,
topicService, topicService,
Optional.empty() kafkaAdmin
); );
ReflectionTestUtils.setField(service, "edgesStatsTtlDays", TTL_DAYS);
ReflectionTestUtils.setField(edgeStatsService, "edgesStatsTtlDays", 30); ReflectionTestUtils.setField(service, "reportIntervalMillis", REPORT_INTERVAL_MILLIS);
ReflectionTestUtils.setField(edgeStatsService, "reportIntervalMillis", 600_000L); return service;
} }
@Test @Test
public void testReportStatsSavesTelemetry() { public void testReportStatsSavesTelemetry() {
// given // GIVEN
setupCounters();
// WHEN
edgeStatsService.reportStats();
// THEN
Map<String, Long> counters = verifyCounters();
Assertions.assertEquals(EXPECTED_MSGS_LAG, counters.get(DOWNLINK_MSGS_LAG.getKey()).longValue());
}
@Test
public void testReportStatsWithKafkaLag() {
// GIVEN
setupCounters();
setupKafkaLag();
// WHEN
edgeStatsService.reportStats();
// THEN
Map<String, Long> valuesByKey = verifyCounters();
Assertions.assertEquals(EXPECTED_MSGS_KAFKA_LAG, valuesByKey.get(DOWNLINK_MSGS_LAG.getKey()));
}
private void setupCounters() {
MsgCounters counters = new MsgCounters(tenantId); MsgCounters counters = new MsgCounters(tenantId);
counters.getMsgsAdded().set(5); counters.getMsgsAdded().set(EXPECTED_MSGS_ADDED);
counters.getMsgsPushed().set(3); counters.getMsgsPushed().set(EXPECTED_MSGS_PUSHED);
counters.getMsgsPermanentlyFailed().set(1); counters.getMsgsPermanentlyFailed().set(EXPECTED_MSGS_PERMANENTLY_FAILED);
counters.getMsgsTmpFailed().set(0); counters.getMsgsTmpFailed().set(EXPECTED_MSGS_TMP_FAILED);
counters.getMsgsLag().set(10); counters.getMsgsLag().set(EXPECTED_MSGS_LAG);
ConcurrentHashMap<EdgeId, MsgCounters> countersByEdge = new ConcurrentHashMap<>(); ConcurrentHashMap<EdgeId, MsgCounters> countersByEdge = new ConcurrentHashMap<>();
countersByEdge.put(edgeId, counters); countersByEdge.put(edgeId, counters);
when(statsCounterService.getCounterByEdge()).thenReturn(countersByEdge); when(statsCounterService.getMsgCountersByEdge()).thenReturn(countersByEdge);
ArgumentCaptor<List<TsKvEntry>> captor = ArgumentCaptor.forClass(List.class);
when(tsService.save(eq(tenantId), eq(edgeId), captor.capture(), anyLong())) when(tsService.save(eq(tenantId), eq(edgeId), captor.capture(), anyLong()))
.thenReturn(Futures.immediateFuture(mock(TimeseriesSaveResult.class))); .thenReturn(Futures.immediateFuture(mock(TimeseriesSaveResult.class)));
}
// when private Map<String, Long> verifyCounters() {
edgeStatsService.reportStats(); verify(tsService, times(1)).save(eq(tenantId), eq(edgeId), anyList(), anyLong());
verify(statsCounterService, times(1)).clear(edgeId);
// then
List<TsKvEntry> entries = captor.getValue(); List<TsKvEntry> entries = captor.getValue();
Assertions.assertEquals(5, entries.size()); Assertions.assertEquals(5, entries.size());
Map<String, Long> valuesByKey = entries.stream() Map<String, Long> valuesByKey = entries.stream()
.collect(Collectors.toMap(TsKvEntry::getKey, e -> e.getLongValue().orElse(-1L))); .collect(Collectors.toMap(TsKvEntry::getKey, e -> e.getLongValue().orElse(-1L)));
Assertions.assertEquals(5L, valuesByKey.get(DOWNLINK_MSGS_ADDED.getKey()).longValue()); Assertions.assertEquals(EXPECTED_MSGS_ADDED, valuesByKey.get(DOWNLINK_MSGS_ADDED.getKey()).longValue());
Assertions.assertEquals(3L, valuesByKey.get(DOWNLINK_MSGS_PUSHED.getKey()).longValue()); Assertions.assertEquals(EXPECTED_MSGS_PUSHED, valuesByKey.get(DOWNLINK_MSGS_PUSHED.getKey()).longValue());
Assertions.assertEquals(1L, valuesByKey.get(DOWNLINK_MSGS_PERMANENTLY_FAILED.getKey()).longValue()); Assertions.assertEquals(EXPECTED_MSGS_PERMANENTLY_FAILED, valuesByKey.get(DOWNLINK_MSGS_PERMANENTLY_FAILED.getKey()).longValue());
Assertions.assertEquals(0L, valuesByKey.get(DOWNLINK_MSGS_TMP_FAILED.getKey()).longValue()); Assertions.assertEquals(EXPECTED_MSGS_TMP_FAILED, valuesByKey.get(DOWNLINK_MSGS_TMP_FAILED.getKey()).longValue());
Assertions.assertEquals(10L, valuesByKey.get(DOWNLINK_MSGS_LAG.getKey()).longValue()); return valuesByKey;
verify(statsCounterService).clear(edgeId);
} }
@Test private void setupKafkaLag() {
public void testReportStatsWithKafkaLag() {
// given
MsgCounters counters = new MsgCounters(tenantId);
counters.getMsgsAdded().set(2);
counters.getMsgsPushed().set(2);
counters.getMsgsPermanentlyFailed().set(0);
counters.getMsgsTmpFailed().set(1);
counters.getMsgsLag().set(0);
ConcurrentHashMap<EdgeId, MsgCounters> countersByEdge = new ConcurrentHashMap<>();
countersByEdge.put(edgeId, counters);
// mocks
when(statsCounterService.getCounterByEdge()).thenReturn(countersByEdge);
String topic = "edge-topic"; String topic = "edge-topic";
TopicPartitionInfo partitionInfo = new TopicPartitionInfo(topic, tenantId, 0, false); TopicPartitionInfo partitionInfo = new TopicPartitionInfo(topic, tenantId, 0, false);
when(topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId)).thenReturn(partitionInfo); when(topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId)).thenReturn(partitionInfo);
KafkaAdmin kafkaAdmin = mock(KafkaAdmin.class); KafkaAdmin kafkaAdmin = mock(KafkaAdmin.class);
when(kafkaAdmin.getTotalLagForGroupsBulk(Set.of(topic))) when(kafkaAdmin.getTotalLagForGroupsBulk(Set.of(topic)))
.thenReturn(Map.of(topic, 15L)); .thenReturn(Map.of(topic, EXPECTED_MSGS_KAFKA_LAG));
ArgumentCaptor<List<TsKvEntry>> captor = ArgumentCaptor.forClass(List.class);
when(tsService.save(eq(tenantId), eq(edgeId), captor.capture(), anyLong()))
.thenReturn(Futures.immediateFuture(mock(TimeseriesSaveResult.class)));
edgeStatsService = new EdgeStatsService(
tsService,
statsCounterService,
topicService,
Optional.of(kafkaAdmin)
);
ReflectionTestUtils.setField(edgeStatsService, "edgesStatsTtlDays", 30);
ReflectionTestUtils.setField(edgeStatsService, "reportIntervalMillis", 600_000L);
// when
edgeStatsService.reportStats();
// then
List<TsKvEntry> entries = captor.getValue();
Map<String, Long> valuesByKey = entries.stream()
.collect(Collectors.toMap(TsKvEntry::getKey, e -> e.getLongValue().orElse(-1L)));
Assertions.assertEquals(15L, valuesByKey.get(DOWNLINK_MSGS_LAG.getKey())); edgeStatsService = createEdgeStatsService(Optional.of(kafkaAdmin));
verify(statsCounterService).clear(edgeId);
} }
} }

2
application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java

@ -60,7 +60,7 @@ import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.controller.AbstractControllerTest;
import org.thingsboard.server.dao.ai.AiModelService; import org.thingsboard.server.dao.ai.AiModelService;
import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.dashboard.DashboardService;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;

62
application/src/test/java/org/thingsboard/server/service/security/auth/pat/ApiKeyAuthenticationProviderTest.java

@ -15,18 +15,24 @@
*/ */
package org.thingsboard.server.service.security.auth.pat; package org.thingsboard.server.service.security.auth.pat;
import org.apache.commons.lang3.RandomStringUtils;
import org.junit.After; import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.mockito.Mockito; import org.mockito.Mockito;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.pat.ApiKey; import org.thingsboard.server.common.data.pat.ApiKey;
import org.thingsboard.server.common.data.pat.ApiKeyInfo; import org.thingsboard.server.common.data.pat.ApiKeyInfo;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.controller.AbstractControllerTest;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.concurrent.TimeUnit;
import static org.awaitility.Awaitility.await;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID;
@ -101,6 +107,62 @@ public class ApiKeyAuthenticationProviderTest extends AbstractControllerTest {
doGetWithApiKey("/api/admin/featuresInfo").andExpect(status().isUnauthorized()); doGetWithApiKey("/api/admin/featuresInfo").andExpect(status().isUnauthorized());
} }
@Test
public void testUnauthorizedWhenUserCredentialsDisabled() throws Exception {
User newUser = new User();
newUser.setAuthority(Authority.TENANT_ADMIN);
newUser.setTenantId(tenantId);
newUser.setEmail("testUser" + RandomStringUtils.secure().nextAlphanumeric(10) + "@thingsboard.org");
newUser.setFirstName("Test");
newUser.setLastName("User");
User savedUser = createUser(newUser, "testPassword1");
ApiKeyInfo apiKeyInfo = new ApiKeyInfo();
apiKeyInfo.setDescription("Test API key for user credentials test");
apiKeyInfo.setEnabled(true);
apiKeyInfo.setUserId(savedUser.getId());
ApiKey testApiKey = doPost("/api/apiKey", apiKeyInfo, ApiKey.class);
setApiKey(testApiKey.getValue());
doGetWithApiKey("/api/admin/repositorySettings/exists").andExpect(status().isOk());
doPost("/api/user/" + savedUser.getId().getId() + "/userCredentialsEnabled?userCredentialsEnabled=false").andExpect(status().isOk());
await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> doGetWithApiKey("/api/admin/repositorySettings/exists").andExpect(status().isUnauthorized()));
resetApiKey();
doDelete("/api/apiKey/" + testApiKey.getId()).andExpect(status().isOk());
loginSysAdmin();
doDelete("/api/user/" + savedUser.getId().getId()).andExpect(status().isOk());
}
@Test
public void testUnauthorizedWhenUserDeleted() throws Exception {
User newUser = new User();
newUser.setAuthority(Authority.TENANT_ADMIN);
newUser.setTenantId(tenantId);
newUser.setEmail("testUser" + RandomStringUtils.secure().nextAlphanumeric(10) + "@thingsboard.org");
newUser.setFirstName("Test");
newUser.setLastName("User");
User savedUser = createUser(newUser, "testPassword1");
ApiKeyInfo apiKeyInfo = new ApiKeyInfo();
apiKeyInfo.setDescription("Test API key for user deletion test");
apiKeyInfo.setEnabled(true);
apiKeyInfo.setUserId(savedUser.getId());
ApiKey testApiKey = doPost("/api/apiKey", apiKeyInfo, ApiKey.class);
setApiKey(testApiKey.getValue());
doGetWithApiKey("/api/admin/repositorySettings/exists").andExpect(status().isOk());
loginSysAdmin();
doDelete("/api/user/" + savedUser.getId().getId()).andExpect(status().isOk());
await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> doGetWithApiKey("/api/admin/repositorySettings/exists").andExpect(status().isUnauthorized()));
}
private ApiKeyInfo constructApiKeyInfo() { private ApiKeyInfo constructApiKeyInfo() {
ApiKeyInfo apiKeyInfo = new ApiKeyInfo(); ApiKeyInfo apiKeyInfo = new ApiKeyInfo();
apiKeyInfo.setDescription("New API key description"); apiKeyInfo.setDescription("New API key description");

5
application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java

@ -119,7 +119,7 @@ class CalculatedFieldUtilsTest {
} }
@Test @Test
void toProtoAndFromProto_shouldCreatePropagationStateWithEmptyPropagationArgument() { void toProtoAndFromProto_shouldCreatePropagationStateWithNotEmptyPropagationArgument() {
// given // given
CalculatedFieldEntityCtxId stateId = mock(CalculatedFieldEntityCtxId.class); CalculatedFieldEntityCtxId stateId = mock(CalculatedFieldEntityCtxId.class);
given(stateId.tenantId()).willReturn(TENANT_ID); given(stateId.tenantId()).willReturn(TENANT_ID);
@ -158,8 +158,7 @@ class CalculatedFieldUtilsTest {
assertThat(propagationState.getEntityId()).isEqualTo(DEVICE_ID); assertThat(propagationState.getEntityId()).isEqualTo(DEVICE_ID);
assertThat(propagationState.getArguments()).isNotNull(); assertThat(propagationState.getArguments()).isNotNull();
assertThat(propagationState.getArguments().get(PROPAGATION_CONFIG_ARGUMENT)).isNotNull(); assertThat(propagationState.getArguments().get(PROPAGATION_CONFIG_ARGUMENT)).isEqualTo(propagationArgumentEntry);
assertThat(propagationState.getArguments().get(PROPAGATION_CONFIG_ARGUMENT).isEmpty()).isTrue();
assertThat(propagationState.getArguments().get("state")).isNotNull().isEqualTo(singleValueArgumentEntry); assertThat(propagationState.getArguments().get("state")).isNotNull().isEqualTo(singleValueArgumentEntry);
assertThat(propagationState.getRequiredArguments()).isNull(); assertThat(propagationState.getRequiredArguments()).isNull();
assertThat(propagationState.getReadinessStatus()).isNull(); assertThat(propagationState.getReadinessStatus()).isNull();

14
common/data/src/main/java/org/thingsboard/server/common/data/device/profile/Lwm2mDeviceProfileTransportConfiguration.java

@ -17,10 +17,15 @@ package org.thingsboard.server.common.data.device.profile;
import lombok.Data; import lombok.Data;
import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.DeviceTransportType;
import org.thingsboard.server.common.data.device.data.PowerMode;
import org.thingsboard.server.common.data.device.profile.lwm2m.OtherConfiguration; import org.thingsboard.server.common.data.device.profile.lwm2m.OtherConfiguration;
import org.thingsboard.server.common.data.device.profile.lwm2m.TelemetryMappingConfiguration; import org.thingsboard.server.common.data.device.profile.lwm2m.TelemetryMappingConfiguration;
import org.thingsboard.server.common.data.device.profile.lwm2m.bootstrap.LwM2MBootstrapServerCredential; import org.thingsboard.server.common.data.device.profile.lwm2m.bootstrap.LwM2MBootstrapServerCredential;
import static org.eclipse.leshan.core.LwM2m.Version.V1_0;
import static org.thingsboard.server.common.data.device.profile.lwm2m.TelemetryObserveStrategy.SINGLE;
import java.util.Collections;
import java.util.List; import java.util.List;
@Data @Data
@ -33,9 +38,18 @@ public class Lwm2mDeviceProfileTransportConfiguration implements DeviceProfileTr
private List<LwM2MBootstrapServerCredential> bootstrap; private List<LwM2MBootstrapServerCredential> bootstrap;
private OtherConfiguration clientLwM2mSettings; private OtherConfiguration clientLwM2mSettings;
public Lwm2mDeviceProfileTransportConfiguration() {
updateDefault();
}
@Override @Override
public DeviceTransportType getType() { public DeviceTransportType getType() {
return DeviceTransportType.LWM2M; return DeviceTransportType.LWM2M;
} }
private void updateDefault(){
this.setBootstrap(Collections.emptyList());
this.setClientLwM2mSettings(new OtherConfiguration(false,1, 1, 1, PowerMode.DRX, null, null, null, null, null, V1_0.toString()));
this.setObserveAttr(new TelemetryMappingConfiguration(Collections.emptyMap(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptyMap(), false, SINGLE));
}
} }

5
dao/src/main/java/org/thingsboard/server/dao/exception/DataValidationException.java → common/data/src/main/java/org/thingsboard/server/exception/DataValidationException.java

@ -13,12 +13,10 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.thingsboard.server.dao.exception; package org.thingsboard.server.exception;
public class DataValidationException extends RuntimeException { public class DataValidationException extends RuntimeException {
private static final long serialVersionUID = 7659985660312721830L;
public DataValidationException(String message) { public DataValidationException(String message) {
super(message); super(message);
} }
@ -26,4 +24,5 @@ public class DataValidationException extends RuntimeException {
public DataValidationException(String message, Throwable cause) { public DataValidationException(String message, Throwable cause) {
super(message, cause); super(message, cause);
} }
} }

4
dao/src/main/java/org/thingsboard/server/dao/exception/EntitiesLimitExceededException.java → common/data/src/main/java/org/thingsboard/server/exception/EntitiesLimitExceededException.java

@ -13,14 +13,13 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.thingsboard.server.dao.exception; package org.thingsboard.server.exception;
import lombok.Getter; import lombok.Getter;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
public class EntitiesLimitExceededException extends DataValidationException { public class EntitiesLimitExceededException extends DataValidationException {
private static final long serialVersionUID = -9211462514373279196L;
@Getter @Getter
private final TenantId tenantId; private final TenantId tenantId;
@ -36,4 +35,5 @@ public class EntitiesLimitExceededException extends DataValidationException {
this.entityType = entityType; this.entityType = entityType;
this.limit = limit; this.limit = limit;
} }
} }

51
common/data/src/main/java/org/thingsboard/server/exception/ThingsboardRuntimeException.java

@ -0,0 +1,51 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.exception;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
public class ThingsboardRuntimeException extends RuntimeException {
private ThingsboardErrorCode errorCode;
public ThingsboardRuntimeException() {
super();
}
public ThingsboardRuntimeException(ThingsboardErrorCode errorCode) {
this.errorCode = errorCode;
}
public ThingsboardRuntimeException(String message, ThingsboardErrorCode errorCode) {
super(message);
this.errorCode = errorCode;
}
public ThingsboardRuntimeException(String message, Throwable cause, ThingsboardErrorCode errorCode) {
super(message, cause);
this.errorCode = errorCode;
}
public ThingsboardRuntimeException(Throwable cause, ThingsboardErrorCode errorCode) {
super(cause);
this.errorCode = errorCode;
}
public ThingsboardErrorCode getErrorCode() {
return errorCode;
}
}

1
common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeRpcClient.java

@ -41,4 +41,5 @@ public interface EdgeRpcClient {
void sendDownlinkResponseMsg(DownlinkResponseMsg downlinkResponseMsg); void sendDownlinkResponseMsg(DownlinkResponseMsg downlinkResponseMsg);
int getServerMaxInboundMessageSize(); int getServerMaxInboundMessageSize();
} }

1
common/proto/src/main/proto/queue.proto

@ -935,6 +935,7 @@ message CalculatedFieldStateProto {
int64 lastArgsUpdateTs = 7; int64 lastArgsUpdateTs = 7;
int64 lastMetricsEvalTs = 8; int64 lastMetricsEvalTs = 8;
repeated ArgumentIntervalProto aggregationArguments = 9; repeated ArgumentIntervalProto aggregationArguments = 9;
repeated EntityIdProto propagationEntityIds = 10;
} }
//Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level.

29
common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaAdmin.java

@ -58,15 +58,12 @@ public class KafkaAdmin {
private final TbKafkaSettings settings; private final TbKafkaSettings settings;
@Value("${queue.kafka.request.timeout.ms:30000}")
private int requestTimeoutMs;
@Value("${queue.kafka.topics_cache_ttl_ms:300000}") // 5 minutes by default
private int topicsCacheTtlMs;
private final LazyInitializer<AdminClient> adminClient; private final LazyInitializer<AdminClient> adminClient;
private final CachedValue<Set<String>> topics; private final CachedValue<Set<String>> topics;
public KafkaAdmin(@Lazy TbKafkaSettings settings) { public KafkaAdmin(@Lazy TbKafkaSettings settings,
@Value("${queue.kafka.topics_cache_ttl_ms:300000}")
int topicsCacheTtlMs) {
this.settings = settings; this.settings = settings;
this.adminClient = LazyInitializer.<AdminClient>builder() this.adminClient = LazyInitializer.<AdminClient>builder()
.setInitializer(() -> AdminClient.create(settings.toAdminProps())) .setInitializer(() -> AdminClient.create(settings.toAdminProps()))
@ -91,7 +88,7 @@ public class KafkaAdmin {
NewTopic newTopic = new NewTopic(topic, partitions, settings.getReplicationFactor()).configs(properties); NewTopic newTopic = new NewTopic(topic, partitions, settings.getReplicationFactor()).configs(properties);
try { try {
getClient().createTopics(List.of(newTopic)).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS); getClient().createTopics(List.of(newTopic)).all().get(settings.getRequestTimeoutMs(), TimeUnit.MILLISECONDS);
topics.add(topic); topics.add(topic);
} catch (ExecutionException ee) { } catch (ExecutionException ee) {
log.trace("Failed to create topic {} with properties {}", topic, properties, ee); log.trace("Failed to create topic {} with properties {}", topic, properties, ee);
@ -110,7 +107,7 @@ public class KafkaAdmin {
public void deleteTopic(String topic) { public void deleteTopic(String topic) {
log.debug("Deleting topic {}", topic); log.debug("Deleting topic {}", topic);
try { try {
getClient().deleteTopics(List.of(topic)).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS); getClient().deleteTopics(List.of(topic)).all().get(settings.getRequestTimeoutMs(), TimeUnit.MILLISECONDS);
} catch (Exception e) { } catch (Exception e) {
log.error("Failed to delete kafka topic [{}].", topic, e); log.error("Failed to delete kafka topic [{}].", topic, e);
} }
@ -122,7 +119,7 @@ public class KafkaAdmin {
public Set<String> listTopics() { public Set<String> listTopics() {
try { try {
Set<String> topics = getClient().listTopics().names().get(requestTimeoutMs, TimeUnit.MILLISECONDS); Set<String> topics = getClient().listTopics().names().get(settings.getRequestTimeoutMs(), TimeUnit.MILLISECONDS);
log.trace("Listed topics: {}", topics); log.trace("Listed topics: {}", topics);
return topics; return topics;
} catch (Exception e) { } catch (Exception e) {
@ -150,7 +147,7 @@ public class KafkaAdmin {
.collect(Collectors.toMap(tp -> tp, tp -> OffsetSpec.latest())); .collect(Collectors.toMap(tp -> tp, tp -> OffsetSpec.latest()));
Map<TopicPartition, ListOffsetsResult.ListOffsetsResultInfo> endOffsets = Map<TopicPartition, ListOffsetsResult.ListOffsetsResultInfo> endOffsets =
getClient().listOffsets(latestOffsetsSpec).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS); getClient().listOffsets(latestOffsetsSpec).all().get(settings.getRequestTimeoutMs(), TimeUnit.MILLISECONDS);
return committedOffsets.entrySet().stream() return committedOffsets.entrySet().stream()
.mapToLong(entry -> { .mapToLong(entry -> {
@ -169,7 +166,7 @@ public class KafkaAdmin {
@SneakyThrows @SneakyThrows
public Map<TopicPartition, OffsetAndMetadata> getConsumerGroupOffsets(String groupId) { public Map<TopicPartition, OffsetAndMetadata> getConsumerGroupOffsets(String groupId) {
return getClient().listConsumerGroupOffsets(groupId).partitionsToOffsetAndMetadata().get(requestTimeoutMs, TimeUnit.MILLISECONDS); return getClient().listConsumerGroupOffsets(groupId).partitionsToOffsetAndMetadata().get(settings.getRequestTimeoutMs(), TimeUnit.MILLISECONDS);
} }
/** /**
@ -212,7 +209,7 @@ public class KafkaAdmin {
} else { } else {
log.info("[{}] SHOULD alter topic offset [{}] less than old node group offset [{}]", tp, existingOffset.offset(), om.offset()); log.info("[{}] SHOULD alter topic offset [{}] less than old node group offset [{}]", tp, existingOffset.offset(), om.offset());
} }
getClient().alterConsumerGroupOffsets(newGroupId, Map.of(tp, om)).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS); getClient().alterConsumerGroupOffsets(newGroupId, Map.of(tp, om)).all().get(settings.getRequestTimeoutMs(), TimeUnit.MILLISECONDS);
log.info("[{}] altered new consumer groupId {}", tp, newGroupId); log.info("[{}] altered new consumer groupId {}", tp, newGroupId);
break; break;
} }
@ -229,7 +226,7 @@ public class KafkaAdmin {
return true; return true;
} }
List<TopicPartition> allPartitions = getClient().describeTopics(existingTopics).allTopicNames().get(requestTimeoutMs, TimeUnit.MILLISECONDS) List<TopicPartition> allPartitions = getClient().describeTopics(existingTopics).allTopicNames().get(settings.getRequestTimeoutMs(), TimeUnit.MILLISECONDS)
.entrySet().stream() .entrySet().stream()
.flatMap(entry -> { .flatMap(entry -> {
String topic = entry.getKey(); String topic = entry.getKey();
@ -239,9 +236,9 @@ public class KafkaAdmin {
.toList(); .toList();
Map<TopicPartition, ListOffsetsResult.ListOffsetsResultInfo> beginningOffsets = getClient().listOffsets(allPartitions.stream() Map<TopicPartition, ListOffsetsResult.ListOffsetsResultInfo> beginningOffsets = getClient().listOffsets(allPartitions.stream()
.collect(Collectors.toMap(partition -> partition, partition -> OffsetSpec.earliest()))).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS); .collect(Collectors.toMap(partition -> partition, partition -> OffsetSpec.earliest()))).all().get(settings.getRequestTimeoutMs(), TimeUnit.MILLISECONDS);
Map<TopicPartition, ListOffsetsResult.ListOffsetsResultInfo> endOffsets = getClient().listOffsets(allPartitions.stream() Map<TopicPartition, ListOffsetsResult.ListOffsetsResultInfo> endOffsets = getClient().listOffsets(allPartitions.stream()
.collect(Collectors.toMap(partition -> partition, partition -> OffsetSpec.latest()))).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS); .collect(Collectors.toMap(partition -> partition, partition -> OffsetSpec.latest()))).all().get(settings.getRequestTimeoutMs(), TimeUnit.MILLISECONDS);
for (TopicPartition partition : allPartitions) { for (TopicPartition partition : allPartitions) {
long beginningOffset = beginningOffsets.get(partition).offset(); long beginningOffset = beginningOffsets.get(partition).offset();
@ -261,7 +258,7 @@ public class KafkaAdmin {
public void deleteConsumerGroup(String consumerGroupId) { public void deleteConsumerGroup(String consumerGroupId) {
try { try {
getClient().deleteConsumerGroups(List.of(consumerGroupId)).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS); getClient().deleteConsumerGroups(List.of(consumerGroupId)).all().get(settings.getRequestTimeoutMs(), TimeUnit.MILLISECONDS);
} catch (Exception e) { } catch (Exception e) {
log.warn("Failed to delete consumer group {}", consumerGroupId, e); log.warn("Failed to delete consumer group {}", consumerGroupId, e);
} }

1
common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java

@ -112,6 +112,7 @@ public class TbKafkaSettings {
@Value("${queue.kafka.fetch_max_bytes:134217728}") @Value("${queue.kafka.fetch_max_bytes:134217728}")
private int fetchMaxBytes; private int fetchMaxBytes;
@Getter
@Value("${queue.kafka.request.timeout.ms:30000}") @Value("${queue.kafka.request.timeout.ms:30000}")
private int requestTimeoutMs; private int requestTimeoutMs;

25
common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClientContextImpl.java

@ -360,20 +360,25 @@ public class LwM2mClientContextImpl implements LwM2mClientContext {
@Override @Override
public Lwm2mDeviceProfileTransportConfiguration getProfile(Registration registration) { public Lwm2mDeviceProfileTransportConfiguration getProfile(Registration registration) {
UUID profileId = getClientByEndpoint(registration.getEndpoint()).getProfileId(); UUID profileId = getClientByEndpoint(registration.getEndpoint()).getProfileId();
return doGetAndCache(profileId); return profileId != null ? doGetAndCache(profileId) : null;
} }
private Lwm2mDeviceProfileTransportConfiguration doGetAndCache(UUID profileId) { private Lwm2mDeviceProfileTransportConfiguration doGetAndCache(UUID profileId) {
Lwm2mDeviceProfileTransportConfiguration result;
Lwm2mDeviceProfileTransportConfiguration result = profiles.get(profileId); if (profileId != null) {
if (result == null) { result = profiles.get(profileId);
log.debug("Fetching profile [{}]", profileId); if (result == null) {
DeviceProfile deviceProfile = deviceProfileCache.get(new DeviceProfileId(profileId)); log.debug("Fetching profile [{}]", profileId);
if (deviceProfile != null) { DeviceProfile deviceProfile = deviceProfileCache.get(new DeviceProfileId(profileId));
result = profileUpdate(deviceProfile); if (deviceProfile != null) {
} else { result = profileUpdate(deviceProfile);
log.warn("Device profile was not found! Most probably device profile [{}] has been removed from the database.", profileId); } else {
log.warn("Device profile was not found! Most probably device profile [{}] has been removed from the database.", profileId);
}
} }
} else {
log.warn("Device profile not found! The device profile ID is null. Return Lwm2mDeviceProfileTransportConfiguration with default.");
result = new Lwm2mDeviceProfileTransportConfiguration();
} }
return result; return result;
} }

58
common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java

@ -201,35 +201,39 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl
} }
var clientSettings = clientContext.getProfile(client.getRegistration()).getClientLwM2mSettings(); var clientSettings = clientContext.getProfile(client.getRegistration()).getClientLwM2mSettings();
initFwStrategy(client, clientSettings); if (clientSettings != null) {
initSwStrategy(client, clientSettings); initFwStrategy(client, clientSettings);
initSwStrategy(client, clientSettings);
if (!attributesToFetch.isEmpty()) {
var future = attributesService.getSharedAttributes(client, attributesToFetch);
DonAsynchron.withCallback(future, attrs -> { if (!attributesToFetch.isEmpty()) {
if (fwInfo.isSupported()) { var future = attributesService.getSharedAttributes(client, attributesToFetch);
Optional<String> newFwTitle = getAttributeValue(attrs, FIRMWARE_TITLE); DonAsynchron.withCallback(future, attrs -> {
Optional<String> newFwVersion = getAttributeValue(attrs, FIRMWARE_VERSION); if (fwInfo.isSupported()) {
Optional<String> newFwTag = getAttributeValue(attrs, FIRMWARE_TAG); Optional<String> newFwTitle = getAttributeValue(attrs, FIRMWARE_TITLE);
Optional<String> newFwUrl = getAttributeValue(attrs, FIRMWARE_URL); Optional<String> newFwVersion = getAttributeValue(attrs, FIRMWARE_VERSION);
if (newFwTitle.isPresent() && newFwVersion.isPresent() && !isOtaDownloading(client) && !UPDATING.equals(fwInfo.status)) { Optional<String> newFwTag = getAttributeValue(attrs, FIRMWARE_TAG);
onTargetFirmwareUpdate(client, newFwTitle.get(), newFwVersion.get(), newFwUrl, newFwTag); Optional<String> newFwUrl = getAttributeValue(attrs, FIRMWARE_URL);
if (newFwTitle.isPresent() && newFwVersion.isPresent() && !isOtaDownloading(client) && !UPDATING.equals(fwInfo.status)) {
onTargetFirmwareUpdate(client, newFwTitle.get(), newFwVersion.get(), newFwUrl, newFwTag);
}
} }
} if (swInfo.isSupported()) {
if (swInfo.isSupported()) { Optional<String> newSwTitle = getAttributeValue(attrs, SOFTWARE_TITLE);
Optional<String> newSwTitle = getAttributeValue(attrs, SOFTWARE_TITLE); Optional<String> newSwVersion = getAttributeValue(attrs, SOFTWARE_VERSION);
Optional<String> newSwVersion = getAttributeValue(attrs, SOFTWARE_VERSION); Optional<String> newSwTag = getAttributeValue(attrs, SOFTWARE_TAG);
Optional<String> newSwTag = getAttributeValue(attrs, SOFTWARE_TAG); Optional<String> newSwUrl = getAttributeValue(attrs, SOFTWARE_URL);
Optional<String> newSwUrl = getAttributeValue(attrs, SOFTWARE_URL); if (newSwTitle.isPresent() && newSwVersion.isPresent()) {
if (newSwTitle.isPresent() && newSwVersion.isPresent()) { onTargetSoftwareUpdate(client, newSwTitle.get(), newSwVersion.get(), newSwUrl, newSwTag);
onTargetSoftwareUpdate(client, newSwTitle.get(), newSwVersion.get(), newSwUrl, newSwTag); }
} }
}
}, throwable -> { }, throwable -> {
if (fwInfo.isSupported()) { if (fwInfo.isSupported()) {
update(fwInfo); update(fwInfo);
} }
}, executor); }, executor);
}
} }
} }

13
common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisRegistrationStore.java

@ -750,11 +750,14 @@ public class TbLwM2mRedisRegistrationStore implements RegistrationStore, Startab
System.currentTimeMillis(), 0, cleanLimit); System.currentTimeMillis(), 0, cleanLimit);
for (byte[] endpoint : endpointsExpired) { for (byte[] endpoint : endpointsExpired) {
Registration r = deserializeReg(connection.get(toEndpointKey(endpoint))); byte[] data = connection.get(toEndpointKey(endpoint));
if (!r.isAlive(gracePeriod)) { if (data != null && data.length > 0) {
Deregistration dereg = removeRegistration(connection, r.getId(), true); Registration r = deserializeReg(data);
if (dereg != null) if (!r.isAlive(gracePeriod)) {
expirationListener.registrationExpired(dereg.getRegistration(), dereg.getObservations()); Deregistration dereg = removeRegistration(connection, r.getId(), true);
if (dereg != null)
expirationListener.registrationExpired(dereg.getRegistration(), dereg.getObservations());
}
} }
} }
} catch (Exception e) { } catch (Exception e) {

22
common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2mUplinkMsgHandler.java

@ -486,13 +486,18 @@ public class DefaultLwM2mUplinkMsgHandler extends LwM2MExecutorAwareService impl
*/ */
private void initClientTelemetry(LwM2mClient lwM2MClient) { private void initClientTelemetry(LwM2mClient lwM2MClient) {
Lwm2mDeviceProfileTransportConfiguration profile = clientContext.getProfile(lwM2MClient.getRegistration()); Lwm2mDeviceProfileTransportConfiguration profile = clientContext.getProfile(lwM2MClient.getRegistration());
Set<String> supportedObjects = clientContext.getSupportedIdVerInClient(lwM2MClient); if (profile != null) {
if (supportedObjects != null && !supportedObjects.isEmpty()) { Set<String> supportedObjects = clientContext.getSupportedIdVerInClient(lwM2MClient);
this.sendInitObserveRequests(lwM2MClient, profile, supportedObjects); if (supportedObjects != null && !supportedObjects.isEmpty()) {
this.sendReadRequests(lwM2MClient, profile, supportedObjects); this.sendInitObserveRequests(lwM2MClient, profile, supportedObjects);
this.sendWriteAttributeRequests(lwM2MClient, profile, supportedObjects); this.sendReadRequests(lwM2MClient, profile, supportedObjects);
this.sendWriteAttributeRequests(lwM2MClient, profile, supportedObjects);
// Removed. Used only for debug. // Removed. Used only for debug.
// this.sendDiscoverRequests(lwM2MClient, profile, supportedObjects); // this.sendDiscoverRequests(lwM2MClient, profile, supportedObjects);
}
} else {
log.warn("[{}] Failed to process initClientTelemetry! Profile is null. Update procedure may not have completed after reboot yet", lwM2MClient.getEndpoint());
logService.log(lwM2MClient, "Failed to process initClientTelemetry. Profile is null. Update procedure may not have completed after reboot yet");
} }
} }
@ -1028,7 +1033,7 @@ public class DefaultLwM2mUplinkMsgHandler extends LwM2MExecutorAwareService impl
}); });
} }
private void updateValueOta(List<LwM2mClient> clients, Lwm2mDeviceProfileTransportConfiguration oldProfile, Lwm2mDeviceProfileTransportConfiguration newProfile) { private void updateValueOta(List<LwM2mClient> clients, Lwm2mDeviceProfileTransportConfiguration newProfile, Lwm2mDeviceProfileTransportConfiguration oldProfile) {
OtherConfiguration newLwM2mSettings = newProfile.getClientLwM2mSettings(); OtherConfiguration newLwM2mSettings = newProfile.getClientLwM2mSettings();
OtherConfiguration oldLwM2mSettings = oldProfile.getClientLwM2mSettings(); OtherConfiguration oldLwM2mSettings = oldProfile.getClientLwM2mSettings();
if (!newLwM2mSettings.getFwUpdateStrategy().equals(oldLwM2mSettings.getFwUpdateStrategy()) if (!newLwM2mSettings.getFwUpdateStrategy().equals(oldLwM2mSettings.getFwUpdateStrategy())
@ -1110,6 +1115,9 @@ public class DefaultLwM2mUplinkMsgHandler extends LwM2MExecutorAwareService impl
v -> attributesService.onAttributesUpdate(lwM2MClient, v, logFailedUpdateOfNonChangedValue), v -> attributesService.onAttributesUpdate(lwM2MClient, v, logFailedUpdateOfNonChangedValue),
t -> log.error("[{}] Failed to get attributes", lwM2MClient.getEndpoint(), t), t -> log.error("[{}] Failed to get attributes", lwM2MClient.getEndpoint(), t),
executor); executor);
} else {
log.warn("[{}] Failed to process initAttributes! Profile is null. Update procedure may not have completed after reboot yet", lwM2MClient.getEndpoint());
logService.log(lwM2MClient, "Failed to process initAttributes. Profile is null. Update procedure may not have completed after reboot yet");
} }
} }
@ -1119,7 +1127,7 @@ public class DefaultLwM2mUplinkMsgHandler extends LwM2MExecutorAwareService impl
private Map<String, String> getNamesFromProfileForSharedAttributes(LwM2mClient lwM2MClient) { private Map<String, String> getNamesFromProfileForSharedAttributes(LwM2mClient lwM2MClient) {
Lwm2mDeviceProfileTransportConfiguration profile = clientContext.getProfile(lwM2MClient.getRegistration()); Lwm2mDeviceProfileTransportConfiguration profile = clientContext.getProfile(lwM2MClient.getRegistration());
return profile.getObserveAttr().getKeyName(); return profile != null ? profile.getObserveAttr().getKeyName() : Collections.emptyMap();
} }
public LwM2MTransportServerConfig getConfig() { public LwM2MTransportServerConfig getConfig() {

2
dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java

@ -64,7 +64,7 @@ import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.ConstraintValidator; import org.thingsboard.server.dao.service.ConstraintValidator;
import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.tenant.TenantService;

2
dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java

@ -37,7 +37,7 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.dao.entity.CachedVersionedEntityService; import org.thingsboard.server.dao.entity.CachedVersionedEntityService;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.resource.ImageService; import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.DataValidator;
import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.PaginatedRemover;

2
dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java

@ -53,7 +53,7 @@ import org.thingsboard.server.dao.entity.EntityCountService;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.DataValidator;
import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.PaginatedRemover;
import org.thingsboard.server.dao.sql.JpaExecutorService; import org.thingsboard.server.dao.sql.JpaExecutorService;

2
dao/src/main/java/org/thingsboard/server/dao/component/BaseComponentDescriptorService.java

@ -30,7 +30,7 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.plugin.ComponentDescriptor; import org.thingsboard.server.common.data.plugin.ComponentDescriptor;
import org.thingsboard.server.common.data.plugin.ComponentScope; import org.thingsboard.server.common.data.plugin.ComponentScope;
import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.plugin.ComponentType;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.DataValidator;
import org.thingsboard.server.dao.service.Validator; import org.thingsboard.server.dao.service.Validator;

2
dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java

@ -45,7 +45,7 @@ import org.thingsboard.server.dao.entity.AbstractCachedEntityService;
import org.thingsboard.server.dao.entity.EntityCountService; import org.thingsboard.server.dao.entity.EntityCountService;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.DataValidator;
import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.PaginatedRemover;

2
dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java

@ -50,7 +50,7 @@ import org.thingsboard.server.dao.entity.EntityCountService;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.resource.ImageService; import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.DataValidator;

2
dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsServiceImpl.java

@ -43,9 +43,9 @@ import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.msg.EncryptionUtil; import org.thingsboard.server.common.msg.EncryptionUtil;
import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.entity.AbstractCachedEntityService;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException;
import org.thingsboard.server.dao.service.validator.DeviceCredentialsDataValidator; import org.thingsboard.server.dao.service.validator.DeviceCredentialsDataValidator;
import org.thingsboard.server.exception.DataValidationException;
import java.util.Objects; import java.util.Objects;

2
dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java

@ -46,11 +46,11 @@ import org.thingsboard.server.common.msg.EncryptionUtil;
import org.thingsboard.server.dao.entity.CachedVersionedEntityService; import org.thingsboard.server.dao.entity.CachedVersionedEntityService;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.resource.ImageService; import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.PaginatedRemover;
import org.thingsboard.server.dao.service.Validator; import org.thingsboard.server.dao.service.Validator;
import org.thingsboard.server.dao.service.validator.DeviceProfileDataValidator; import org.thingsboard.server.dao.service.validator.DeviceProfileDataValidator;
import org.thingsboard.server.exception.DataValidationException;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.security.cert.Certificate; import java.security.cert.Certificate;

2
dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java

@ -80,7 +80,7 @@ import org.thingsboard.server.dao.event.EventService;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.PaginatedRemover;
import org.thingsboard.server.dao.service.validator.DeviceDataValidator; import org.thingsboard.server.dao.service.validator.DeviceDataValidator;

2
dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java

@ -66,7 +66,7 @@ import org.thingsboard.server.dao.entity.EntityCountService;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.DataValidator;

3
dao/src/main/java/org/thingsboard/server/dao/edge/PostgresEdgeEventService.java

@ -68,7 +68,8 @@ public class PostgresEdgeEventService extends BaseEdgeEventService {
Futures.addCallback(saveFuture, new FutureCallback<>() { Futures.addCallback(saveFuture, new FutureCallback<>() {
@Override @Override
public void onSuccess(Void result) { public void onSuccess(Void result) {
statsCounterService.ifPresent(statsCounterService -> statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edgeEvent.getTenantId(), edgeEvent.getEdgeId(), 1)); statsCounterService.ifPresent(statsCounterService ->
statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edgeEvent.getTenantId(), edgeEvent.getEdgeId(), 1));
eventPublisher.publishEvent(SaveEntityEvent.builder() eventPublisher.publishEvent(SaveEntityEvent.builder()
.tenantId(edgeEvent.getTenantId()) .tenantId(edgeEvent.getTenantId())
.entityId(edgeEvent.getEdgeId()) .entityId(edgeEvent.getEdgeId())

15
dao/src/main/java/org/thingsboard/server/dao/edge/stats/EdgeStatsCounterService.java

@ -24,13 +24,13 @@ import org.thingsboard.server.common.data.id.TenantId;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
@ConditionalOnProperty(prefix = "edges.stats", name = "enabled", havingValue = "true", matchIfMissing = false) @ConditionalOnProperty(prefix = "edges.stats", name = "enabled", havingValue = "true")
@Service @Service
@Slf4j @Slf4j
@Getter @Getter
public class EdgeStatsCounterService { public class EdgeStatsCounterService {
private final ConcurrentHashMap<EdgeId, MsgCounters> counterByEdge = new ConcurrentHashMap<>(); private final ConcurrentHashMap<EdgeId, MsgCounters> msgCountersByEdge = new ConcurrentHashMap<>();
public void recordEvent(EdgeStatsKey type, TenantId tenantId, EdgeId edgeId, long value) { public void recordEvent(EdgeStatsKey type, TenantId tenantId, EdgeId edgeId, long value) {
MsgCounters counters = getOrCreateCounters(tenantId, edgeId); MsgCounters counters = getOrCreateCounters(tenantId, edgeId);
@ -39,19 +39,16 @@ public class EdgeStatsCounterService {
case DOWNLINK_MSGS_PUSHED -> counters.getMsgsPushed().addAndGet(value); case DOWNLINK_MSGS_PUSHED -> counters.getMsgsPushed().addAndGet(value);
case DOWNLINK_MSGS_PERMANENTLY_FAILED -> counters.getMsgsPermanentlyFailed().addAndGet(value); case DOWNLINK_MSGS_PERMANENTLY_FAILED -> counters.getMsgsPermanentlyFailed().addAndGet(value);
case DOWNLINK_MSGS_TMP_FAILED -> counters.getMsgsTmpFailed().addAndGet(value); case DOWNLINK_MSGS_TMP_FAILED -> counters.getMsgsTmpFailed().addAndGet(value);
case DOWNLINK_MSGS_LAG -> counters.getMsgsLag().set(value);
} }
} }
public void setDownlinkMsgsLag(TenantId tenantId, EdgeId edgeId, long value) { public MsgCounters getOrCreateCounters(TenantId tenantId, EdgeId edgeId) {
getOrCreateCounters(tenantId, edgeId).getMsgsLag().set(value); return msgCountersByEdge.computeIfAbsent(edgeId, id -> new MsgCounters(tenantId));
} }
public void clear(EdgeId edgeId) { public void clear(EdgeId edgeId) {
counterByEdge.remove(edgeId); msgCountersByEdge.remove(edgeId);
}
public MsgCounters getOrCreateCounters(TenantId tenantId, EdgeId edgeId) {
return counterByEdge.computeIfAbsent(edgeId, id -> new MsgCounters(tenantId));
} }
} }

2
dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java

@ -44,7 +44,7 @@ import org.thingsboard.server.dao.alarm.AlarmService;
import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.cf.CalculatedFieldService;
import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.housekeeper.CleanUpService; import org.thingsboard.server.dao.housekeeper.CleanUpService;
import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TbTenantProfileCache;

2
dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java

@ -51,7 +51,7 @@ import org.thingsboard.server.dao.entity.CachedVersionedEntityService;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.PaginatedRemover;
import org.thingsboard.server.dao.service.validator.EntityViewDataValidator; import org.thingsboard.server.dao.service.validator.EntityViewDataValidator;
import org.thingsboard.server.dao.sql.JpaExecutorService; import org.thingsboard.server.dao.sql.JpaExecutorService;

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save