Browse Source

Merge branch 'master' of github.com:thingsboard/thingsboard into aggregation-cf/improvements

pull/14565/head
IrynaMatveieva 10 months ago
parent
commit
d3e1d2123b
  1. 26
      application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java
  2. 7
      application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java
  3. 3
      application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java
  4. 24
      application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java
  5. 4
      application/src/main/java/org/thingsboard/server/controller/BaseController.java
  6. 2
      application/src/main/java/org/thingsboard/server/controller/EdgeController.java
  7. 1
      application/src/main/java/org/thingsboard/server/exception/ThingsboardErrorResponseHandler.java
  8. 24
      application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java
  9. 8
      application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java
  10. 4
      application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java
  11. 7
      application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java
  12. 39
      application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java
  13. 10
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java
  14. 51
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java
  15. 36
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java
  16. 15
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java
  17. 24
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java
  18. 4
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java
  19. 2
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java
  20. 19
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java
  21. 21
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/ScheduledRefreshSupported.java
  22. 46
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java
  23. 67
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java
  24. 3
      application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java
  25. 1
      application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java
  26. 12
      application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java
  27. 3
      application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java
  28. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/ai/AiModelEdgeProcessor.java
  29. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java
  30. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessor.java
  31. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/cf/CalculatedFieldEdgeProcessor.java
  32. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java
  33. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java
  34. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessor.java
  35. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java
  36. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java
  37. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java
  38. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java
  39. 13
      application/src/main/java/org/thingsboard/server/service/edge/stats/EdgeStatsService.java
  40. 2
      application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java
  41. 2
      application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java
  42. 2
      application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java
  43. 2
      application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/DefaultJwtSettingsValidator.java
  44. 2
      application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java
  45. 2
      application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java
  46. 2
      application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java
  47. 2
      application/src/main/java/org/thingsboard/server/service/security/system/SystemSecurityService.java
  48. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java
  49. 2
      application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java
  50. 12
      application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java
  51. 2
      application/src/main/java/org/thingsboard/server/utils/LwM2mObjectModelUtils.java
  52. 68
      application/src/main/resources/thingsboard.yml
  53. 57
      application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java
  54. 2
      application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java
  55. 2
      application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java
  56. 2
      application/src/test/java/org/thingsboard/server/controller/AssetProfileControllerTest.java
  57. 2
      application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java
  58. 2
      application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java
  59. 2
      application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java
  60. 2
      application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java
  61. 2
      application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java
  62. 2
      application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java
  63. 2
      application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java
  64. 2
      application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java
  65. 2
      application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java
  66. 2
      application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java
  67. 2
      application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java
  68. 148
      application/src/test/java/org/thingsboard/server/edge/EdgeStatsIntegrationTest.java
  69. 77
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationArgumentEntryTest.java
  70. 141
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java
  71. 133
      application/src/test/java/org/thingsboard/server/service/edge/EdgeStatsTest.java
  72. 2
      application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java
  73. 62
      application/src/test/java/org/thingsboard/server/service/security/auth/pat/ApiKeyAuthenticationProviderTest.java
  74. 5
      application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java
  75. 14
      common/data/src/main/java/org/thingsboard/server/common/data/device/profile/Lwm2mDeviceProfileTransportConfiguration.java
  76. 5
      common/data/src/main/java/org/thingsboard/server/exception/DataValidationException.java
  77. 4
      common/data/src/main/java/org/thingsboard/server/exception/EntitiesLimitExceededException.java
  78. 51
      common/data/src/main/java/org/thingsboard/server/exception/ThingsboardRuntimeException.java
  79. 1
      common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeRpcClient.java
  80. 1
      common/proto/src/main/proto/queue.proto
  81. 29
      common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaAdmin.java
  82. 1
      common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java
  83. 25
      common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClientContextImpl.java
  84. 58
      common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java
  85. 13
      common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisRegistrationStore.java
  86. 22
      common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2mUplinkMsgHandler.java
  87. 2
      dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java
  88. 2
      dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java
  89. 2
      dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java
  90. 2
      dao/src/main/java/org/thingsboard/server/dao/component/BaseComponentDescriptorService.java
  91. 2
      dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java
  92. 2
      dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java
  93. 2
      dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsServiceImpl.java
  94. 2
      dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java
  95. 2
      dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java
  96. 2
      dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java
  97. 3
      dao/src/main/java/org/thingsboard/server/dao/edge/PostgresEdgeEventService.java
  98. 15
      dao/src/main/java/org/thingsboard/server/dao/edge/stats/EdgeStatsCounterService.java
  99. 2
      dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java
  100. 2
      dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java

26
application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java

@ -132,6 +132,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
} else {
removeState(cfId);
}
msg.getCallback().onSuccess();
}
public void process(CalculatedFieldStatePartitionRestoreMsg msg) {
@ -242,7 +243,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
}
if (state instanceof PropagationCalculatedFieldState propagationState) {
PropagationArgumentEntry entry = new PropagationArgumentEntry();
entry.setAdded(msg.getRelatedEntityId());
entry.setAdded(List.of(msg.getRelatedEntityId()));
updatedArgs = propagationState.update(Map.of(PROPAGATION_CONFIG_ARGUMENT, entry), ctx);
}
if (CollectionsUtil.isEmpty(updatedArgs)) {
@ -421,19 +422,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
if (state == null) {
state = createState(ctx);
justRestored = true;
} else if (ctx.shouldFetchRelationQueryDynamicArgumentsFromDb(state)) {
log.debug("[{}][{}] Going to update dynamic arguments for CF.", entityId, ctx.getCfId());
try {
Map<String, ArgumentEntry> dynamicArgsFromDb = cfService.fetchDynamicArgsFromDb(ctx, entityId);
dynamicArgsFromDb.forEach(newArgValues::putIfAbsent);
if (ctx.getCfType() == CalculatedFieldType.GEOFENCING) {
var geofencingState = (GeofencingCalculatedFieldState) state;
geofencingState.updateLastDynamicArgumentsRefreshTs();
}
} catch (Exception e) {
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build();
}
} else if (ctx.shouldFetchEntityRelations(state)) {
} else if (ctx.shouldFetchRelatedEntities(state)) {
log.debug("[{}][{}] Going to update related entities for CF.", entityId, ctx.getCfId());
try {
if (state instanceof RelatedEntitiesAggregationCalculatedFieldState relatedEntitiesState) {
@ -447,6 +436,11 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
justRestored = true;
}
}
if (state instanceof GeofencingCalculatedFieldState geofencingCalculatedFieldState) {
Map<String, ArgumentEntry> dynamicArgsFromDb = cfService.fetchDynamicArgsFromDb(ctx, entityId);
dynamicArgsFromDb.forEach(newArgValues::putIfAbsent);
geofencingCalculatedFieldState.updateScheduledRefreshTs();
}
} catch (Exception e) {
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build();
}
@ -476,9 +470,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
state.setCtx(ctx, actorCtx);
state.init(false);
if (ctx.getCfType() == CalculatedFieldType.GEOFENCING && ctx.isRelationQueryDynamicArguments()) {
if (ctx.getCfType() == CalculatedFieldType.GEOFENCING && ctx.isCfHasRelationPathQuerySource()) {
GeofencingCalculatedFieldState geofencingState = (GeofencingCalculatedFieldState) state;
geofencingState.updateLastDynamicArgumentsRefreshTs();
geofencingState.updateScheduledRefreshTs();
}
Map<String, ArgumentEntry> arguments = fetchArguments(ctx);

7
application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java

@ -167,10 +167,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware
if (ctx != null) {
msg.setCtx(ctx);
log.debug("Pushing CF state restore msg to specific actor [{}]", msg.getId().entityId());
log.debug("[{}] Pushing CF state restore msg to specific actor [{}]", tenantId, msg.getId().entityId());
getOrCreateActor(msg.getId().entityId()).tellWithHighPriority(msg);
} else {
} else if (msg.getState() != null) {
log.debug("[{}] Received CF state restore msg for non-existing CF [{}]. Removing state", tenantId, cfId);
cfStateService.deleteState(msg.getId(), msg.getCallback());
} else {
msg.getCallback().onSuccess();
}
}

3
application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java

@ -19,6 +19,7 @@ import lombok.Data;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.MsgType;
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
@ -30,6 +31,7 @@ public class CalculatedFieldStateRestoreMsg implements ToCalculatedFieldSystemMs
private final CalculatedFieldEntityCtxId id;
private final CalculatedFieldState state;
private final TopicPartitionInfo partition;
private final TbCallback callback;
private CalculatedFieldCtx ctx;
@Override
@ -41,4 +43,5 @@ public class CalculatedFieldStateRestoreMsg implements ToCalculatedFieldSystemMs
public TenantId getTenantId() {
return id.tenantId();
}
}

24
application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java

@ -43,7 +43,6 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.msg.MsgType;
import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.TbActorStopReason;
import org.thingsboard.server.common.msg.TbMsg;
@ -139,13 +138,22 @@ public class TenantActor extends RuleChainManagerActor {
@Override
protected boolean doProcess(TbActorMsg msg) {
if (cantFindTenant) {
log.info("[{}] Processing missing Tenant msg: {}", tenantId, msg);
if (msg.getMsgType().equals(MsgType.QUEUE_TO_RULE_ENGINE_MSG)) {
QueueToRuleEngineMsg queueMsg = (QueueToRuleEngineMsg) msg;
queueMsg.getMsg().getCallback().onSuccess();
} else if (msg.getMsgType().equals(MsgType.TRANSPORT_TO_DEVICE_ACTOR_MSG)) {
TransportToDeviceActorMsgWrapper transportMsg = (TransportToDeviceActorMsgWrapper) msg;
transportMsg.getCallback().onSuccess();
log.debug("[{}] Processing message for non-existing tenant: {}", tenantId, msg);
switch (msg.getMsgType()) {
case QUEUE_TO_RULE_ENGINE_MSG -> {
((QueueToRuleEngineMsg) msg).getMsg().getCallback().onSuccess();
}
case TRANSPORT_TO_DEVICE_ACTOR_MSG -> {
((TransportToDeviceActorMsgWrapper) msg).getCallback().onSuccess();
}
case CF_STATE_RESTORE_MSG -> {
((CalculatedFieldStateRestoreMsg) msg).getCallback().onSuccess();
}
default -> {
if (!log.isDebugEnabled()) {
log.info("[{}] Processing message for non-existing tenant: {}", tenantId, msg);
}
}
}
return true;
}

4
application/src/main/java/org/thingsboard/server/controller/BaseController.java

@ -150,8 +150,6 @@ import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.domain.DomainService;
import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.exception.EntitiesLimitExceededException;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.job.JobService;
import org.thingsboard.server.dao.mobile.MobileAppBundleService;
@ -175,6 +173,8 @@ import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.dao.widget.WidgetTypeService;
import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.exception.EntitiesLimitExceededException;
import org.thingsboard.server.exception.ThingsboardErrorResponseHandler;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;

2
application/src/main/java/org/thingsboard/server/controller/EdgeController.java

@ -54,7 +54,7 @@ import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportReques
import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportResult;
import org.thingsboard.server.common.msg.edge.FromEdgeSyncResponse;
import org.thingsboard.server.config.annotations.ApiOperation;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.queue.util.TbCoreComponent;

1
application/src/main/java/org/thingsboard/server/exception/ThingsboardErrorResponseHandler.java

@ -52,7 +52,6 @@ import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.msg.tools.MaxPayloadSizeExceededException;
import org.thingsboard.server.common.msg.tools.TbRateLimitsException;
import org.thingsboard.server.dao.exception.EntitiesLimitExceededException;
import org.thingsboard.server.service.security.exception.AuthMethodNotSupportedException;
import org.thingsboard.server.service.security.exception.JwtExpiredTokenException;
import org.thingsboard.server.service.security.exception.UserPasswordExpiredException;

24
application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java

@ -22,17 +22,18 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.gson.JsonElement;
import com.google.gson.JsonParser;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.function.TriConsumer;
import org.thingsboard.common.util.DonAsynchron;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.rule.engine.api.AttributesSaveRequest;
import org.thingsboard.rule.engine.api.AttributesSaveRequest.Strategy;
import org.thingsboard.rule.engine.api.TimeseriesSaveRequest;
import org.thingsboard.server.actors.calculatedField.MultipleTbCallback;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.adaptor.JsonConverter;
import org.thingsboard.server.common.data.AttributeScope;
@ -80,7 +81,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.function.Function;
@ -391,6 +391,24 @@ public abstract class AbstractCalculatedFieldProcessingService {
return new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, endTs, 0, limit, Aggregation.NONE);
}
protected void handlePropagationResults(PropagationCalculatedFieldResult propagationResult, TbCallback callback,
TriConsumer<EntityId, TelemetryCalculatedFieldResult, TbCallback> telemetryResultHandler) {
List<EntityId> propagationEntityIds = propagationResult.getEntityIds();
if (propagationEntityIds.isEmpty()) {
callback.onSuccess();
return;
}
if (propagationEntityIds.size() == 1) {
EntityId propagationEntityId = propagationEntityIds.get(0);
telemetryResultHandler.accept(propagationEntityId, propagationResult.getResult(), callback);
return;
}
MultipleTbCallback multipleTbCallback = new MultipleTbCallback(propagationEntityIds.size(), callback);
for (var propagationEntityId : propagationEntityIds) {
telemetryResultHandler.accept(propagationEntityId, propagationResult.getResult(), multipleTbCallback);
}
}
protected void sendMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbCallback callback, TbMsg msg) {
try {
clusterService.pushMsgToRuleEngine(tenantId, entityId, msg, new TbQueueCallback() {
@ -413,7 +431,7 @@ public abstract class AbstractCalculatedFieldProcessingService {
protected void saveTelemetryResult(TenantId tenantId, EntityId entityId, String cfName, TelemetryCalculatedFieldResult cfResult, List<CalculatedFieldId> cfIds, TbCallback callback) {
OutputType type = cfResult.getType();
JsonElement jsonResult = JsonParser.parseString(Objects.requireNonNull(cfResult.stringValue()));
JsonElement jsonResult = cfResult.toJsonElement();
log.trace("[{}][{}] Saving CF result: {}", tenantId, entityId, jsonResult);
switch (type) {

8
application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java

@ -68,7 +68,7 @@ public abstract class AbstractCalculatedFieldStateService implements CalculatedF
protected abstract void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback);
protected void processRestoredState(CalculatedFieldStateProto stateMsg, TopicPartitionInfo partition) {
protected void processRestoredState(CalculatedFieldStateProto stateMsg, TopicPartitionInfo partition, TbCallback callback) {
var id = fromProto(stateMsg.getId());
if (partition == null) {
try {
@ -79,12 +79,12 @@ public abstract class AbstractCalculatedFieldStateService implements CalculatedF
}
}
var state = fromProto(id, stateMsg);
processRestoredState(id, state, partition);
processRestoredState(id, state, partition, callback);
}
protected void processRestoredState(CalculatedFieldEntityCtxId id, CalculatedFieldState state, TopicPartitionInfo partition) {
protected void processRestoredState(CalculatedFieldEntityCtxId id, CalculatedFieldState state, TopicPartitionInfo partition, TbCallback callback) {
partition = partition.withTopic(DataConstants.CF_STATES_QUEUE_NAME);
actorSystemContext.tellWithHighPriority(new CalculatedFieldStateRestoreMsg(id, state, partition));
actorSystemContext.tellWithHighPriority(new CalculatedFieldStateRestoreMsg(id, state, partition, callback));
}
@Override

4
application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java

@ -27,9 +27,11 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntry;
import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry;
import java.util.List;
import java.util.Map;
import java.util.Optional;
public interface CalculatedFieldProcessingService {
@ -37,6 +39,8 @@ public interface CalculatedFieldProcessingService {
Map<String, ArgumentEntry> fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId);
Optional<PropagationArgumentEntry> fetchPropagationArgumentFromDb(CalculatedFieldCtx ctx, EntityId entityId);
List<EntityId> fetchRelatedEntities(CalculatedFieldCtx ctx, EntityId entityId);
Map<String, ArgumentEntry> fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map<String, Argument> arguments);

7
application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java

@ -15,11 +15,14 @@
*/
package org.thingsboard.server.service.cf;
import com.google.gson.JsonElement;
import com.google.gson.JsonParser;
import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.msg.TbMsg;
import java.util.List;
import java.util.Objects;
public interface CalculatedFieldResult {
@ -29,4 +32,8 @@ public interface CalculatedFieldResult {
boolean isEmpty();
default JsonElement toJsonElement() {
return JsonParser.parseString(Objects.requireNonNull(stringValue()));
}
}

39
application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java

@ -17,13 +17,13 @@ package org.thingsboard.server.service.cf;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.function.TriConsumer;
import org.springframework.stereotype.Service;
import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg;
import org.thingsboard.server.actors.calculatedField.MultipleTbCallback;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Argument;
import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric;
import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration;
@ -50,6 +50,7 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntry;
import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry;
import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService;
import java.util.ArrayList;
@ -57,6 +58,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
@ -94,11 +96,18 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF
@Override
public Map<String, ArgumentEntry> fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId) {
return switch (ctx.getCfType()) {
case GEOFENCING -> resolveArgumentFutures(fetchGeofencingCalculatedFieldArguments(ctx, entityId, true, System.currentTimeMillis()));
case PROPAGATION -> resolveArgumentFutures(Map.of(PROPAGATION_CONFIG_ARGUMENT, fetchPropagationCalculatedFieldArgument(ctx, entityId)));
default -> Collections.emptyMap();
};
return ctx.getCfType() == CalculatedFieldType.GEOFENCING ?
resolveArgumentFutures(fetchGeofencingCalculatedFieldArguments(ctx, entityId, true, System.currentTimeMillis())) :
Collections.emptyMap();
}
@Override
public Optional<PropagationArgumentEntry> fetchPropagationArgumentFromDb(CalculatedFieldCtx ctx, EntityId entityId) {
if (ctx.getCfType() != CalculatedFieldType.PROPAGATION) {
return Optional.empty();
}
return Optional.of((PropagationArgumentEntry)
resolveArgumentValue(PROPAGATION_CONFIG_ARGUMENT, fetchPropagationCalculatedFieldArgument(ctx, entityId)));
}
@Override
@ -169,24 +178,6 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF
sendMsgToRuleEngine(tenantId, entityId, callback, result.toTbMsg(entityId, cfName, cfIds));
}
private void handlePropagationResults(PropagationCalculatedFieldResult propagationResult, TbCallback callback,
TriConsumer<EntityId, TelemetryCalculatedFieldResult, TbCallback> telemetryResultHandler) {
List<EntityId> propagationEntityIds = propagationResult.getEntityIds();
if (propagationEntityIds.isEmpty()) {
callback.onSuccess();
return;
}
if (propagationEntityIds.size() == 1) {
EntityId propagationEntityId = propagationEntityIds.get(0);
telemetryResultHandler.accept(propagationEntityId, propagationResult.getResult(), callback);
return;
}
MultipleTbCallback multipleTbCallback = new MultipleTbCallback(propagationEntityIds.size(), callback);
for (var propagationEntityId : propagationEntityIds) {
telemetryResultHandler.accept(propagationEntityId, propagationResult.getResult(), multipleTbCallback);
}
}
@Override
public void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List<CalculatedFieldEntityCtxId> linkedCalculatedFields, TbCallback callback) {
Map<TopicPartitionInfo, List<CalculatedFieldEntityCtxId>> unicasts = new HashMap<>();

10
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java

@ -63,7 +63,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState,
this.ctx = ctx;
this.actorCtx = actorCtx;
this.requiredArguments = ctx.getArgNames();
this.readinessStatus = checkReadiness(requiredArguments, arguments);
this.readinessStatus = checkReadiness();
}
@Override
@ -108,7 +108,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState,
if (updatedArguments == null) {
return Collections.emptyMap();
}
readinessStatus = checkReadiness(requiredArguments, arguments);
readinessStatus = checkReadiness();
return updatedArguments;
}
@ -183,13 +183,13 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState,
return latestTs;
}
protected ReadinessStatus checkReadiness(List<String> requiredArguments, Map<String, ArgumentEntry> currentArguments) {
if (currentArguments == null) {
protected ReadinessStatus checkReadiness() {
if (arguments == null) {
return ReadinessStatus.from(requiredArguments);
}
List<String> emptyArguments = null;
for (String requiredArgumentKey : requiredArguments) {
ArgumentEntry argumentEntry = currentArguments.get(requiredArgumentKey);
ArgumentEntry argumentEntry = arguments.get(requiredArgumentKey);
if (argumentEntry == null || argumentEntry.isEmpty()) {
if (emptyArguments == null) {
emptyArguments = new ArrayList<>();

51
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java

@ -63,8 +63,7 @@ import org.thingsboard.server.dao.util.TimeUtils;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto;
import org.thingsboard.server.service.cf.CalculatedFieldProcessingService;
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState;
import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState;
import org.thingsboard.server.service.cf.ctx.state.geofencing.ScheduledRefreshSupported;
import org.thingsboard.server.service.telemetry.AlarmSubscriptionService;
import java.io.Closeable;
@ -122,14 +121,14 @@ public class CalculatedFieldCtx implements Closeable {
private long maxSingleValueArgumentSize;
private long intermediateAggregationIntervalMillis;
private boolean relationQueryDynamicArguments;
private boolean cfHasRelationPathQuerySource;
private List<String> mainEntityGeofencingArgumentNames;
private List<String> linkedEntityAndCurrentOwnerGeofencingArgumentNames;
private List<String> relatedEntityArgumentNames;
private long scheduledUpdateIntervalMillis;
private long cfCheckReevaluationInterval;
private long alarmReevaluationInterval;
private long cfCheckReevaluationIntervalMillis;
private long alarmReevaluationIntervalMillis;
private Argument propagationArgument;
private boolean applyExpressionForResolvedArguments;
@ -161,10 +160,11 @@ public class CalculatedFieldCtx implements Closeable {
if (refId == null) {
if (CalculatedFieldType.RELATED_ENTITIES_AGGREGATION.equals(cfType)) {
relatedEntityArguments.compute(refKey, (key, existingNames) -> CollectionsUtil.addToSet(existingNames, entry.getKey()));
cfHasRelationPathQuerySource = true;
continue;
}
if (entry.getValue().hasRelationQuerySource()) {
relationQueryDynamicArguments = true;
cfHasRelationPathQuerySource = true;
continue;
}
if (entry.getValue().hasOwnerSource()) {
@ -201,7 +201,7 @@ public class CalculatedFieldCtx implements Closeable {
if (calculatedField.getConfiguration() instanceof PropagationCalculatedFieldConfiguration propagationConfig) {
propagationArgument = propagationConfig.toPropagationArgument();
applyExpressionForResolvedArguments = propagationConfig.isApplyExpressionToResolvedArguments();
relationQueryDynamicArguments = true;
cfHasRelationPathQuerySource = true;
}
}
if (calculatedField.getConfiguration() instanceof ScheduledUpdateSupportedCalculatedFieldConfiguration scheduledConfig) {
@ -246,8 +246,7 @@ public class CalculatedFieldCtx implements Closeable {
boolean requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation();
if (calculatedField.getConfiguration() instanceof AlarmCalculatedFieldConfiguration) {
if (requiresScheduledReevaluation) {
long reevaluationIntervalMillis = TimeUnit.SECONDS.toMillis(alarmReevaluationInterval);
if (now - lastReevaluationTs >= reevaluationIntervalMillis) {
if (now - lastReevaluationTs >= alarmReevaluationIntervalMillis) {
lastReevaluationTs = now;
return true;
}
@ -306,8 +305,8 @@ public class CalculatedFieldCtx implements Closeable {
this.maxStateSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes) * 1024;
this.maxSingleValueArgumentSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxSingleValueArgumentSizeInKBytes) * 1024;
this.intermediateAggregationIntervalMillis = TimeUnit.SECONDS.toMillis(apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getIntermediateAggregationIntervalInSecForCF));
this.cfCheckReevaluationInterval = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getCfReevaluationCheckInterval);
this.alarmReevaluationInterval = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getAlarmsReevaluationInterval);
this.cfCheckReevaluationIntervalMillis = TimeUnit.SECONDS.toMillis(apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getCfReevaluationCheckInterval));
this.alarmReevaluationIntervalMillis = TimeUnit.SECONDS.toMillis(apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getAlarmsReevaluationInterval));
}
public double evaluateSimpleExpression(Expression expression, CalculatedFieldState state) {
@ -757,38 +756,20 @@ public class CalculatedFieldCtx implements Closeable {
return scheduledUpdateIntervalMillis == DISABLED_INTERVAL_VALUE;
}
public boolean shouldFetchRelationQueryDynamicArgumentsFromDb(CalculatedFieldState state) {
if (!relationQueryDynamicArguments) {
public boolean shouldFetchRelatedEntities(CalculatedFieldState state) {
if (!cfHasRelationPathQuerySource) {
return false;
}
return switch (cfType) {
case PROPAGATION -> true;
case GEOFENCING -> {
if (isScheduledUpdateDisabled()) {
yield false;
}
var geofencingState = (GeofencingCalculatedFieldState) state;
if (geofencingState.getLastDynamicArgumentsRefreshTs() == DEFAULT_LAST_UPDATE_TS) {
yield true;
}
yield geofencingState.getLastDynamicArgumentsRefreshTs() <
System.currentTimeMillis() - scheduledUpdateIntervalMillis;
}
default -> false;
};
}
public boolean shouldFetchEntityRelations(CalculatedFieldState state) {
if (!(state instanceof RelatedEntitiesAggregationCalculatedFieldState relatedEntitiesAggState)) {
if (isScheduledUpdateDisabled()) {
return false;
}
if (isScheduledUpdateDisabled()) {
if (!(state instanceof ScheduledRefreshSupported scheduledRefreshSupported)) {
return false;
}
if (relatedEntitiesAggState.getLastRelatedEntitiesRefreshTs() == DEFAULT_LAST_UPDATE_TS) {
if (scheduledRefreshSupported.getLastScheduledRefreshTs() == DEFAULT_LAST_UPDATE_TS) {
return true;
}
return relatedEntitiesAggState.getLastRelatedEntitiesRefreshTs() < System.currentTimeMillis() - scheduledUpdateIntervalMillis;
return scheduledRefreshSupported.getLastScheduledRefreshTs() < System.currentTimeMillis() - scheduledUpdateIntervalMillis;
}
@Override

36
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java

@ -43,6 +43,8 @@ import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory;
import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService;
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.bytesToString;
@ -61,6 +63,8 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta
@Value("${queue.calculated_fields.poll_interval:25}")
private long pollInterval;
@Value("${queue.calculated_fields.pack_processing_timeout:60000}")
private long packProcessingTimeout;
private TbKafkaProducerTemplate<TbProtoQueueMsg<CalculatedFieldStateProto>> stateProducer;
@ -74,21 +78,39 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta
.topic(partitionService.getTopic(queueKey))
.pollInterval(pollInterval)
.msgPackProcessor((msgs, consumer, consumerKey, config) -> {
CountDownLatch completionLatch = new CountDownLatch(msgs.size());
for (TbProtoQueueMsg<CalculatedFieldStateProto> msg : msgs) {
TbCallback callback = new TbCallback() {
@Override
public void onSuccess() {
int processedMsgCount = counter.incrementAndGet();
if (processedMsgCount % 10000 == 0) {
log.info("Processed {} CF state messages", processedMsgCount);
}
completionLatch.countDown();
}
@Override
public void onFailure(Throwable t) {
log.error("Failed to process CF state message: {}", msg, t);
completionLatch.countDown();
}
};
try {
if (msg.getValue() != null) {
processRestoredState(msg.getValue(), consumerKey.partition());
processRestoredState(msg.getValue(), consumerKey.partition(), callback);
} else {
processRestoredState(getStateId(msg.getHeaders()), null, consumerKey.partition());
processRestoredState(getStateId(msg.getHeaders()), null, consumerKey.partition(), callback);
}
} catch (Throwable t) {
log.error("Failed to process state message: {}", msg, t);
callback.onFailure(t);
}
}
int processedMsgCount = counter.incrementAndGet();
if (processedMsgCount % 10000 == 0) {
log.info("Processed {} calculated field state msgs", processedMsgCount);
}
boolean success = completionLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS);
if (!success) {
log.error("Timeout to process CF state messages pack of size {}", msgs.size());
}
})
.consumerCreator((queueConfig, tpi) -> queueFactory.createCalculatedFieldStateConsumer())

15
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java

@ -62,11 +62,22 @@ public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldS
public void restore(QueueKey queueKey, Set<TopicPartitionInfo> partitions) {
if (stateService.getPartitions().isEmpty()) {
cfRocksDb.forEach((key, value) -> {
CalculatedFieldStateProto stateMsg;
try {
processRestoredState(CalculatedFieldStateProto.parseFrom(value), null);
stateMsg = CalculatedFieldStateProto.parseFrom(value);
} catch (Exception e) {
log.error("[{}] Failed to process restored state", key, e);
log.error("Failed to parse CalculatedFieldStateProto for key {}", key, e);
return;
}
processRestoredState(stateMsg, null, new TbCallback() {
@Override
public void onSuccess() {}
@Override
public void onFailure(Throwable t) {
log.error("Failed to process CF state message: {}", stateMsg, t);
}
});
});
}
super.restore(queueKey, partitions);

24
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java

@ -41,6 +41,7 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType;
import org.thingsboard.server.service.cf.ctx.state.BaseCalculatedFieldState;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import org.thingsboard.server.service.cf.ctx.state.aggregation.function.AggEntry;
import org.thingsboard.server.service.cf.ctx.state.geofencing.ScheduledRefreshSupported;
import java.util.ArrayList;
import java.util.HashMap;
@ -55,14 +56,14 @@ import static org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx.DIS
import static org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState.ReadinessStatus.MISSING_AGGREGATION_ENTITIES_ERROR;
@Slf4j
@Getter
public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculatedFieldState {
public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculatedFieldState implements ScheduledRefreshSupported {
@Setter
@Getter
private long lastArgsRefreshTs = DEFAULT_LAST_UPDATE_TS;
@Setter
@Getter
private long lastMetricsEvalTs = DEFAULT_LAST_UPDATE_TS;
@Setter
private long lastRelatedEntitiesRefreshTs = DEFAULT_LAST_UPDATE_TS;
private long deduplicationIntervalMs = DISABLED_INTERVAL_VALUE;
private Map<String, AggMetric> metrics;
@ -104,13 +105,24 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat
@Override
public void reset() { // must reset everything dependent on arguments
super.reset();
resetScheduledRefreshTs();
lastArgsRefreshTs = DEFAULT_LAST_UPDATE_TS;
lastMetricsEvalTs = DEFAULT_LAST_UPDATE_TS;
lastRelatedEntitiesRefreshTs = DEFAULT_LAST_UPDATE_TS;
metrics = null;
}
public void updateLastRelatedEntitiesRefreshTs() {
@Override
public void resetScheduledRefreshTs() {
lastRelatedEntitiesRefreshTs = DEFAULT_LAST_UPDATE_TS;
}
@Override
public long getLastScheduledRefreshTs() {
return lastRelatedEntitiesRefreshTs;
}
@Override
public void updateScheduledRefreshTs() {
lastRelatedEntitiesRefreshTs = System.currentTimeMillis();
}
@ -128,7 +140,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat
public List<EntityId> checkRelatedEntities(List<EntityId> relatedEntities) {
Map<EntityId, Map<String, ArgumentEntry>> entityInputs = prepareInputs();
findOutdatedEntities(entityInputs, relatedEntities).forEach(this::cleanupEntityData);
updateLastRelatedEntitiesRefreshTs();
updateScheduledRefreshTs();
return findMissingEntities(entityInputs, relatedEntities);
}

4
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java

@ -205,7 +205,7 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt
handleExpiredInterval(intervalEntry, args, results);
expiredIntervals.add(intervalEntry);
} else if (now - startTs >= intervalEntry.getIntervalDuration()) {
handleActiveInterval(ctx.getCfCheckReevaluationInterval(), intervalEntry, args, results);
handleActiveInterval(ctx.getCfCheckReevaluationIntervalMillis(), intervalEntry, args, results);
if (watermarkDuration == 0) {
expiredIntervals.add(intervalEntry);
}
@ -288,7 +288,7 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt
}
if (!metricsNode.isEmpty()) {
ObjectNode resultNode = JacksonUtil.newObjectNode();
resultNode.put("ts", interval.getEndTs() - 1);
resultNode.put("ts", interval.getStartTs());
resultNode.set("values", metricsNode);
result.add(resultNode);

2
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java

@ -94,8 +94,6 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState {
private Alarm currentAlarm;
private boolean initialFetchDone;
// TODO: deprecate device profile node, describe the differences and improvements
public AlarmCalculatedFieldState(EntityId entityId) {
super(entityId);
}

19
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java

@ -21,8 +21,6 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.geo.Coordinates;
@ -52,11 +50,9 @@ import static org.thingsboard.server.common.data.cf.configuration.geofencing.Ent
import static org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingPresenceStatus.INSIDE;
import static org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingPresenceStatus.OUTSIDE;
@Getter
@Setter
@Slf4j
@EqualsAndHashCode(callSuper = true)
public class GeofencingCalculatedFieldState extends BaseCalculatedFieldState {
public class GeofencingCalculatedFieldState extends BaseCalculatedFieldState implements ScheduledRefreshSupported {
private long lastDynamicArgumentsRefreshTs = DEFAULT_LAST_UPDATE_TS;
@ -147,10 +143,21 @@ public class GeofencingCalculatedFieldState extends BaseCalculatedFieldState {
@Override
public void reset() {
super.reset();
resetScheduledRefreshTs();
}
@Override
public void resetScheduledRefreshTs() {
lastDynamicArgumentsRefreshTs = DEFAULT_LAST_UPDATE_TS;
}
public void updateLastDynamicArgumentsRefreshTs() {
@Override
public long getLastScheduledRefreshTs() {
return lastDynamicArgumentsRefreshTs;
}
@Override
public void updateScheduledRefreshTs() {
lastDynamicArgumentsRefreshTs = System.currentTimeMillis();
}

21
ui-ngx/src/app/modules/login/pages/login/reset-password-request.component.scss → application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/ScheduledRefreshSupported.java

@ -13,17 +13,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@import '../../../../../scss/constants';
package org.thingsboard.server.service.cf.ctx.state.geofencing;
public interface ScheduledRefreshSupported {
void resetScheduledRefreshTs();
long getLastScheduledRefreshTs();
void updateScheduledRefreshTs();
:host {
display: flex;
flex: 1 1 0;
.tb-request-password-reset-content {
background-color: #eee;
.tb-request-password-reset-card {
@media #{$mat-gt-xs} {
width: 450px !important;
}
}
}
}

46
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java

@ -22,6 +22,8 @@ import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry;
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@ -30,10 +32,11 @@ import java.util.Set;
public class PropagationArgumentEntry implements ArgumentEntry {
private Set<EntityId> entityIds;
private transient EntityId added;
private transient List<EntityId> added;
private transient EntityId removed;
private boolean forceResetPrevious;
private transient boolean forceResetPrevious;
private transient boolean ignoreRemovedEntities;
public PropagationArgumentEntry() {
this.entityIds = new HashSet<>();
@ -57,27 +60,44 @@ public class PropagationArgumentEntry implements ArgumentEntry {
@Override
public boolean updateEntry(ArgumentEntry entry) {
if (!(entry instanceof PropagationArgumentEntry propagationArgumentEntry)) {
if (!(entry instanceof PropagationArgumentEntry updated)) {
throw new IllegalArgumentException("Unsupported argument entry type for propagation argument entry: " + entry.getType());
}
if (propagationArgumentEntry.getAdded() != null) {
boolean updated = entityIds.add(propagationArgumentEntry.getAdded());
if (updated) {
added = propagationArgumentEntry.getAdded();
}
return updated;
if (updated.getAdded() != null) {
return checkAdded(updated.getAdded());
}
if (updated.getRemoved() != null) {
return entityIds.remove(updated.getRemoved());
}
if (propagationArgumentEntry.getRemoved() != null) {
return entityIds.remove(propagationArgumentEntry.getRemoved());
if (updated.isIgnoreRemovedEntities()) {
Set<EntityId> updatedIds = updated.getEntityIds();
if (updatedIds.isEmpty()) {
entityIds.clear();
return false;
}
entityIds.retainAll(updatedIds);
return checkAdded(updatedIds);
}
if (propagationArgumentEntry.isEmpty()) {
if (updated.isEmpty()) {
entityIds.clear();
return true;
}
entityIds = propagationArgumentEntry.getEntityIds();
entityIds = updated.getEntityIds();
return true;
}
private boolean checkAdded(Collection<EntityId> updatedIds) {
for (EntityId id : updatedIds) {
if (entityIds.add(id)) {
if (added == null) {
added = new ArrayList<>();
}
added.add(id);
}
}
return added != null && !added.isEmpty();
}
@Override
public boolean isEmpty() {
return entityIds.isEmpty();

67
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java

@ -25,6 +25,8 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Output;
import org.thingsboard.server.common.data.cf.configuration.OutputType;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.util.CollectionsUtil;
import org.thingsboard.server.service.cf.CalculatedFieldProcessingService;
import org.thingsboard.server.service.cf.CalculatedFieldResult;
import org.thingsboard.server.service.cf.PropagationCalculatedFieldResult;
import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult;
@ -41,6 +43,8 @@ import static org.thingsboard.server.common.data.cf.configuration.PropagationCal
public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState {
private CalculatedFieldProcessingService cfProcessingService;
public PropagationCalculatedFieldState(EntityId entityId) {
super(entityId);
}
@ -49,14 +53,30 @@ public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState
public void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx) {
this.ctx = ctx;
this.actorCtx = actorCtx;
this.cfProcessingService = ctx.getCfProcessingService();
this.requiredArguments = new ArrayList<>(ctx.getArgNames());
requiredArguments.add(PROPAGATION_CONFIG_ARGUMENT);
this.readinessStatus = checkReadiness(requiredArguments, arguments);
this.readinessStatus = checkReadiness();
if (ctx.isApplyExpressionForResolvedArguments()) {
this.tbelExpression = ctx.getTbelExpressions().get(ctx.getExpression());
}
}
@Override
public void init(boolean restored) {
super.init(restored);
if (restored) {
cfProcessingService.fetchPropagationArgumentFromDb(ctx, entityId).ifPresent(fromDb -> {
fromDb.setIgnoreRemovedEntities(true);
var updatedArgs = update(Map.of(PROPAGATION_CONFIG_ARGUMENT, fromDb), ctx);
if (updatedArgs.isEmpty()) {
return;
}
ctx.scheduleReevaluation(0L, actorCtx);
});
}
}
@Override
public CalculatedFieldType getType() {
return CalculatedFieldType.PROPAGATION;
@ -68,9 +88,10 @@ public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState
if (!(argumentEntry instanceof PropagationArgumentEntry propagationArgumentEntry)) {
return Futures.immediateFuture(PropagationCalculatedFieldResult.builder().build());
}
boolean newEntityAdded = propagationArgumentEntry.getAdded() != null;
List<EntityId> entityIds;
if (propagationArgumentEntry.getAdded() != null) {
entityIds = List.of(propagationArgumentEntry.getAdded());
if (newEntityAdded) {
entityIds = propagationArgumentEntry.getAdded();
propagationArgumentEntry.setAdded(null);
} else {
if (propagationArgumentEntry.getEntityIds().isEmpty()) {
@ -86,13 +107,43 @@ public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState
.build(),
MoreExecutors.directExecutor());
}
if (newEntityAdded || CollectionsUtil.isEmpty(updatedArgs)) {
updatedArgs = arguments;
}
return Futures.immediateFuture(PropagationCalculatedFieldResult.builder()
.entityIds(entityIds)
.result(toTelemetryResult(ctx))
.result(toTelemetryResult(ctx, updatedArgs))
.build());
}
private TelemetryCalculatedFieldResult toTelemetryResult(CalculatedFieldCtx ctx) {
@Override
protected ReadinessStatus checkReadiness() {
if (ctx.isApplyExpressionForResolvedArguments() || arguments == null) {
return super.checkReadiness();
}
boolean propagationNotEmpty = false;
boolean hasOtherNonEmpty = false;
List<String> emptyArguments = null;
for (String requiredArgumentKey : requiredArguments) {
ArgumentEntry argumentEntry = arguments.get(requiredArgumentKey);
if (argumentEntry == null || argumentEntry.isEmpty()) {
if (emptyArguments == null) {
emptyArguments = new ArrayList<>();
}
emptyArguments.add(requiredArgumentKey);
} else if (PROPAGATION_CONFIG_ARGUMENT.equals(requiredArgumentKey)) {
propagationNotEmpty = true;
} else {
hasOtherNonEmpty = true;
}
}
if (propagationNotEmpty && hasOtherNonEmpty) {
return ReadinessStatus.READY;
}
return ReadinessStatus.from(emptyArguments);
}
private TelemetryCalculatedFieldResult toTelemetryResult(CalculatedFieldCtx ctx, Map<String, ArgumentEntry> updatedArgs) {
Output output = ctx.getOutput();
TelemetryCalculatedFieldResult.TelemetryCalculatedFieldResultBuilder telemetryCfBuilder =
TelemetryCalculatedFieldResult.builder()
@ -100,12 +151,14 @@ public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState
.type(output.getType())
.scope(output.getScope());
ObjectNode valuesNode = JacksonUtil.newObjectNode();
arguments.forEach((outputKey, argumentEntry) -> {
updatedArgs.forEach((outputKey, argumentEntry) -> {
if (argumentEntry instanceof PropagationArgumentEntry) {
return;
}
if (argumentEntry instanceof SingleValueArgumentEntry singleArgumentEntry) {
JacksonUtil.addKvEntry(valuesNode, singleArgumentEntry.getKvEntryValue(), outputKey);
if (!singleArgumentEntry.isEmpty()) {
JacksonUtil.addKvEntry(valuesNode, singleArgumentEntry.getKvEntryValue(), outputKey);
}
return;
}
throw new IllegalArgumentException("Unsupported argument type: " + argumentEntry.getType() + " detected for argument: " + outputKey + ". " +

3
application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java

@ -258,8 +258,7 @@ public class DeviceBulkImportService extends AbstractBulkImportService<Device> {
Lwm2mDeviceProfileTransportConfiguration transportConfiguration = new Lwm2mDeviceProfileTransportConfiguration();
transportConfiguration.setBootstrap(Collections.emptyList());
transportConfiguration.setClientLwM2mSettings(new OtherConfiguration(false,1, 1, 1, PowerMode.DRX, null, null, null, null, null, V1_0.toString()));
transportConfiguration.setObserveAttr(new TelemetryMappingConfiguration(Collections.emptyMap(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptyMap(), false, SINGLE));
transportConfiguration.setClientLwM2mSettings(new OtherConfiguration());
DeviceProfileData deviceProfileData = new DeviceProfileData();
DefaultDeviceProfileConfiguration configuration = new DefaultDeviceProfileConfiguration();

1
application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java

@ -206,7 +206,6 @@ public class EdgeContextComponent {
@Autowired
private Optional<EdgeStatsCounterService> statsCounterService;
// processors
@Autowired
private AlarmProcessor alarmProcessor;

12
application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java

@ -306,7 +306,8 @@ public abstract class EdgeGrpcSession implements Closeable {
if (isConnected() && !pageData.getData().isEmpty()) {
if (fetcher instanceof GeneralEdgeEventFetcher) {
long queueSize = pageData.getTotalElements() - ((long) pageLink.getPageSize() * pageLink.getPage());
ctx.getStatsCounterService().ifPresent(statsCounterService -> statsCounterService.setDownlinkMsgsLag(edge.getTenantId(), edge.getId(), queueSize));
ctx.getStatsCounterService().ifPresent(statsCounterService ->
statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_LAG, tenantId, edge.getId(), queueSize));
}
log.trace("[{}][{}][{}] event(s) are going to be processed.", tenantId, edge.getId(), pageData.getData().size());
List<DownlinkMsg> downlinkMsgsPack = convertToDownlinkMsgsPack(pageData.getData());
@ -504,7 +505,8 @@ public abstract class EdgeGrpcSession implements Closeable {
ctx.getRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId())
.customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg)
.error("Failed to deliver messages after " + MAX_DOWNLINK_ATTEMPTS + " attempts").build());
ctx.getStatsCounterService().ifPresent(statsCounterService -> statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_PERMANENTLY_FAILED, edge.getTenantId(), edge.getId(), copy.size()));
ctx.getStatsCounterService().ifPresent(statsCounterService ->
statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_PERMANENTLY_FAILED, edge.getTenantId(), edge.getId(), copy.size()));
stopCurrentSendDownlinkMsgsTask(false);
}
} else {
@ -544,7 +546,8 @@ public abstract class EdgeGrpcSession implements Closeable {
try {
if (msg.getSuccess()) {
sessionState.getPendingMsgsMap().remove(msg.getDownlinkMsgId());
ctx.getStatsCounterService().ifPresent(statsCounterService -> statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_PUSHED, edge.getTenantId(), edge.getId(), 1));
ctx.getStatsCounterService().ifPresent(statsCounterService ->
statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_PUSHED, edge.getTenantId(), edge.getId(), 1));
log.debug("[{}][{}][{}] Msg has been processed successfully! Msg Id: [{}], Msg: {}", tenantId, edge.getId(), sessionId, msg.getDownlinkMsgId(), msg);
} else {
log.debug("[{}][{}][{}] Msg processing failed! Msg Id: [{}], Error msg: {}", tenantId, edge.getId(), sessionId, msg.getDownlinkMsgId(), msg.getErrorMsg());
@ -813,7 +816,8 @@ public abstract class EdgeGrpcSession implements Closeable {
}
}
highPriorityQueue.add(edgeEvent);
ctx.getStatsCounterService().ifPresent(statsCounterService -> statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edge.getTenantId(), edgeEvent.getEdgeId(), 1));
ctx.getStatsCounterService().ifPresent(statsCounterService ->
statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edge.getTenantId(), edgeEvent.getEdgeId(), 1));
}
protected ListenableFuture<List<Void>> processUplinkMsg(UplinkMsg uplinkMsg) {

3
application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java

@ -52,7 +52,8 @@ public class KafkaEdgeEventService extends BaseEdgeEventService {
TopicPartitionInfo tpi = topicService.getEdgeEventNotificationsTopic(edgeEvent.getTenantId(), edgeEvent.getEdgeId());
ToEdgeEventNotificationMsg msg = ToEdgeEventNotificationMsg.newBuilder().setEdgeEventMsg(ProtoUtils.toProto(edgeEvent)).build();
producerProvider.getTbEdgeEventsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null);
statsCounterService.ifPresent(statsCounterService -> statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edgeEvent.getTenantId(), edgeEvent.getEdgeId(), 1));
statsCounterService.ifPresent(statsCounterService ->
statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edgeEvent.getTenantId(), edgeEvent.getEdgeId(), 1));
return Futures.immediateFuture(null);
}

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/ai/AiModelEdgeProcessor.java

@ -29,7 +29,7 @@ import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.data.id.AiModelId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.AiModelUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java

@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessor.java

@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/cf/CalculatedFieldEdgeProcessor.java

@ -36,7 +36,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.CalculatedFieldUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java

@ -29,7 +29,7 @@ import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DashboardId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java

@ -44,7 +44,7 @@ import org.thingsboard.server.common.msg.TbMsgDataType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponseActorMsg;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DeviceRpcCallMsg;
import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessor.java

@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java

@ -32,7 +32,7 @@ import org.thingsboard.server.common.data.id.EntityViewId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;
import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java

@ -27,7 +27,7 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;
import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java

@ -31,7 +31,7 @@ import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java

@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;
import org.thingsboard.server.gen.edge.v1.UpdateMsgType;

13
application/src/main/java/org/thingsboard/server/service/edge/stats/EdgeStatsService.java

@ -54,7 +54,7 @@ import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_P
import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_TMP_FAILED;
@TbCoreComponent
@ConditionalOnProperty(prefix = "edges.stats", name = "enabled", havingValue = "true", matchIfMissing = false)
@ConditionalOnProperty(prefix = "edges.stats", name = "enabled", havingValue = "true")
@RequiredArgsConstructor
@Service
@Slf4j
@ -70,7 +70,6 @@ public class EdgeStatsService {
@Value("${edges.stats.report-interval-millis:600000}")
private long reportIntervalMillis;
@Scheduled(
fixedDelayString = "${edges.stats.report-interval-millis:600000}",
initialDelayString = "${edges.stats.report-interval-millis:600000}"
@ -80,13 +79,13 @@ public class EdgeStatsService {
long now = System.currentTimeMillis();
long ts = now - (now % reportIntervalMillis);
Map<EdgeId, MsgCounters> countersByEdge = statsCounterService.getCounterByEdge();
Map<EdgeId, Long> lagByEdgeId = kafkaAdmin.isPresent() ? getEdgeLagByEdgeId(countersByEdge) : Collections.emptyMap();
Map<EdgeId, MsgCounters> countersByEdgeSnapshot = new HashMap<>(statsCounterService.getCounterByEdge());
Map<EdgeId, MsgCounters> countersByEdgeSnapshot = new HashMap<>(statsCounterService.getMsgCountersByEdge());
boolean isKafkaStats = kafkaAdmin.isPresent();
Map<EdgeId, Long> lagByEdgeId = isKafkaStats ? getLagByEdgeId(countersByEdgeSnapshot) : Collections.emptyMap();
countersByEdgeSnapshot.forEach((edgeId, counters) -> {
TenantId tenantId = counters.getTenantId();
if (kafkaAdmin.isPresent()) {
if (isKafkaStats) {
counters.getMsgsLag().set(lagByEdgeId.getOrDefault(edgeId, 0L));
}
List<TsKvEntry> statsEntries = List.of(
@ -102,7 +101,7 @@ public class EdgeStatsService {
});
}
private Map<EdgeId, Long> getEdgeLagByEdgeId(Map<EdgeId, MsgCounters> countersByEdge) {
private Map<EdgeId, Long> getLagByEdgeId(Map<EdgeId, MsgCounters> countersByEdge) {
Map<EdgeId, String> edgeToTopicMap = countersByEdge.entrySet().stream()
.collect(Collectors.toMap(
Map.Entry::getKey,

2
application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java

@ -103,7 +103,7 @@ import org.thingsboard.server.dao.device.DeviceConnectivityConfiguration;
import org.thingsboard.server.dao.device.DeviceCredentialsService;
import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.mobile.MobileAppDao;
import org.thingsboard.server.dao.notification.NotificationSettingsService;
import org.thingsboard.server.dao.notification.NotificationTargetService;

2
application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java

@ -39,7 +39,7 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.dao.dashboard.DashboardService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService;
import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.resource.ResourceService;

2
application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java

@ -69,7 +69,7 @@ import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.ota.OtaPackageService;
import org.thingsboard.server.dao.resource.ResourceService;

2
application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/DefaultJwtSettingsValidator.java

@ -21,7 +21,7 @@ import org.apache.commons.lang3.StringUtils;
import org.bouncycastle.util.Arrays;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.security.model.JwtSettings;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import java.util.Base64;
import java.util.Optional;

2
application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java

@ -29,11 +29,11 @@ import org.thingsboard.server.common.data.security.model.mfa.account.AccountTwoF
import org.thingsboard.server.common.data.security.model.mfa.account.TwoFaAccountConfig;
import org.thingsboard.server.common.data.security.model.mfa.provider.TwoFaProviderConfig;
import org.thingsboard.server.common.data.security.model.mfa.provider.TwoFaProviderType;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.service.ConstraintValidator;
import org.thingsboard.server.dao.settings.AdminSettingsDao;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.dao.user.UserAuthSettingsDao;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.service.security.auth.mfa.TwoFactorAuthService;
import java.util.Comparator;

2
application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java

@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.security.model.SecuritySettings;
import org.thingsboard.server.common.data.security.model.UserPasswordPolicy;
import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.settings.SecuritySettingsService;
import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.queue.util.TbCoreComponent;

2
application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java

@ -50,7 +50,7 @@ import org.thingsboard.server.common.data.security.model.SecuritySettings;
import org.thingsboard.server.common.data.security.model.UserPasswordPolicy;
import org.thingsboard.server.common.data.security.model.mfa.PlatformTwoFaSettings;
import org.thingsboard.server.dao.audit.AuditLogService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.dao.settings.SecuritySettingsService;
import org.thingsboard.server.dao.user.UserService;

2
application/src/main/java/org/thingsboard/server/service/security/system/SystemSecurityService.java

@ -24,7 +24,7 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.security.model.UserPasswordPolicy;
import org.thingsboard.server.common.data.security.model.mfa.PlatformTwoFaSettings;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.service.security.model.SecurityUser;
public interface SystemSecurityService {

2
application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java

@ -31,7 +31,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.common.data.util.ThrowingRunnable;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.TbLogEntityActionService;

2
application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java

@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.protobuf.ByteString;
import org.thingsboard.server.exception.EntitiesLimitExceededException;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor;
@ -76,7 +77,6 @@ import org.thingsboard.server.dao.device.provision.ProvisionFailedException;
import org.thingsboard.server.dao.device.provision.ProvisionRequest;
import org.thingsboard.server.dao.device.provision.ProvisionResponse;
import org.thingsboard.server.dao.device.provision.ProvisionResponseStatus;
import org.thingsboard.server.dao.exception.EntitiesLimitExceededException;
import org.thingsboard.server.dao.ota.OtaPackageService;
import org.thingsboard.server.dao.queue.QueueService;
import org.thingsboard.server.dao.relation.RelationService;

12
application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java

@ -33,6 +33,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ArgumentIntervalProt
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto;
import org.thingsboard.server.gen.transport.TransportProtos.EntityIdProto;
import org.thingsboard.server.gen.transport.TransportProtos.GeofencingArgumentProto;
import org.thingsboard.server.gen.transport.TransportProtos.GeofencingZoneProto;
import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto;
@ -61,6 +62,7 @@ import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgume
import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalculatedFieldState;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.TreeMap;
@ -108,6 +110,7 @@ public class CalculatedFieldUtils {
case SINGLE_VALUE -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry));
case TS_ROLLING -> builder.addRollingValueArguments(toRollingArgumentProto(argName, (TsRollingArgumentEntry) argEntry));
case GEOFENCING -> builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry));
case PROPAGATION -> builder.addAllPropagationEntityIds(toPropagationEntityIdsProto((PropagationArgumentEntry) argEntry));
case RELATED_ENTITIES -> {
RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry;
Map<EntityId, ArgumentEntry> entityInputs = relatedEntitiesArgumentEntry.getEntityInputs();
@ -140,6 +143,10 @@ public class CalculatedFieldUtils {
return builder.build();
}
private static List<EntityIdProto> toPropagationEntityIdsProto(PropagationArgumentEntry argEntry) {
return argEntry.getEntityIds().stream().map(ProtoUtils::toProto).collect(Collectors.toList());
}
private static AlarmRuleStateProto toAlarmRuleStateProto(AlarmRuleState ruleState) {
return AlarmRuleStateProto.newBuilder()
.setSeverity(Optional.ofNullable(ruleState.getSeverity()).map(Enum::name).orElse(""))
@ -275,7 +282,10 @@ public class CalculatedFieldUtils {
state.getArguments().put(argProto.getKey(), fromRollingArgumentProto(argProto)));
case GEOFENCING -> proto.getGeofencingArgumentsList().forEach(argProto ->
state.getArguments().put(argProto.getArgName(), fromGeofencingArgumentProto(argProto)));
case PROPAGATION -> state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, new PropagationArgumentEntry());
case PROPAGATION -> {
List<EntityId> propagationEntityIds = proto.getPropagationEntityIdsList().stream().map(ProtoUtils::fromProto).toList();
state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, new PropagationArgumentEntry(propagationEntityIds));
}
case ALARM -> {
AlarmCalculatedFieldState alarmState = (AlarmCalculatedFieldState) state;
AlarmStateProto alarmStateProto = proto.getAlarmState();

2
application/src/main/java/org/thingsboard/server/utils/LwM2mObjectModelUtils.java

@ -26,7 +26,7 @@ import org.thingsboard.server.common.data.lwm2m.LwM2mInstance;
import org.thingsboard.server.common.data.lwm2m.LwM2mObject;
import org.thingsboard.server.common.data.lwm2m.LwM2mResourceObserve;
import org.thingsboard.server.common.data.util.TbDDFFileParser;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import java.io.ByteArrayInputStream;
import java.io.IOException;

68
application/src/main/resources/thingsboard.yml

@ -516,14 +516,14 @@ actors:
response_timeout_ms: "${ACTORS_RPC_RESPONSE_TIMEOUT_MS:30000}"
# Close transport session if RPC delivery timed out. If enabled, RPC will be reverted to the queued state.
# Note:
# - For MQTT transport:
# - QoS level 0: This feature does not apply, as no acknowledgment is expected, and therefore no timeout is triggered.
# - QoS level 1: This feature applies, as an acknowledgment is expected.
# - QoS level 2: Unsupported.
# - For CoAP transport:
# - Confirmable requests: This feature applies, as delivery confirmation is expected.
# - Non-confirmable requests: This feature does not apply, as no delivery acknowledgment is expected.
# - For HTTP and SNPM transports: RPC is considered delivered immediately, and there is no logic to await acknowledgment.
# <ul><li> For MQTT transport:
# <ul><li> QoS level 0: This feature does not apply, as no acknowledgment is expected, and therefore no timeout is triggered.</li>
# <li> QoS level 1: This feature applies, as an acknowledgment is expected.</li>
# <li> QoS level 2: Unsupported.</li></ul></li>
# <li> For CoAP transport:
# <ul><li> Confirmable requests: This feature applies, as delivery confirmation is expected.</li>
# <li> Non-confirmable requests: This feature does not apply, as no delivery acknowledgment is expected.</li></ul></li>
# <li> For HTTP and SNPM transports: RPC is considered delivered immediately, and there is no logic to await acknowledgment.</li></ul>
close_session_on_rpc_delivery_timeout: "${ACTORS_RPC_CLOSE_SESSION_ON_RPC_DELIVERY_TIMEOUT:false}"
statistics:
# Enable/disable actor statistics
@ -733,13 +733,13 @@ redis:
# if set false will be used pool config build from values of the pool config section
useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}"
sentinel:
# name of the master node
# Name of the master node
master: "${REDIS_MASTER:}"
# comma-separated list of "host:port" pairs of sentinels
# Comma-separated list of "host:port" pairs of sentinels
sentinels: "${REDIS_SENTINELS:}"
# password to authenticate with sentinel
# Password to authenticate with sentinel
password: "${REDIS_SENTINEL_PASSWORD:}"
# if set false will be used pool config build from values of the pool config section
# If set false will be used pool config build from values of the pool config section
useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}"
# db index
db: "${REDIS_DB:0}"
@ -1039,10 +1039,10 @@ transport:
activity:
# This property specifies the strategy for reporting activity events within each reporting period.
# The accepted values are 'FIRST', 'LAST', 'FIRST_AND_LAST' and 'ALL'.
# - 'FIRST': Only the first activity event in each reporting period is reported.
# - 'LAST': Only the last activity event in the reporting period is reported.
# - 'FIRST_AND_LAST': Both the first and last activity events in the reporting period are reported.
# - 'ALL': All activity events in the reporting period are reported.
# <ul><li> 'FIRST': Only the first activity event in each reporting period is reported.</li>
# <li> 'LAST': Only the last activity event in the reporting period is reported.</li>
# <li> 'FIRST_AND_LAST': Both the first and last activity events in the reporting period are reported.</li>
# <li> 'ALL': All activity events in the reporting period are reported.</li></ul>
reporting_strategy: "${TB_TRANSPORT_ACTIVITY_REPORTING_STRATEGY:LAST}"
json:
# Cast String data types to Numeric if possible when processing Telemetry/Attributes JSON
@ -1160,15 +1160,15 @@ transport:
dtls:
# RFC7925_RETRANSMISSION_TIMEOUT_IN_MILLISECONDS = 9000
retransmission_timeout: "${LWM2M_DTLS_RETRANSMISSION_TIMEOUT_MS:9000}"
# CoAP DTLS connection ID length for LWM2M. RFC 9146, Connection Identifier for DTLS 1.2
# Default: off
# LWM2M DTLS connection ID length for LWM2M. RFC 9146, Connection Identifier for DTLS 1.2
# Default: off. <br>
# Control usage of DTLS connection ID length (CID).
# - 'off' to deactivate it.
# - 'on' to activate Connection ID support (same as CID 0 or more 0).
# - A positive value defines generated CID size in bytes.
# - A value of 0 means we accept using CID but will not generate one for foreign peer (enables support but not for incoming traffic).
# - A value between 0 and <= 4: SingleNodeConnectionIdGenerator is used
# - A value that are > 4: MultiNodeConnectionIdGenerator is used
# <ul> <li> 'off' to deactivate it. </li>
# <li> 'on' to activate Connection ID support (same as CID 0 or more 0). </li>
# <li> A positive value defines generated CID size in bytes.</li>
# <li> A value of 0 means we accept using CID but will not generate one for foreign peer (enables support but not for incoming traffic). </li>
# <li> A value between 0 and <= 4: SingleNodeConnectionIdGenerator is used </li>
# <li> A value that are > 4: MultiNodeConnectionIdGenerator is used </li> </ul>
connection_id_length: "${LWM2M_DTLS_CONNECTION_ID_LENGTH:8}"
server:
# LwM2M Server ID
@ -1351,14 +1351,14 @@ coap:
# CoAP DTLS bind port
bind_port: "${COAP_DTLS_BIND_PORT:5684}"
# CoAP DTLS connection ID length. RFC 9146, Connection Identifier for DTLS 1.2
# Default: off
# Default: off. <br>
# Control usage of DTLS connection ID length (CID).
# - 'off' to deactivate it.
# - 'on' to activate Connection ID support (same as CID 0 or more 0).
# - A positive value defines generated CID size in bytes.
# - A value of 0 means we accept using CID but will not generate one for foreign peer (enables support but not for incoming traffic).
# - A value between 0 and <= 4: SingleNodeConnectionIdGenerator is used
# - A value that are > 4: MultiNodeConnectionIdGenerator is used
# <ul> <li> 'off' to deactivate it. </li>
# <li> 'on' to activate Connection ID support (same as CID 0 or more 0). </li>
# <li> A positive value defines generated CID size in bytes.</li>
# <li> A value of 0 means we accept using CID but will not generate one for foreign peer (enables support but not for incoming traffic).</li>
# <li> A value between 0 and <= 4: SingleNodeConnectionIdGenerator is used</li>
# <li> A value that are > 4: MultiNodeConnectionIdGenerator is used</li> </ul>
connection_id_length: "${COAP_DTLS_CONNECTION_ID_LENGTH:8}"
# Specify the MTU (Maximum Transmission Unit).
# Should be used if LAN MTU is not used, e.g. if IP tunnels are used or if the client uses a smaller value than the LAN MTU.
@ -1375,13 +1375,13 @@ coap:
# In order to negotiate smaller maximum fragment lengths,
# clients MAY include an extension of type "max_fragment_length" in the (extended) client hello.
# The "extension_data" field of this extension SHALL contain:
# enum {
# <pre> enum {
# 2^9(1) == 512,
# 2^10(2) == 1024,
# 2^11(3) == 2048,
# 2^12(4) == 4096,
# (255)
# } MaxFragmentLength;
# } MaxFragmentLength; </pre>
# TLS already requires clients and servers to support fragmentation of handshake messages.
max_fragment_length: "${COAP_DTLS_MAX_FRAGMENT_LENGTH:1024}"
# Server DTLS credentials
@ -1745,6 +1745,8 @@ queue:
print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}"
# Time to wait for the stats-loading requests to Kafka to finish
kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}"
# Topics cache TTL in milliseconds. 5 minutes by default
topics_cache_ttl_ms: "${TB_QUEUE_KAFKA_TOPICS_CACHE_TTL_MS:300000}"
partitions:
hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256
transport_api:

57
application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java

@ -1089,7 +1089,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
// Telemetry on device
doPost("/api/plugins/telemetry/DEVICE/" + device.getUuidId() + "/timeseries/unusedScope",
JacksonUtil.toJsonNode("{\"temperature\":12.5}")).andExpect(status().isOk());
JacksonUtil.toJsonNode("{\"temperature\":12.5, \"humidity\":85}")).andExpect(status().isOk());
// --- Build CF: PROPAGATION with expression ---
CalculatedField cf = new CalculatedField();
@ -1102,11 +1102,14 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
cfg.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE));
cfg.setApplyExpressionToResolvedArguments(true);
Argument arg = new Argument();
arg.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null));
cfg.setArguments(Map.of("t", arg));
Argument arg1 = new Argument();
arg1.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null));
cfg.setExpression("{\"testResult\": t * 2}");
Argument arg2 = new Argument();
arg2.setRefEntityKey(new ReferencedEntityKey("humidity", ArgumentType.TS_LATEST, null));
cfg.setArguments(Map.of("t", arg1, "h", arg2));
cfg.setExpression("return { testResult: (t + h) / 2};");
AttributesOutput output = new AttributesOutput();
output.setScope(AttributeScope.SERVER_SCOPE);
@ -1125,8 +1128,8 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
ArrayNode attrs2 = getServerAttributes(asset2.getId(), "testResult");
assertThat(attrs1).isNotNull();
assertThat(attrs2).isNotNull();
assertThat(attrs1.get(0).get("value").asDouble()).isEqualTo(25.0);
assertThat(attrs2.get(0).get("value").asDouble()).isEqualTo(25.0);
assertThat(attrs1.get(0).get("value").asDouble()).isEqualTo(48.75);
assertThat(attrs2.get(0).get("value").asDouble()).isEqualTo(48.75);
});
String deleteUrl = String.format("/api/v2/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s",
@ -1148,7 +1151,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
ArrayNode attrs2 = getServerAttributes(asset2.getId(), "testResult");
assertThat(attrs1).isNullOrEmpty();
assertThat(attrs2).isNotNull();
assertThat(attrs2.get(0).get("value").asDouble()).isEqualTo(50);
assertThat(attrs2.get(0).get("value").asDouble()).isEqualTo(55);
});
}
@ -1167,7 +1170,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
// Telemetry on device
long ts = System.currentTimeMillis() - 300000L;
postTelemetry(device.getId(), String.format("{\"ts\": %s, \"values\": {\"temperature\":12.5}}", ts));
postTelemetry(device.getId(), String.format("{\"ts\": %s, \"values\": {\"temperature\":12.5, \"humidity\":85}}", ts));
// --- Build CF: PROPAGATION without expression ---
CalculatedField cf = new CalculatedField();
@ -1180,9 +1183,12 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
cfg.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE));
cfg.setApplyExpressionToResolvedArguments(false); // arguments-only mode
Argument arg = new Argument();
arg.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null));
cfg.setArguments(Map.of("temperatureComputed", arg));
Argument arg1 = new Argument();
arg1.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null));
Argument arg2 = new Argument();
arg2.setRefEntityKey(new ReferencedEntityKey("humidity", ArgumentType.TS_LATEST, null));
cfg.setArguments(Map.of("temperatureComputed", arg1, "humidityComputed", arg2));
TimeSeriesOutput output = new TimeSeriesOutput();
output.setStrategy(new TimeSeriesImmediateOutputStrategy(0, true, true, true, true));
@ -1197,14 +1203,18 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
.atMost(TIMEOUT, TimeUnit.SECONDS)
.pollInterval(POLL_INTERVAL, TimeUnit.SECONDS)
.untilAsserted(() -> {
ObjectNode telemetry1 = getLatestTelemetry(asset1.getId(), "temperatureComputed");
ObjectNode telemetry2 = getLatestTelemetry(asset2.getId(), "temperatureComputed");
ObjectNode telemetry1 = getLatestTelemetry(asset1.getId(), "temperatureComputed,humidityComputed");
ObjectNode telemetry2 = getLatestTelemetry(asset2.getId(), "temperatureComputed,humidityComputed");
assertThat(telemetry1).isNotNull();
assertThat(telemetry2).isNotNull();
assertThat(telemetry1.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts));
assertThat(telemetry1.get("temperatureComputed").get(0).get("value").asDouble()).isEqualTo(12.5);
assertThat(telemetry1.get("humidityComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts));
assertThat(telemetry1.get("humidityComputed").get(0).get("value").asDouble()).isEqualTo(85);
assertThat(telemetry2.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts));
assertThat(telemetry2.get("temperatureComputed").get(0).get("value").asDouble()).isEqualTo(12.5);
assertThat(telemetry2.get("humidityComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts));
assertThat(telemetry2.get("humidityComputed").get(0).get("value").asDouble()).isEqualTo(85);
});
String deleteUrl = String.format("/api/v2/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s",
@ -1212,10 +1222,10 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
EntityRelation.CONTAINS_TYPE, device.getId().getId(), EntityType.DEVICE
);
doDelete(deleteUrl).andExpect(status().isOk());
doDelete("/api/plugins/telemetry/ASSET/" + asset1.getId() + "/timeseries/delete?keys=temperatureComputed&deleteAllDataForKeys=true").andExpect(status().isOk());
doDelete("/api/plugins/telemetry/ASSET/" + asset1.getId() + "/timeseries/delete?keys=temperatureComputed,humidityComputed&deleteAllDataForKeys=true").andExpect(status().isOk());
// Update telemetry on device
long newTs = System.currentTimeMillis() - 300000L;
// Update telemetry on the device
long newTs = ts + 300000L;
postTelemetry(device.getId(), String.format("{\"ts\": %s, \"values\": {\"temperature\":25}}", newTs));
// --- Assert propagated calculation (arguments-only mode after update) ---
@ -1223,13 +1233,18 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
.atMost(TIMEOUT, TimeUnit.SECONDS)
.pollInterval(POLL_INTERVAL, TimeUnit.SECONDS)
.untilAsserted(() -> {
ObjectNode telemetry1 = getLatestTelemetry(asset1.getId(), "temperatureComputed");
ObjectNode telemetry2 = getLatestTelemetry(asset2.getId(), "temperatureComputed");
ObjectNode telemetry1 = getLatestTelemetry(asset1.getId(), "temperatureComputed,humidityComputed");
ObjectNode telemetry2 = getLatestTelemetry(asset2.getId(), "temperatureComputed,humidityComputed");
assertThat(telemetry1).isNotNull();
assertThat(telemetry2).isNotNull();
assertThat(telemetry1.get("temperatureComputed").get(0).get("value")).isEqualTo(NullNode.instance);
assertThat(telemetry1.get("humidityComputed").get(0).get("value")).isEqualTo(NullNode.instance);
assertThat(telemetry2.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(newTs));
assertThat(telemetry2.get("temperatureComputed").get(0).get("value").asDouble()).isEqualTo(25);
// TS for humidity is not updated -> expected
assertThat(telemetry2.get("humidityComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts));
assertThat(telemetry2.get("humidityComputed").get(0).get("value").asDouble()).isEqualTo(85);
});
Asset asset3 = createAsset("Propagated Asset 3", null);
@ -1241,10 +1256,12 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
.atMost(TIMEOUT, TimeUnit.SECONDS)
.pollInterval(POLL_INTERVAL, TimeUnit.SECONDS)
.untilAsserted(() -> {
ObjectNode telemetry = getLatestTelemetry(asset3.getId(), "temperatureComputed");
ObjectNode telemetry = getLatestTelemetry(asset3.getId(), "temperatureComputed,humidityComputed");
assertThat(telemetry).isNotNull();
assertThat(telemetry.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(newTs));
assertThat(telemetry.get("temperatureComputed").get(0).get("value").asDouble()).isEqualTo(25);
assertThat(telemetry.get("humidityComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(newTs));
assertThat(telemetry.get("humidityComputed").get(0).get("value").asDouble()).isEqualTo(85);
});
}

2
application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java

@ -1209,7 +1209,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
Map<CalculatedFieldId, CalculatedFieldState> statesMap = (Map<CalculatedFieldId, CalculatedFieldState>) ReflectionTestUtils.getField(processor, "states");
Awaitility.await("CF state for entity actor ready to refresh dynamic arguments").atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> {
CalculatedFieldState calculatedFieldState = statesMap.get(cfId);
boolean isReady = calculatedFieldState != null && ((GeofencingCalculatedFieldState) calculatedFieldState).getLastDynamicArgumentsRefreshTs() <
boolean isReady = calculatedFieldState != null && ((GeofencingCalculatedFieldState) calculatedFieldState).getLastScheduledRefreshTs() <
System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(scheduledUpdateInterval);
log.warn("entityId {}, cfId {}, state ready to refresh == {}", entityId, cfId, isReady);
return isReady;

2
application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java

@ -50,7 +50,7 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.asset.AssetDao;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.service.DaoSqlTest;

2
application/src/test/java/org/thingsboard/server/controller/AssetProfileControllerTest.java

@ -42,7 +42,7 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.asset.AssetProfileDao;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.ArrayList;

2
application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java

@ -44,7 +44,7 @@ import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.customer.CustomerDao;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.ArrayList;

2
application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java

@ -51,7 +51,7 @@ import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.dashboard.DashboardDao;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.ArrayList;

2
application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java

@ -72,7 +72,7 @@ import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportColumn
import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportRequest;
import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportResult;
import org.thingsboard.server.dao.device.DeviceDao;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.service.DaoSqlTest;

2
application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java

@ -52,7 +52,7 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.device.DeviceProfileDao;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.ArrayList;

2
application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java

@ -72,7 +72,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.security.model.JwtSettings;
import org.thingsboard.server.dao.edge.EdgeDao;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.edge.imitator.EdgeImitator;

2
application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java

@ -63,7 +63,7 @@ import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.dao.entityview.EntityViewDao;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.service.DaoSqlTest;

2
application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java

@ -37,7 +37,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest;
import java.nio.ByteBuffer;

2
application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java

@ -47,7 +47,7 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.rule.RuleChainDao;
import org.thingsboard.server.dao.service.DaoSqlTest;

2
application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java

@ -48,7 +48,7 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.widget.WidgetType;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.ArrayList;

2
application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java

@ -50,7 +50,7 @@ import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.settings.StarredDashboardInfo;
import org.thingsboard.server.common.data.settings.UserDashboardsInfo;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.dao.user.UserDao;

2
application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java

@ -29,7 +29,7 @@ import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.ArrayList;

148
application/src/test/java/org/thingsboard/server/edge/EdgeStatsIntegrationTest.java

@ -0,0 +1,148 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.edge;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.extern.slf4j.Slf4j;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.edge.EdgeEvent;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.dao.edge.stats.EdgeStatsCounterService;
import org.thingsboard.server.dao.edge.stats.EdgeStatsKey;
import org.thingsboard.server.dao.edge.stats.MsgCounters;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.service.edge.stats.EdgeStatsService;
import java.time.Duration;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import static org.awaitility.Awaitility.await;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_ADDED;
import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_PERMANENTLY_FAILED;
import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_PUSHED;
import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_TMP_FAILED;
@DaoSqlTest
@Slf4j
public class EdgeStatsIntegrationTest extends AbstractEdgeTest {
private static final String STATISTICS_DEVICE_PROFILE = "STATISTICS";
private static final long EXPECTED_MSGS_ADDED = 6L;
private static final long EXPECTED_MSGS_PUSHED = 6L;
private static final long EXPECTED_MSGS_PERMANENTLY_FAILED = 0L;
private static final long EXPECTED_MSGS_TMP_FAILED = 0L;
@Autowired
private EdgeStatsService edgeStatsService;
@Autowired
private EdgeStatsCounterService statsCounterService;
@Test
public void testReportStats() throws Exception {
// GIVEN
simulateEdgeEventsAddedDownlinkPushed();
// Await Edge Counters Updated
await().atMost(10, TimeUnit.SECONDS).pollInterval(Duration.ofMillis(200)).untilAsserted(() -> {
MsgCounters counters = statsCounterService.getMsgCountersByEdge().get(edge.getId());
assertEquals(EXPECTED_MSGS_ADDED, counters.getMsgsAdded().get());
assertEquals(EXPECTED_MSGS_PUSHED, counters.getMsgsPushed().get());
assertEquals(EXPECTED_MSGS_PERMANENTLY_FAILED, counters.getMsgsPermanentlyFailed().get());
assertEquals(EXPECTED_MSGS_TMP_FAILED, counters.getMsgsTmpFailed().get());
});
Thread.sleep(1000);
// WHEN
edgeStatsService.reportStats();
// THEN
await().atMost(10, TimeUnit.SECONDS).pollInterval(Duration.ofMillis(200)).untilAsserted(() -> {
List<TsKvEntry> actualStats = fetchLatestStats();
assertEquals(EXPECTED_MSGS_ADDED, getStatsLongValue(actualStats, DOWNLINK_MSGS_ADDED));
assertEquals(EXPECTED_MSGS_PUSHED, getStatsLongValue(actualStats, DOWNLINK_MSGS_PUSHED));
assertEquals(EXPECTED_MSGS_PERMANENTLY_FAILED, getStatsLongValue(actualStats, DOWNLINK_MSGS_PERMANENTLY_FAILED));
assertEquals(EXPECTED_MSGS_TMP_FAILED, getStatsLongValue(actualStats, DOWNLINK_MSGS_TMP_FAILED));
});
}
private long getStatsLongValue(List<TsKvEntry> stats, EdgeStatsKey key) {
return stats.stream().filter(e -> e.getKey().equals(key.getKey())).findFirst().get().getLongValue().orElse(0L);
}
private List<TsKvEntry> fetchLatestStats() throws ExecutionException, InterruptedException {
return tsService.findLatest(
tenantId,
edge.getId(),
Arrays.stream(EdgeStatsKey.values()).map(EdgeStatsKey::getKey).toList()).get();
}
private void simulateEdgeEventsAddedDownlinkPushed() throws InterruptedException, ExecutionException {
statsCounterService.clear(edge.getId());
// Save device and assign to edge
// 2 DOWNLINK_MSGS_ADDED, EdgeEvents: [{DEVICE_PROFILE: ADDED}, {DEVICE: ASSIGNED_TO_EDGE}]
// 2 DOWNLINK_MSGS_PUSHED, Downlinks: [{deviceProfileUpdateMsg}, {deviceUpdateMsg, deviceProfileUpdateMsg, deviceCredentialsUpdateMsg}]
edgeImitator.expectMessageAmount(4);
Device savedDevice = saveDevice("StatisticDevice", STATISTICS_DEVICE_PROFILE);
doPost("/api/edge/" + edge.getUuidId() + "/device/" + savedDevice.getUuidId(), Device.class);
Assert.assertTrue(edgeImitator.waitForMessages());
// Save asset and assign to edge
// 1 DOWNLINK_MSGS_ADDED, EdgeEvents: [{ASSET: ASSIGNED_TO_EDGE}]
// 1 DOWNLINK_MSGS_PUSHED, Downlinks: [{assetUpdateMsg, assetProfileUpdateMsg}]
edgeImitator.expectMessageAmount(2);
Asset savedAsset = saveAsset("Edge Asset");
doPost("/api/edge/" + edge.getUuidId()
+ "/asset/" + savedAsset.getUuidId(), Asset.class);
Assert.assertTrue(edgeImitator.waitForMessages());
// Create customer and assign edge to the customer
// 2 DOWNLINK_MSGS_ADDED, EdgeEvents: [{CUSTOMER: ADDED}, {EDGE: ASSIGNED_TO_CUSTOMER}]
// 2 DOWNLINK_MSGS_PUSHED, Downlinks: [{customerUpdateMsg}, {edgeConfiguration}]
edgeImitator.expectMessageAmount(2);
Customer customer = new Customer();
customer.setTitle("Edge Customer");
Customer savedCustomer = doPost("/api/customer", customer, Customer.class);
doPost("/api/customer/" + savedCustomer.getUuidId()
+ "/edge/" + edge.getUuidId(), Edge.class);
Assert.assertTrue(edgeImitator.waitForMessages());
// Send device telemetry downlink for the device
// 1 DOWNLINK_MSGS_ADDED, EdgeEvents: [{DEVICE: TIMESERIES_UPDATED}]
// 1 DOWNLINK_MSGS_PUSHED, Downlinks: [{entityData}]
edgeImitator.expectMessageAmount(1);
String timeseriesData = "{\"data\":{\"temperature\":25},\"ts\":" + System.currentTimeMillis() + "}";
JsonNode timeseriesEntityData = JacksonUtil.toJsonNode(timeseriesData);
EdgeEvent edgeEvent = constructEdgeEvent(tenantId, edge.getId(), EdgeEventActionType.TIMESERIES_UPDATED, savedDevice.getId().getId(), EdgeEventType.DEVICE, timeseriesEntityData);
edgeEventService.saveAsync(edgeEvent).get();
Assert.assertTrue(edgeImitator.waitForMessages());
}
}

77
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationArgumentEntryTest.java

@ -104,19 +104,19 @@ public class PropagationArgumentEntryTest {
@Test
void testUpdateEntryWhenAdded() {
var added = new PropagationArgumentEntry();
added.setAdded(ENTITY_3_ID);
added.setAdded(List.of(ENTITY_3_ID));
boolean changed = entry.updateEntry(added);
assertThat(changed).isTrue();
assertThat(entry.getEntityIds()).containsExactlyInAnyOrder(ENTITY_1_ID, ENTITY_2_ID, ENTITY_3_ID);
assertThat(entry.getAdded()).isEqualTo(ENTITY_3_ID);
assertThat(entry.getAdded()).isEqualTo(List.of(ENTITY_3_ID));
}
@Test
void testUpdateEntryWhenAddedExistingEntity() {
var added = new PropagationArgumentEntry();
added.setAdded(ENTITY_2_ID);
added.setAdded(List.of(ENTITY_2_ID));
boolean changed = entry.updateEntry(added);
@ -149,6 +149,77 @@ public class PropagationArgumentEntryTest {
assertThat(entry.getRemoved()).isNull();
}
@Test
void testUpdateEntryWhenPartitionStateRestoreAddsMissingIds() {
var restore = new PropagationArgumentEntry(List.of(ENTITY_1_ID, ENTITY_2_ID, ENTITY_3_ID));
restore.setIgnoreRemovedEntities(true);
boolean changed = entry.updateEntry(restore);
assertThat(changed).isTrue();
assertThat(entry.getEntityIds()).containsExactlyInAnyOrder(ENTITY_1_ID, ENTITY_2_ID, ENTITY_3_ID);
assertThat(entry.getAdded()).containsExactly(ENTITY_3_ID);
assertThat(entry.getRemoved()).isNull();
assertThat(entry.isIgnoreRemovedEntities()).isFalse();
}
@Test
void testUpdateEntryWhenPartitionStateRestoreRemovesStaleIds() {
var restore = new PropagationArgumentEntry(List.of(ENTITY_1_ID));
restore.setIgnoreRemovedEntities(true);
boolean changed = entry.updateEntry(restore);
assertThat(changed).isFalse(); // expected no change, since we consider the removal of stale ids as no-op
assertThat(entry.getEntityIds()).containsExactlyInAnyOrder(ENTITY_1_ID);
assertThat(entry.getAdded()).isNull();
assertThat(entry.getRemoved()).isNull();
assertThat(entry.isIgnoreRemovedEntities()).isFalse();
}
@Test
void testUpdateEntryWhenPartitionStateRestoreAddsAndRemoves() {
var restore = new PropagationArgumentEntry(List.of(ENTITY_1_ID, ENTITY_3_ID));
restore.setIgnoreRemovedEntities(true);
boolean changed = entry.updateEntry(restore);
assertThat(changed).isTrue();
assertThat(entry.getEntityIds()).containsExactlyInAnyOrder(ENTITY_1_ID, ENTITY_3_ID);
assertThat(entry.getAdded()).containsExactly(ENTITY_3_ID);
assertThat(entry.getRemoved()).isNull();
assertThat(entry.isIgnoreRemovedEntities()).isFalse();
}
@Test
void testUpdateEntryWhenPartitionStateRestoreNoChanges() {
var restore = new PropagationArgumentEntry(List.of(ENTITY_1_ID, ENTITY_2_ID));
restore.setIgnoreRemovedEntities(true);
boolean changed = entry.updateEntry(restore);
assertThat(changed).isFalse();
assertThat(entry.getEntityIds()).containsExactlyInAnyOrder(ENTITY_1_ID, ENTITY_2_ID);
assertThat(entry.getAdded()).isNull();
assertThat(entry.getRemoved()).isNull();
assertThat(entry.isIgnoreRemovedEntities()).isFalse();
}
@Test
void testUpdateEntryWhenPartitionStateRestoreEmptySet() {
var restore = new PropagationArgumentEntry(List.of());
restore.setIgnoreRemovedEntities(true);
boolean changed = entry.updateEntry(restore);
assertThat(changed).isFalse(); // expected no change, since we consider the removal of stale ids as no-op
assertThat(entry.getEntityIds()).isEmpty();
assertThat(entry.getAdded()).isNull();
assertThat(entry.getRemoved()).isNull();
assertThat(entry.isIgnoreRemovedEntities()).isFalse();
}
@Test
@SuppressWarnings("unchecked")
void testToTbelCfArgWithValues() {

141
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java

@ -47,6 +47,7 @@ import org.thingsboard.server.common.data.relation.EntitySearchDirection;
import org.thingsboard.server.common.data.relation.RelationPathLevel;
import org.thingsboard.server.common.stats.DefaultStatsFactory;
import org.thingsboard.server.dao.usagerecord.ApiLimitService;
import org.thingsboard.server.service.cf.CalculatedFieldProcessingService;
import org.thingsboard.server.service.cf.PropagationCalculatedFieldResult;
import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult;
import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry;
@ -57,12 +58,17 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.stream.Stream;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration.PROPAGATION_CONFIG_ARGUMENT;
@ -70,17 +76,26 @@ import static org.thingsboard.server.common.data.cf.configuration.PropagationCal
public class PropagationCalculatedFieldStateTest {
private static final String TEMPERATURE_ARGUMENT_NAME = "t";
private static final String HUMIDITY_ARGUMENT_NAME = "h";
private static final String TEST_RESULT_EXPRESSION_KEY = "testResult";
private static final double TEMPERATURE_VALUE = 12.5;
private static final double HUMIDITY_VALUE = 85;
private static final PropagationArgumentEntry EMPTY_PROPAGATION_ARGUMENT = new PropagationArgumentEntry(Collections.emptyList());
private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("6c3513cb-85e7-4510-8746-1ba01859a8ce"));
private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("be960a50-c029-4698-b2ec-c56a543c561c"));
private final AssetId ASSET_ID_1 = new AssetId(UUID.fromString("d26f0e5b-7d7d-4a61-9f5e-08ab97b30734"));
private final AssetId ASSET_ID_2 = new AssetId(UUID.fromString("1933a317-4df5-4d36-9800-68aded74579b"));
private final SingleValueArgumentEntry singleValueArgEntry =
private final SingleValueArgumentEntry EMPTY_SINGLE_VALUE_ARGUMENT = new SingleValueArgumentEntry();
private final SingleValueArgumentEntry temperatureArgumentEntry =
new SingleValueArgumentEntry(System.currentTimeMillis(), new DoubleDataEntry("temperature", TEMPERATURE_VALUE), 99L);
private final SingleValueArgumentEntry humidityArgumentEntry =
new SingleValueArgumentEntry(System.currentTimeMillis(), new DoubleDataEntry("humidity", HUMIDITY_VALUE), 99L);
private final PropagationArgumentEntry propagationArgEntry =
new PropagationArgumentEntry(new ArrayList<>(List.of(ASSET_ID_2, ASSET_ID_1)));
@ -96,15 +111,19 @@ public class PropagationCalculatedFieldStateTest {
@MockitoBean
private ActorSystemContext actorSystemContext;
@MockitoBean
private CalculatedFieldProcessingService cfProcessingService;
@BeforeEach
void setUp() {
when(actorSystemContext.getTbelInvokeService()).thenReturn(tbelInvokeService);
when(actorSystemContext.getApiLimitService()).thenReturn(apiLimitService);
when(actorSystemContext.getCalculatedFieldProcessingService()).thenReturn(cfProcessingService);
when(apiLimitService.getLimit(any(), any())).thenReturn(1000L);
}
void initCtxAndState(boolean applyExpressionToResolvedArguments) {
ctx = new CalculatedFieldCtx(getCalculatedField(applyExpressionToResolvedArguments), actorSystemContext);
ctx = spy(new CalculatedFieldCtx(getCalculatedField(applyExpressionToResolvedArguments), actorSystemContext));
ctx.init();
state = new PropagationCalculatedFieldState(ctx.getEntityId());
@ -121,7 +140,7 @@ public class PropagationCalculatedFieldStateTest {
@Test
void testInitAddsRequiredArgument() {
initCtxAndState(false);
assertThat(state.getRequiredArguments()).containsExactlyInAnyOrder(TEMPERATURE_ARGUMENT_NAME, PROPAGATION_CONFIG_ARGUMENT);
assertThat(state.getRequiredArguments()).containsExactlyInAnyOrder(TEMPERATURE_ARGUMENT_NAME, HUMIDITY_ARGUMENT_NAME, PROPAGATION_CONFIG_ARGUMENT);
}
@Test
@ -133,7 +152,7 @@ public class PropagationCalculatedFieldStateTest {
private static Stream<ArgumentEntry> provideInvalidPropagationArgs() {
return Stream.of(
null,
new PropagationArgumentEntry(Collections.emptyList())
EMPTY_PROPAGATION_ARGUMENT
);
}
@ -143,7 +162,8 @@ public class PropagationCalculatedFieldStateTest {
initCtxAndState(false);
Map<String, ArgumentEntry> args = new HashMap<>();
args.put(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry); // Valid user arg
args.put(TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry); // Valid user arg
args.put(HUMIDITY_ARGUMENT_NAME, humidityArgumentEntry); // Valid user arg
if (propagationEntry != null) {
args.put(PROPAGATION_CONFIG_ARGUMENT, propagationEntry);
@ -155,19 +175,54 @@ public class PropagationCalculatedFieldStateTest {
}
@Test
void testIsReadyWhenPropagationArgHasEntities() {
void testIsReadyWithoutExpressionWhenAllArgumentsAreNotEmpty() {
initCtxAndState(false);
state.update(Map.of(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry, PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry), ctx);
Map<String, ArgumentEntry> updatedArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, humidityArgumentEntry,
PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry
);
state.update(updatedArgs, ctx);
assertThat(state.isReady()).isTrue();
assertThat(state.getReadinessStatus().errorMsg()).isNull();
}
@Test
void testIsReadyWithoutExpressionWhenAtLeastOneArgumentIsNotEmpty() {
initCtxAndState(false);
Map<String, ArgumentEntry> updatedArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, EMPTY_SINGLE_VALUE_ARGUMENT,
PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry);
state.update(updatedArgs, ctx);
assertThat(state.isReady()).isTrue();
assertThat(state.getReadinessStatus().errorMsg()).isNull();
}
@Test
void testIsNotReadyWithExpressionWhenAtLeastOneArgumentIsEmpty() {
initCtxAndState(true);
Map<String, ArgumentEntry> updatedArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, EMPTY_SINGLE_VALUE_ARGUMENT,
PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry);
state.update(updatedArgs, ctx);
assertThat(state.isReady()).isFalse();
assertThat(state.getReadinessStatus().errorMsg()).isEqualTo("Required arguments are missing: h");
}
@Test
void testPerformCalculationWithEmptyPropagationArg() throws Exception {
initCtxAndState(false);
state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, new PropagationArgumentEntry(Collections.emptyList()));
Map<String, ArgumentEntry> initArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, humidityArgumentEntry,
PROPAGATION_CONFIG_ARGUMENT, EMPTY_PROPAGATION_ARGUMENT);
state.update(initArgs, ctx);
assertThat(state.isReady()).isFalse();
// test empty propagation argument calculation
PropagationCalculatedFieldResult result = performCalculation();
assertThat(result).isNotNull();
@ -178,8 +233,12 @@ public class PropagationCalculatedFieldStateTest {
@Test
void testPerformCalculationWithArgumentsOnlyMode() throws Exception {
initCtxAndState(false);
state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry);
state.getArguments().put(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry);
Map<String, ArgumentEntry> initArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, EMPTY_SINGLE_VALUE_ARGUMENT,
PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry);
state.update(initArgs, ctx);
assertThat(state.isReady()).isTrue();
PropagationCalculatedFieldResult propagationResult = performCalculation();
@ -193,7 +252,7 @@ public class PropagationCalculatedFieldStateTest {
assertThat(result.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE);
ObjectNode expectedNode = JacksonUtil.newObjectNode();
JacksonUtil.addKvEntry(expectedNode, singleValueArgEntry.getKvEntryValue(), TEMPERATURE_ARGUMENT_NAME);
JacksonUtil.addKvEntry(expectedNode, temperatureArgumentEntry.getKvEntryValue(), TEMPERATURE_ARGUMENT_NAME);
assertThat(result.getResult()).isEqualTo(expectedNode);
}
@ -201,9 +260,13 @@ public class PropagationCalculatedFieldStateTest {
@Test
void testPerformCalculationWithExpressionResultMode() throws Exception {
initCtxAndState(true);
state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry);
state.getArguments().put(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry);
Map<String, ArgumentEntry> initArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, humidityArgumentEntry,
PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry
);
state.update(initArgs, ctx);
assertThat(state.isReady()).isTrue();
PropagationCalculatedFieldResult propagationResult = performCalculation();
assertThat(propagationResult).isNotNull();
@ -216,7 +279,7 @@ public class PropagationCalculatedFieldStateTest {
assertThat(result.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE);
ObjectNode expectedNode = JacksonUtil.newObjectNode();
expectedNode.put(TEST_RESULT_EXPRESSION_KEY, TEMPERATURE_VALUE * 2);
expectedNode.put(TEST_RESULT_EXPRESSION_KEY, (TEMPERATURE_VALUE + HUMIDITY_VALUE) / 2);
assertThat(result.getResult()).isEqualTo(expectedNode);
}
@ -224,18 +287,52 @@ public class PropagationCalculatedFieldStateTest {
@Test
void testPropagationWithUpdatedPropagationArgument() throws ExecutionException, InterruptedException {
initCtxAndState(false);
state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry);
state.getArguments().put(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry);
Map<String, ArgumentEntry> initArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, EMPTY_SINGLE_VALUE_ARGUMENT,
PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry
);
state.update(initArgs, ctx);
assertThat(state.isReady()).isTrue();
AssetId newEntityId = new AssetId(UUID.fromString("83e2c962-eeae-4708-984e-e6a24760f9c3"));
PropagationArgumentEntry propagationArgumentEntry = new PropagationArgumentEntry();
propagationArgumentEntry.setAdded(newEntityId);
propagationArgumentEntry.setAdded(List.of(newEntityId));
Map<String, ArgumentEntry> updated = state.update(Map.of(PROPAGATION_CONFIG_ARGUMENT, propagationArgumentEntry), ctx);
assertThat(updated).isNotNull().containsEntry(PROPAGATION_CONFIG_ARGUMENT, propagationArgumentEntry);
PropagationCalculatedFieldResult propagationCalculatedFieldResult = performCalculation(updated);
assertThat(propagationCalculatedFieldResult).isNotNull();
assertThat(propagationCalculatedFieldResult.getEntityIds()).isNotNull().containsExactly(newEntityId);
assertThat(propagationCalculatedFieldResult.getResult()).isNotNull();
assertThat(propagationCalculatedFieldResult.getResult().getResult()).isNotNull();
assertThat(propagationCalculatedFieldResult.getResult().getResult()).isEqualTo(JacksonUtil.newObjectNode().put(TEMPERATURE_ARGUMENT_NAME, TEMPERATURE_VALUE));
}
@Test
void testPropapagationStateInitWithRestoredSetToFalse() {
initCtxAndState(false);
verify(cfProcessingService, never()).fetchPropagationArgumentFromDb(any(), any());
verify(ctx, never()).scheduleReevaluation(anyLong(), any());
}
@Test
void testPropapagationStateInitWithRestoredSetToTrue() {
initCtxAndState(false);
Map<String, ArgumentEntry> initArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, humidityArgumentEntry,
PROPAGATION_CONFIG_ARGUMENT, new PropagationArgumentEntry(Collections.emptyList())
);
state.update(initArgs, ctx);
assertThat(state.isReady()).isFalse();
when(cfProcessingService.fetchPropagationArgumentFromDb(any(), any())).thenReturn(Optional.of(propagationArgEntry));
state.init(true);
verify(cfProcessingService).fetchPropagationArgumentFromDb(ctx, state.getEntityId());
verify(ctx).scheduleReevaluation(0L, state.getActorCtx());
}
private CalculatedField getCalculatedField(boolean applyExpressionToResolvedArguments) {
@ -260,8 +357,12 @@ public class PropagationCalculatedFieldStateTest {
ReferencedEntityKey tempKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null);
temperatureArg.setRefEntityKey(tempKey);
config.setArguments(Map.of(TEMPERATURE_ARGUMENT_NAME, temperatureArg));
config.setExpression("{" + TEST_RESULT_EXPRESSION_KEY + ": " + TEMPERATURE_ARGUMENT_NAME + " * 2}");
Argument humidityArg = new Argument();
ReferencedEntityKey humidityKey = new ReferencedEntityKey("humidity", ArgumentType.TS_LATEST, null);
humidityArg.setRefEntityKey(humidityKey);
config.setArguments(Map.of(TEMPERATURE_ARGUMENT_NAME, temperatureArg, HUMIDITY_ARGUMENT_NAME, humidityArg));
config.setExpression("return { " + TEST_RESULT_EXPRESSION_KEY + ": (" + TEMPERATURE_ARGUMENT_NAME + " + " + HUMIDITY_ARGUMENT_NAME + ") / 2};");
AttributesOutput output = new AttributesOutput();
output.setScope(AttributeScope.SERVER_SCOPE);

133
application/src/test/java/org/thingsboard/server/service/edge/EdgeStatsTest.java

@ -21,6 +21,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.test.util.ReflectionTestUtils;
@ -44,9 +45,11 @@ import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_ADDED;
@ -58,6 +61,16 @@ import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_T
@ExtendWith(MockitoExtension.class)
public class EdgeStatsTest {
private static final int TTL_DAYS = 30;
private static final long REPORT_INTERVAL_MILLIS = 600_000L;
private static final long EXPECTED_MSGS_ADDED = 5L;
private static final long EXPECTED_MSGS_PUSHED = 3L;
private static final long EXPECTED_MSGS_PERMANENTLY_FAILED = 1L;
private static final long EXPECTED_MSGS_TMP_FAILED = 0L;
private static final long EXPECTED_MSGS_LAG = 10L;
private static final long EXPECTED_MSGS_KAFKA_LAG = 15L;
@Mock
private TimeseriesService tsService;
@Mock
@ -66,108 +79,100 @@ public class EdgeStatsTest {
private EdgeStatsCounterService statsCounterService;
private EdgeStatsService edgeStatsService;
@Captor
private ArgumentCaptor<List<TsKvEntry>> captor;
private final TenantId tenantId = TenantId.fromUUID(UUID.randomUUID());
private final EdgeId edgeId = new EdgeId(UUID.randomUUID());
@BeforeEach
void setUp() {
edgeStatsService = new EdgeStatsService(
edgeStatsService = createEdgeStatsService(Optional.empty());
}
private EdgeStatsService createEdgeStatsService(Optional<KafkaAdmin> kafkaAdmin) {
EdgeStatsService service = new EdgeStatsService(
tsService,
statsCounterService,
topicService,
Optional.empty()
kafkaAdmin
);
ReflectionTestUtils.setField(edgeStatsService, "edgesStatsTtlDays", 30);
ReflectionTestUtils.setField(edgeStatsService, "reportIntervalMillis", 600_000L);
ReflectionTestUtils.setField(service, "edgesStatsTtlDays", TTL_DAYS);
ReflectionTestUtils.setField(service, "reportIntervalMillis", REPORT_INTERVAL_MILLIS);
return service;
}
@Test
public void testReportStatsSavesTelemetry() {
// given
// GIVEN
setupCounters();
// WHEN
edgeStatsService.reportStats();
// THEN
Map<String, Long> counters = verifyCounters();
Assertions.assertEquals(EXPECTED_MSGS_LAG, counters.get(DOWNLINK_MSGS_LAG.getKey()).longValue());
}
@Test
public void testReportStatsWithKafkaLag() {
// GIVEN
setupCounters();
setupKafkaLag();
// WHEN
edgeStatsService.reportStats();
// THEN
Map<String, Long> valuesByKey = verifyCounters();
Assertions.assertEquals(EXPECTED_MSGS_KAFKA_LAG, valuesByKey.get(DOWNLINK_MSGS_LAG.getKey()));
}
private void setupCounters() {
MsgCounters counters = new MsgCounters(tenantId);
counters.getMsgsAdded().set(5);
counters.getMsgsPushed().set(3);
counters.getMsgsPermanentlyFailed().set(1);
counters.getMsgsTmpFailed().set(0);
counters.getMsgsLag().set(10);
counters.getMsgsAdded().set(EXPECTED_MSGS_ADDED);
counters.getMsgsPushed().set(EXPECTED_MSGS_PUSHED);
counters.getMsgsPermanentlyFailed().set(EXPECTED_MSGS_PERMANENTLY_FAILED);
counters.getMsgsTmpFailed().set(EXPECTED_MSGS_TMP_FAILED);
counters.getMsgsLag().set(EXPECTED_MSGS_LAG);
ConcurrentHashMap<EdgeId, MsgCounters> countersByEdge = new ConcurrentHashMap<>();
countersByEdge.put(edgeId, counters);
when(statsCounterService.getCounterByEdge()).thenReturn(countersByEdge);
when(statsCounterService.getMsgCountersByEdge()).thenReturn(countersByEdge);
ArgumentCaptor<List<TsKvEntry>> captor = ArgumentCaptor.forClass(List.class);
when(tsService.save(eq(tenantId), eq(edgeId), captor.capture(), anyLong()))
.thenReturn(Futures.immediateFuture(mock(TimeseriesSaveResult.class)));
}
// when
edgeStatsService.reportStats();
private Map<String, Long> verifyCounters() {
verify(tsService, times(1)).save(eq(tenantId), eq(edgeId), anyList(), anyLong());
verify(statsCounterService, times(1)).clear(edgeId);
// then
List<TsKvEntry> entries = captor.getValue();
Assertions.assertEquals(5, entries.size());
Map<String, Long> valuesByKey = entries.stream()
.collect(Collectors.toMap(TsKvEntry::getKey, e -> e.getLongValue().orElse(-1L)));
Assertions.assertEquals(5L, valuesByKey.get(DOWNLINK_MSGS_ADDED.getKey()).longValue());
Assertions.assertEquals(3L, valuesByKey.get(DOWNLINK_MSGS_PUSHED.getKey()).longValue());
Assertions.assertEquals(1L, valuesByKey.get(DOWNLINK_MSGS_PERMANENTLY_FAILED.getKey()).longValue());
Assertions.assertEquals(0L, valuesByKey.get(DOWNLINK_MSGS_TMP_FAILED.getKey()).longValue());
Assertions.assertEquals(10L, valuesByKey.get(DOWNLINK_MSGS_LAG.getKey()).longValue());
verify(statsCounterService).clear(edgeId);
Assertions.assertEquals(EXPECTED_MSGS_ADDED, valuesByKey.get(DOWNLINK_MSGS_ADDED.getKey()).longValue());
Assertions.assertEquals(EXPECTED_MSGS_PUSHED, valuesByKey.get(DOWNLINK_MSGS_PUSHED.getKey()).longValue());
Assertions.assertEquals(EXPECTED_MSGS_PERMANENTLY_FAILED, valuesByKey.get(DOWNLINK_MSGS_PERMANENTLY_FAILED.getKey()).longValue());
Assertions.assertEquals(EXPECTED_MSGS_TMP_FAILED, valuesByKey.get(DOWNLINK_MSGS_TMP_FAILED.getKey()).longValue());
return valuesByKey;
}
@Test
public void testReportStatsWithKafkaLag() {
// given
MsgCounters counters = new MsgCounters(tenantId);
counters.getMsgsAdded().set(2);
counters.getMsgsPushed().set(2);
counters.getMsgsPermanentlyFailed().set(0);
counters.getMsgsTmpFailed().set(1);
counters.getMsgsLag().set(0);
ConcurrentHashMap<EdgeId, MsgCounters> countersByEdge = new ConcurrentHashMap<>();
countersByEdge.put(edgeId, counters);
// mocks
when(statsCounterService.getCounterByEdge()).thenReturn(countersByEdge);
private void setupKafkaLag() {
String topic = "edge-topic";
TopicPartitionInfo partitionInfo = new TopicPartitionInfo(topic, tenantId, 0, false);
when(topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId)).thenReturn(partitionInfo);
KafkaAdmin kafkaAdmin = mock(KafkaAdmin.class);
when(kafkaAdmin.getTotalLagForGroupsBulk(Set.of(topic)))
.thenReturn(Map.of(topic, 15L));
ArgumentCaptor<List<TsKvEntry>> captor = ArgumentCaptor.forClass(List.class);
when(tsService.save(eq(tenantId), eq(edgeId), captor.capture(), anyLong()))
.thenReturn(Futures.immediateFuture(mock(TimeseriesSaveResult.class)));
edgeStatsService = new EdgeStatsService(
tsService,
statsCounterService,
topicService,
Optional.of(kafkaAdmin)
);
ReflectionTestUtils.setField(edgeStatsService, "edgesStatsTtlDays", 30);
ReflectionTestUtils.setField(edgeStatsService, "reportIntervalMillis", 600_000L);
// when
edgeStatsService.reportStats();
// then
List<TsKvEntry> entries = captor.getValue();
Map<String, Long> valuesByKey = entries.stream()
.collect(Collectors.toMap(TsKvEntry::getKey, e -> e.getLongValue().orElse(-1L)));
.thenReturn(Map.of(topic, EXPECTED_MSGS_KAFKA_LAG));
Assertions.assertEquals(15L, valuesByKey.get(DOWNLINK_MSGS_LAG.getKey()));
verify(statsCounterService).clear(edgeId);
edgeStatsService = createEdgeStatsService(Optional.of(kafkaAdmin));
}
}

2
application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java

@ -60,7 +60,7 @@ import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.controller.AbstractControllerTest;
import org.thingsboard.server.dao.ai.AiModelService;
import org.thingsboard.server.dao.dashboard.DashboardService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.service.DaoSqlTest;

62
application/src/test/java/org/thingsboard/server/service/security/auth/pat/ApiKeyAuthenticationProviderTest.java

@ -15,18 +15,24 @@
*/
package org.thingsboard.server.service.security.auth.pat;
import org.apache.commons.lang3.RandomStringUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.pat.ApiKey;
import org.thingsboard.server.common.data.pat.ApiKeyInfo;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.controller.AbstractControllerTest;
import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.concurrent.TimeUnit;
import static org.awaitility.Awaitility.await;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID;
@ -101,6 +107,62 @@ public class ApiKeyAuthenticationProviderTest extends AbstractControllerTest {
doGetWithApiKey("/api/admin/featuresInfo").andExpect(status().isUnauthorized());
}
@Test
public void testUnauthorizedWhenUserCredentialsDisabled() throws Exception {
User newUser = new User();
newUser.setAuthority(Authority.TENANT_ADMIN);
newUser.setTenantId(tenantId);
newUser.setEmail("testUser" + RandomStringUtils.secure().nextAlphanumeric(10) + "@thingsboard.org");
newUser.setFirstName("Test");
newUser.setLastName("User");
User savedUser = createUser(newUser, "testPassword1");
ApiKeyInfo apiKeyInfo = new ApiKeyInfo();
apiKeyInfo.setDescription("Test API key for user credentials test");
apiKeyInfo.setEnabled(true);
apiKeyInfo.setUserId(savedUser.getId());
ApiKey testApiKey = doPost("/api/apiKey", apiKeyInfo, ApiKey.class);
setApiKey(testApiKey.getValue());
doGetWithApiKey("/api/admin/repositorySettings/exists").andExpect(status().isOk());
doPost("/api/user/" + savedUser.getId().getId() + "/userCredentialsEnabled?userCredentialsEnabled=false").andExpect(status().isOk());
await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> doGetWithApiKey("/api/admin/repositorySettings/exists").andExpect(status().isUnauthorized()));
resetApiKey();
doDelete("/api/apiKey/" + testApiKey.getId()).andExpect(status().isOk());
loginSysAdmin();
doDelete("/api/user/" + savedUser.getId().getId()).andExpect(status().isOk());
}
@Test
public void testUnauthorizedWhenUserDeleted() throws Exception {
User newUser = new User();
newUser.setAuthority(Authority.TENANT_ADMIN);
newUser.setTenantId(tenantId);
newUser.setEmail("testUser" + RandomStringUtils.secure().nextAlphanumeric(10) + "@thingsboard.org");
newUser.setFirstName("Test");
newUser.setLastName("User");
User savedUser = createUser(newUser, "testPassword1");
ApiKeyInfo apiKeyInfo = new ApiKeyInfo();
apiKeyInfo.setDescription("Test API key for user deletion test");
apiKeyInfo.setEnabled(true);
apiKeyInfo.setUserId(savedUser.getId());
ApiKey testApiKey = doPost("/api/apiKey", apiKeyInfo, ApiKey.class);
setApiKey(testApiKey.getValue());
doGetWithApiKey("/api/admin/repositorySettings/exists").andExpect(status().isOk());
loginSysAdmin();
doDelete("/api/user/" + savedUser.getId().getId()).andExpect(status().isOk());
await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> doGetWithApiKey("/api/admin/repositorySettings/exists").andExpect(status().isUnauthorized()));
}
private ApiKeyInfo constructApiKeyInfo() {
ApiKeyInfo apiKeyInfo = new ApiKeyInfo();
apiKeyInfo.setDescription("New API key description");

5
application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java

@ -119,7 +119,7 @@ class CalculatedFieldUtilsTest {
}
@Test
void toProtoAndFromProto_shouldCreatePropagationStateWithEmptyPropagationArgument() {
void toProtoAndFromProto_shouldCreatePropagationStateWithNotEmptyPropagationArgument() {
// given
CalculatedFieldEntityCtxId stateId = mock(CalculatedFieldEntityCtxId.class);
given(stateId.tenantId()).willReturn(TENANT_ID);
@ -158,8 +158,7 @@ class CalculatedFieldUtilsTest {
assertThat(propagationState.getEntityId()).isEqualTo(DEVICE_ID);
assertThat(propagationState.getArguments()).isNotNull();
assertThat(propagationState.getArguments().get(PROPAGATION_CONFIG_ARGUMENT)).isNotNull();
assertThat(propagationState.getArguments().get(PROPAGATION_CONFIG_ARGUMENT).isEmpty()).isTrue();
assertThat(propagationState.getArguments().get(PROPAGATION_CONFIG_ARGUMENT)).isEqualTo(propagationArgumentEntry);
assertThat(propagationState.getArguments().get("state")).isNotNull().isEqualTo(singleValueArgumentEntry);
assertThat(propagationState.getRequiredArguments()).isNull();
assertThat(propagationState.getReadinessStatus()).isNull();

14
common/data/src/main/java/org/thingsboard/server/common/data/device/profile/Lwm2mDeviceProfileTransportConfiguration.java

@ -17,10 +17,15 @@ package org.thingsboard.server.common.data.device.profile;
import lombok.Data;
import org.thingsboard.server.common.data.DeviceTransportType;
import org.thingsboard.server.common.data.device.data.PowerMode;
import org.thingsboard.server.common.data.device.profile.lwm2m.OtherConfiguration;
import org.thingsboard.server.common.data.device.profile.lwm2m.TelemetryMappingConfiguration;
import org.thingsboard.server.common.data.device.profile.lwm2m.bootstrap.LwM2MBootstrapServerCredential;
import static org.eclipse.leshan.core.LwM2m.Version.V1_0;
import static org.thingsboard.server.common.data.device.profile.lwm2m.TelemetryObserveStrategy.SINGLE;
import java.util.Collections;
import java.util.List;
@Data
@ -33,9 +38,18 @@ public class Lwm2mDeviceProfileTransportConfiguration implements DeviceProfileTr
private List<LwM2MBootstrapServerCredential> bootstrap;
private OtherConfiguration clientLwM2mSettings;
public Lwm2mDeviceProfileTransportConfiguration() {
updateDefault();
}
@Override
public DeviceTransportType getType() {
return DeviceTransportType.LWM2M;
}
private void updateDefault(){
this.setBootstrap(Collections.emptyList());
this.setClientLwM2mSettings(new OtherConfiguration(false,1, 1, 1, PowerMode.DRX, null, null, null, null, null, V1_0.toString()));
this.setObserveAttr(new TelemetryMappingConfiguration(Collections.emptyMap(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptyMap(), false, SINGLE));
}
}

5
dao/src/main/java/org/thingsboard/server/dao/exception/DataValidationException.java → common/data/src/main/java/org/thingsboard/server/exception/DataValidationException.java

@ -13,12 +13,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.exception;
package org.thingsboard.server.exception;
public class DataValidationException extends RuntimeException {
private static final long serialVersionUID = 7659985660312721830L;
public DataValidationException(String message) {
super(message);
}
@ -26,4 +24,5 @@ public class DataValidationException extends RuntimeException {
public DataValidationException(String message, Throwable cause) {
super(message, cause);
}
}

4
dao/src/main/java/org/thingsboard/server/dao/exception/EntitiesLimitExceededException.java → common/data/src/main/java/org/thingsboard/server/exception/EntitiesLimitExceededException.java

@ -13,14 +13,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.exception;
package org.thingsboard.server.exception;
import lombok.Getter;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.TenantId;
public class EntitiesLimitExceededException extends DataValidationException {
private static final long serialVersionUID = -9211462514373279196L;
@Getter
private final TenantId tenantId;
@ -36,4 +35,5 @@ public class EntitiesLimitExceededException extends DataValidationException {
this.entityType = entityType;
this.limit = limit;
}
}

51
common/data/src/main/java/org/thingsboard/server/exception/ThingsboardRuntimeException.java

@ -0,0 +1,51 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.exception;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
public class ThingsboardRuntimeException extends RuntimeException {
private ThingsboardErrorCode errorCode;
public ThingsboardRuntimeException() {
super();
}
public ThingsboardRuntimeException(ThingsboardErrorCode errorCode) {
this.errorCode = errorCode;
}
public ThingsboardRuntimeException(String message, ThingsboardErrorCode errorCode) {
super(message);
this.errorCode = errorCode;
}
public ThingsboardRuntimeException(String message, Throwable cause, ThingsboardErrorCode errorCode) {
super(message, cause);
this.errorCode = errorCode;
}
public ThingsboardRuntimeException(Throwable cause, ThingsboardErrorCode errorCode) {
super(cause);
this.errorCode = errorCode;
}
public ThingsboardErrorCode getErrorCode() {
return errorCode;
}
}

1
common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeRpcClient.java

@ -41,4 +41,5 @@ public interface EdgeRpcClient {
void sendDownlinkResponseMsg(DownlinkResponseMsg downlinkResponseMsg);
int getServerMaxInboundMessageSize();
}

1
common/proto/src/main/proto/queue.proto

@ -935,6 +935,7 @@ message CalculatedFieldStateProto {
int64 lastArgsUpdateTs = 7;
int64 lastMetricsEvalTs = 8;
repeated ArgumentIntervalProto aggregationArguments = 9;
repeated EntityIdProto propagationEntityIds = 10;
}
//Used to report session state to tb-Service and persist this state in the cache on the tb-Service level.

29
common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaAdmin.java

@ -58,15 +58,12 @@ public class KafkaAdmin {
private final TbKafkaSettings settings;
@Value("${queue.kafka.request.timeout.ms:30000}")
private int requestTimeoutMs;
@Value("${queue.kafka.topics_cache_ttl_ms:300000}") // 5 minutes by default
private int topicsCacheTtlMs;
private final LazyInitializer<AdminClient> adminClient;
private final CachedValue<Set<String>> topics;
public KafkaAdmin(@Lazy TbKafkaSettings settings) {
public KafkaAdmin(@Lazy TbKafkaSettings settings,
@Value("${queue.kafka.topics_cache_ttl_ms:300000}")
int topicsCacheTtlMs) {
this.settings = settings;
this.adminClient = LazyInitializer.<AdminClient>builder()
.setInitializer(() -> AdminClient.create(settings.toAdminProps()))
@ -91,7 +88,7 @@ public class KafkaAdmin {
NewTopic newTopic = new NewTopic(topic, partitions, settings.getReplicationFactor()).configs(properties);
try {
getClient().createTopics(List.of(newTopic)).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS);
getClient().createTopics(List.of(newTopic)).all().get(settings.getRequestTimeoutMs(), TimeUnit.MILLISECONDS);
topics.add(topic);
} catch (ExecutionException ee) {
log.trace("Failed to create topic {} with properties {}", topic, properties, ee);
@ -110,7 +107,7 @@ public class KafkaAdmin {
public void deleteTopic(String topic) {
log.debug("Deleting topic {}", topic);
try {
getClient().deleteTopics(List.of(topic)).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS);
getClient().deleteTopics(List.of(topic)).all().get(settings.getRequestTimeoutMs(), TimeUnit.MILLISECONDS);
} catch (Exception e) {
log.error("Failed to delete kafka topic [{}].", topic, e);
}
@ -122,7 +119,7 @@ public class KafkaAdmin {
public Set<String> listTopics() {
try {
Set<String> topics = getClient().listTopics().names().get(requestTimeoutMs, TimeUnit.MILLISECONDS);
Set<String> topics = getClient().listTopics().names().get(settings.getRequestTimeoutMs(), TimeUnit.MILLISECONDS);
log.trace("Listed topics: {}", topics);
return topics;
} catch (Exception e) {
@ -150,7 +147,7 @@ public class KafkaAdmin {
.collect(Collectors.toMap(tp -> tp, tp -> OffsetSpec.latest()));
Map<TopicPartition, ListOffsetsResult.ListOffsetsResultInfo> endOffsets =
getClient().listOffsets(latestOffsetsSpec).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS);
getClient().listOffsets(latestOffsetsSpec).all().get(settings.getRequestTimeoutMs(), TimeUnit.MILLISECONDS);
return committedOffsets.entrySet().stream()
.mapToLong(entry -> {
@ -169,7 +166,7 @@ public class KafkaAdmin {
@SneakyThrows
public Map<TopicPartition, OffsetAndMetadata> getConsumerGroupOffsets(String groupId) {
return getClient().listConsumerGroupOffsets(groupId).partitionsToOffsetAndMetadata().get(requestTimeoutMs, TimeUnit.MILLISECONDS);
return getClient().listConsumerGroupOffsets(groupId).partitionsToOffsetAndMetadata().get(settings.getRequestTimeoutMs(), TimeUnit.MILLISECONDS);
}
/**
@ -212,7 +209,7 @@ public class KafkaAdmin {
} else {
log.info("[{}] SHOULD alter topic offset [{}] less than old node group offset [{}]", tp, existingOffset.offset(), om.offset());
}
getClient().alterConsumerGroupOffsets(newGroupId, Map.of(tp, om)).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS);
getClient().alterConsumerGroupOffsets(newGroupId, Map.of(tp, om)).all().get(settings.getRequestTimeoutMs(), TimeUnit.MILLISECONDS);
log.info("[{}] altered new consumer groupId {}", tp, newGroupId);
break;
}
@ -229,7 +226,7 @@ public class KafkaAdmin {
return true;
}
List<TopicPartition> allPartitions = getClient().describeTopics(existingTopics).allTopicNames().get(requestTimeoutMs, TimeUnit.MILLISECONDS)
List<TopicPartition> allPartitions = getClient().describeTopics(existingTopics).allTopicNames().get(settings.getRequestTimeoutMs(), TimeUnit.MILLISECONDS)
.entrySet().stream()
.flatMap(entry -> {
String topic = entry.getKey();
@ -239,9 +236,9 @@ public class KafkaAdmin {
.toList();
Map<TopicPartition, ListOffsetsResult.ListOffsetsResultInfo> beginningOffsets = getClient().listOffsets(allPartitions.stream()
.collect(Collectors.toMap(partition -> partition, partition -> OffsetSpec.earliest()))).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS);
.collect(Collectors.toMap(partition -> partition, partition -> OffsetSpec.earliest()))).all().get(settings.getRequestTimeoutMs(), TimeUnit.MILLISECONDS);
Map<TopicPartition, ListOffsetsResult.ListOffsetsResultInfo> endOffsets = getClient().listOffsets(allPartitions.stream()
.collect(Collectors.toMap(partition -> partition, partition -> OffsetSpec.latest()))).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS);
.collect(Collectors.toMap(partition -> partition, partition -> OffsetSpec.latest()))).all().get(settings.getRequestTimeoutMs(), TimeUnit.MILLISECONDS);
for (TopicPartition partition : allPartitions) {
long beginningOffset = beginningOffsets.get(partition).offset();
@ -261,7 +258,7 @@ public class KafkaAdmin {
public void deleteConsumerGroup(String consumerGroupId) {
try {
getClient().deleteConsumerGroups(List.of(consumerGroupId)).all().get(requestTimeoutMs, TimeUnit.MILLISECONDS);
getClient().deleteConsumerGroups(List.of(consumerGroupId)).all().get(settings.getRequestTimeoutMs(), TimeUnit.MILLISECONDS);
} catch (Exception e) {
log.warn("Failed to delete consumer group {}", consumerGroupId, e);
}

1
common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java

@ -112,6 +112,7 @@ public class TbKafkaSettings {
@Value("${queue.kafka.fetch_max_bytes:134217728}")
private int fetchMaxBytes;
@Getter
@Value("${queue.kafka.request.timeout.ms:30000}")
private int requestTimeoutMs;

25
common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClientContextImpl.java

@ -360,20 +360,25 @@ public class LwM2mClientContextImpl implements LwM2mClientContext {
@Override
public Lwm2mDeviceProfileTransportConfiguration getProfile(Registration registration) {
UUID profileId = getClientByEndpoint(registration.getEndpoint()).getProfileId();
return doGetAndCache(profileId);
return profileId != null ? doGetAndCache(profileId) : null;
}
private Lwm2mDeviceProfileTransportConfiguration doGetAndCache(UUID profileId) {
Lwm2mDeviceProfileTransportConfiguration result = profiles.get(profileId);
if (result == null) {
log.debug("Fetching profile [{}]", profileId);
DeviceProfile deviceProfile = deviceProfileCache.get(new DeviceProfileId(profileId));
if (deviceProfile != null) {
result = profileUpdate(deviceProfile);
} else {
log.warn("Device profile was not found! Most probably device profile [{}] has been removed from the database.", profileId);
Lwm2mDeviceProfileTransportConfiguration result;
if (profileId != null) {
result = profiles.get(profileId);
if (result == null) {
log.debug("Fetching profile [{}]", profileId);
DeviceProfile deviceProfile = deviceProfileCache.get(new DeviceProfileId(profileId));
if (deviceProfile != null) {
result = profileUpdate(deviceProfile);
} else {
log.warn("Device profile was not found! Most probably device profile [{}] has been removed from the database.", profileId);
}
}
} else {
log.warn("Device profile not found! The device profile ID is null. Return Lwm2mDeviceProfileTransportConfiguration with default.");
result = new Lwm2mDeviceProfileTransportConfiguration();
}
return result;
}

58
common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/ota/DefaultLwM2MOtaUpdateService.java

@ -201,35 +201,39 @@ public class DefaultLwM2MOtaUpdateService extends LwM2MExecutorAwareService impl
}
var clientSettings = clientContext.getProfile(client.getRegistration()).getClientLwM2mSettings();
initFwStrategy(client, clientSettings);
initSwStrategy(client, clientSettings);
if (!attributesToFetch.isEmpty()) {
var future = attributesService.getSharedAttributes(client, attributesToFetch);
DonAsynchron.withCallback(future, attrs -> {
if (fwInfo.isSupported()) {
Optional<String> newFwTitle = getAttributeValue(attrs, FIRMWARE_TITLE);
Optional<String> newFwVersion = getAttributeValue(attrs, FIRMWARE_VERSION);
Optional<String> newFwTag = getAttributeValue(attrs, FIRMWARE_TAG);
Optional<String> newFwUrl = getAttributeValue(attrs, FIRMWARE_URL);
if (newFwTitle.isPresent() && newFwVersion.isPresent() && !isOtaDownloading(client) && !UPDATING.equals(fwInfo.status)) {
onTargetFirmwareUpdate(client, newFwTitle.get(), newFwVersion.get(), newFwUrl, newFwTag);
if (clientSettings != null) {
initFwStrategy(client, clientSettings);
initSwStrategy(client, clientSettings);
if (!attributesToFetch.isEmpty()) {
var future = attributesService.getSharedAttributes(client, attributesToFetch);
DonAsynchron.withCallback(future, attrs -> {
if (fwInfo.isSupported()) {
Optional<String> newFwTitle = getAttributeValue(attrs, FIRMWARE_TITLE);
Optional<String> newFwVersion = getAttributeValue(attrs, FIRMWARE_VERSION);
Optional<String> newFwTag = getAttributeValue(attrs, FIRMWARE_TAG);
Optional<String> newFwUrl = getAttributeValue(attrs, FIRMWARE_URL);
if (newFwTitle.isPresent() && newFwVersion.isPresent() && !isOtaDownloading(client) && !UPDATING.equals(fwInfo.status)) {
onTargetFirmwareUpdate(client, newFwTitle.get(), newFwVersion.get(), newFwUrl, newFwTag);
}
}
}
if (swInfo.isSupported()) {
Optional<String> newSwTitle = getAttributeValue(attrs, SOFTWARE_TITLE);
Optional<String> newSwVersion = getAttributeValue(attrs, SOFTWARE_VERSION);
Optional<String> newSwTag = getAttributeValue(attrs, SOFTWARE_TAG);
Optional<String> newSwUrl = getAttributeValue(attrs, SOFTWARE_URL);
if (newSwTitle.isPresent() && newSwVersion.isPresent()) {
onTargetSoftwareUpdate(client, newSwTitle.get(), newSwVersion.get(), newSwUrl, newSwTag);
if (swInfo.isSupported()) {
Optional<String> newSwTitle = getAttributeValue(attrs, SOFTWARE_TITLE);
Optional<String> newSwVersion = getAttributeValue(attrs, SOFTWARE_VERSION);
Optional<String> newSwTag = getAttributeValue(attrs, SOFTWARE_TAG);
Optional<String> newSwUrl = getAttributeValue(attrs, SOFTWARE_URL);
if (newSwTitle.isPresent() && newSwVersion.isPresent()) {
onTargetSoftwareUpdate(client, newSwTitle.get(), newSwVersion.get(), newSwUrl, newSwTag);
}
}
}
}, throwable -> {
if (fwInfo.isSupported()) {
update(fwInfo);
}
}, executor);
}, throwable -> {
if (fwInfo.isSupported()) {
update(fwInfo);
}
}, executor);
}
}
}

13
common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisRegistrationStore.java

@ -750,11 +750,14 @@ public class TbLwM2mRedisRegistrationStore implements RegistrationStore, Startab
System.currentTimeMillis(), 0, cleanLimit);
for (byte[] endpoint : endpointsExpired) {
Registration r = deserializeReg(connection.get(toEndpointKey(endpoint)));
if (!r.isAlive(gracePeriod)) {
Deregistration dereg = removeRegistration(connection, r.getId(), true);
if (dereg != null)
expirationListener.registrationExpired(dereg.getRegistration(), dereg.getObservations());
byte[] data = connection.get(toEndpointKey(endpoint));
if (data != null && data.length > 0) {
Registration r = deserializeReg(data);
if (!r.isAlive(gracePeriod)) {
Deregistration dereg = removeRegistration(connection, r.getId(), true);
if (dereg != null)
expirationListener.registrationExpired(dereg.getRegistration(), dereg.getObservations());
}
}
}
} catch (Exception e) {

22
common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2mUplinkMsgHandler.java

@ -486,13 +486,18 @@ public class DefaultLwM2mUplinkMsgHandler extends LwM2MExecutorAwareService impl
*/
private void initClientTelemetry(LwM2mClient lwM2MClient) {
Lwm2mDeviceProfileTransportConfiguration profile = clientContext.getProfile(lwM2MClient.getRegistration());
Set<String> supportedObjects = clientContext.getSupportedIdVerInClient(lwM2MClient);
if (supportedObjects != null && !supportedObjects.isEmpty()) {
this.sendInitObserveRequests(lwM2MClient, profile, supportedObjects);
this.sendReadRequests(lwM2MClient, profile, supportedObjects);
this.sendWriteAttributeRequests(lwM2MClient, profile, supportedObjects);
if (profile != null) {
Set<String> supportedObjects = clientContext.getSupportedIdVerInClient(lwM2MClient);
if (supportedObjects != null && !supportedObjects.isEmpty()) {
this.sendInitObserveRequests(lwM2MClient, profile, supportedObjects);
this.sendReadRequests(lwM2MClient, profile, supportedObjects);
this.sendWriteAttributeRequests(lwM2MClient, profile, supportedObjects);
// Removed. Used only for debug.
// this.sendDiscoverRequests(lwM2MClient, profile, supportedObjects);
}
} else {
log.warn("[{}] Failed to process initClientTelemetry! Profile is null. Update procedure may not have completed after reboot yet", lwM2MClient.getEndpoint());
logService.log(lwM2MClient, "Failed to process initClientTelemetry. Profile is null. Update procedure may not have completed after reboot yet");
}
}
@ -1028,7 +1033,7 @@ public class DefaultLwM2mUplinkMsgHandler extends LwM2MExecutorAwareService impl
});
}
private void updateValueOta(List<LwM2mClient> clients, Lwm2mDeviceProfileTransportConfiguration oldProfile, Lwm2mDeviceProfileTransportConfiguration newProfile) {
private void updateValueOta(List<LwM2mClient> clients, Lwm2mDeviceProfileTransportConfiguration newProfile, Lwm2mDeviceProfileTransportConfiguration oldProfile) {
OtherConfiguration newLwM2mSettings = newProfile.getClientLwM2mSettings();
OtherConfiguration oldLwM2mSettings = oldProfile.getClientLwM2mSettings();
if (!newLwM2mSettings.getFwUpdateStrategy().equals(oldLwM2mSettings.getFwUpdateStrategy())
@ -1110,6 +1115,9 @@ public class DefaultLwM2mUplinkMsgHandler extends LwM2MExecutorAwareService impl
v -> attributesService.onAttributesUpdate(lwM2MClient, v, logFailedUpdateOfNonChangedValue),
t -> log.error("[{}] Failed to get attributes", lwM2MClient.getEndpoint(), t),
executor);
} else {
log.warn("[{}] Failed to process initAttributes! Profile is null. Update procedure may not have completed after reboot yet", lwM2MClient.getEndpoint());
logService.log(lwM2MClient, "Failed to process initAttributes. Profile is null. Update procedure may not have completed after reboot yet");
}
}
@ -1119,7 +1127,7 @@ public class DefaultLwM2mUplinkMsgHandler extends LwM2MExecutorAwareService impl
private Map<String, String> getNamesFromProfileForSharedAttributes(LwM2mClient lwM2MClient) {
Lwm2mDeviceProfileTransportConfiguration profile = clientContext.getProfile(lwM2MClient.getRegistration());
return profile.getObserveAttr().getKeyName();
return profile != null ? profile.getObserveAttr().getKeyName() : Collections.emptyMap();
}
public LwM2MTransportServerConfig getConfig() {

2
dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java

@ -64,7 +64,7 @@ import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.ConstraintValidator;
import org.thingsboard.server.dao.tenant.TenantService;

2
dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java

@ -37,7 +37,7 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.dao.entity.CachedVersionedEntityService;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.service.DataValidator;
import org.thingsboard.server.dao.service.PaginatedRemover;

2
dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java

@ -53,7 +53,7 @@ import org.thingsboard.server.dao.entity.EntityCountService;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DataValidator;
import org.thingsboard.server.dao.service.PaginatedRemover;
import org.thingsboard.server.dao.sql.JpaExecutorService;

2
dao/src/main/java/org/thingsboard/server/dao/component/BaseComponentDescriptorService.java

@ -30,7 +30,7 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.plugin.ComponentDescriptor;
import org.thingsboard.server.common.data.plugin.ComponentScope;
import org.thingsboard.server.common.data.plugin.ComponentType;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.service.DataValidator;
import org.thingsboard.server.dao.service.Validator;

2
dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java

@ -45,7 +45,7 @@ import org.thingsboard.server.dao.entity.AbstractCachedEntityService;
import org.thingsboard.server.dao.entity.EntityCountService;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.service.DataValidator;
import org.thingsboard.server.dao.service.PaginatedRemover;

2
dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java

@ -50,7 +50,7 @@ import org.thingsboard.server.dao.entity.EntityCountService;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.dao.service.DataValidator;

2
dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsServiceImpl.java

@ -43,9 +43,9 @@ import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.msg.EncryptionUtil;
import org.thingsboard.server.dao.entity.AbstractCachedEntityService;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException;
import org.thingsboard.server.dao.service.validator.DeviceCredentialsDataValidator;
import org.thingsboard.server.exception.DataValidationException;
import java.util.Objects;

2
dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java

@ -46,11 +46,11 @@ import org.thingsboard.server.common.msg.EncryptionUtil;
import org.thingsboard.server.dao.entity.CachedVersionedEntityService;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.service.PaginatedRemover;
import org.thingsboard.server.dao.service.Validator;
import org.thingsboard.server.dao.service.validator.DeviceProfileDataValidator;
import org.thingsboard.server.exception.DataValidationException;
import java.io.ByteArrayInputStream;
import java.security.cert.Certificate;

2
dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java

@ -80,7 +80,7 @@ import org.thingsboard.server.dao.event.EventService;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.service.PaginatedRemover;
import org.thingsboard.server.dao.service.validator.DeviceDataValidator;

2
dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java

@ -66,7 +66,7 @@ import org.thingsboard.server.dao.entity.EntityCountService;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.service.DataValidator;

3
dao/src/main/java/org/thingsboard/server/dao/edge/PostgresEdgeEventService.java

@ -68,7 +68,8 @@ public class PostgresEdgeEventService extends BaseEdgeEventService {
Futures.addCallback(saveFuture, new FutureCallback<>() {
@Override
public void onSuccess(Void result) {
statsCounterService.ifPresent(statsCounterService -> statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edgeEvent.getTenantId(), edgeEvent.getEdgeId(), 1));
statsCounterService.ifPresent(statsCounterService ->
statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edgeEvent.getTenantId(), edgeEvent.getEdgeId(), 1));
eventPublisher.publishEvent(SaveEntityEvent.builder()
.tenantId(edgeEvent.getTenantId())
.entityId(edgeEvent.getEdgeId())

15
dao/src/main/java/org/thingsboard/server/dao/edge/stats/EdgeStatsCounterService.java

@ -24,13 +24,13 @@ import org.thingsboard.server.common.data.id.TenantId;
import java.util.concurrent.ConcurrentHashMap;
@ConditionalOnProperty(prefix = "edges.stats", name = "enabled", havingValue = "true", matchIfMissing = false)
@ConditionalOnProperty(prefix = "edges.stats", name = "enabled", havingValue = "true")
@Service
@Slf4j
@Getter
public class EdgeStatsCounterService {
private final ConcurrentHashMap<EdgeId, MsgCounters> counterByEdge = new ConcurrentHashMap<>();
private final ConcurrentHashMap<EdgeId, MsgCounters> msgCountersByEdge = new ConcurrentHashMap<>();
public void recordEvent(EdgeStatsKey type, TenantId tenantId, EdgeId edgeId, long value) {
MsgCounters counters = getOrCreateCounters(tenantId, edgeId);
@ -39,19 +39,16 @@ public class EdgeStatsCounterService {
case DOWNLINK_MSGS_PUSHED -> counters.getMsgsPushed().addAndGet(value);
case DOWNLINK_MSGS_PERMANENTLY_FAILED -> counters.getMsgsPermanentlyFailed().addAndGet(value);
case DOWNLINK_MSGS_TMP_FAILED -> counters.getMsgsTmpFailed().addAndGet(value);
case DOWNLINK_MSGS_LAG -> counters.getMsgsLag().set(value);
}
}
public void setDownlinkMsgsLag(TenantId tenantId, EdgeId edgeId, long value) {
getOrCreateCounters(tenantId, edgeId).getMsgsLag().set(value);
public MsgCounters getOrCreateCounters(TenantId tenantId, EdgeId edgeId) {
return msgCountersByEdge.computeIfAbsent(edgeId, id -> new MsgCounters(tenantId));
}
public void clear(EdgeId edgeId) {
counterByEdge.remove(edgeId);
}
public MsgCounters getOrCreateCounters(TenantId tenantId, EdgeId edgeId) {
return counterByEdge.computeIfAbsent(edgeId, id -> new MsgCounters(tenantId));
msgCountersByEdge.remove(edgeId);
}
}

2
dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java

@ -44,7 +44,7 @@ import org.thingsboard.server.dao.alarm.AlarmService;
import org.thingsboard.server.dao.cf.CalculatedFieldService;
import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.housekeeper.CleanUpService;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;

2
dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java

@ -51,7 +51,7 @@ import org.thingsboard.server.dao.entity.CachedVersionedEntityService;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.PaginatedRemover;
import org.thingsboard.server.dao.service.validator.EntityViewDataValidator;
import org.thingsboard.server.dao.sql.JpaExecutorService;

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save