Browse Source

UI: merge with master and resolve conflict

pull/14625/head
ArtemDzhereleiko 7 hours ago
parent
commit
91aca5eaed
  1. 95
      application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java
  2. 4
      application/src/main/java/org/thingsboard/server/controller/BaseController.java
  3. 2
      application/src/main/java/org/thingsboard/server/controller/EdgeController.java
  4. 1
      application/src/main/java/org/thingsboard/server/exception/ThingsboardErrorResponseHandler.java
  5. 24
      application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java
  6. 4
      application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java
  7. 7
      application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java
  8. 39
      application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java
  9. 10
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java
  10. 51
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java
  11. 15
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java
  12. 32
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java
  13. 40
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java
  14. 4
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java
  15. 2
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java
  16. 19
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java
  17. 26
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/ScheduledRefreshSupported.java
  18. 46
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java
  19. 67
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java
  20. 1
      application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java
  21. 133
      application/src/main/java/org/thingsboard/server/service/edge/EdgeMsgConstructorUtils.java
  22. 17
      application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java
  23. 1
      application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java
  24. 3
      application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java
  25. 8
      application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java
  26. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/ai/AiModelEdgeProcessor.java
  27. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java
  28. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessor.java
  29. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/cf/CalculatedFieldEdgeProcessor.java
  30. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java
  31. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java
  32. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessor.java
  33. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java
  34. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java
  35. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java
  36. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java
  37. 13
      application/src/main/java/org/thingsboard/server/service/edge/stats/EdgeStatsService.java
  38. 2
      application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java
  39. 2
      application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java
  40. 13
      application/src/main/java/org/thingsboard/server/service/install/SqlEntityDatabaseSchemaService.java
  41. 2
      application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java
  42. 2
      application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/DefaultJwtSettingsValidator.java
  43. 2
      application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java
  44. 2
      application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java
  45. 2
      application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java
  46. 2
      application/src/main/java/org/thingsboard/server/service/security/system/SystemSecurityService.java
  47. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java
  48. 18
      application/src/main/java/org/thingsboard/server/service/system/SystemPatchApplier.java
  49. 2
      application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java
  50. 25
      application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java
  51. 2
      application/src/main/java/org/thingsboard/server/utils/LwM2mObjectModelUtils.java
  52. 135
      application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java
  53. 2
      application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java
  54. 2
      application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java
  55. 2
      application/src/test/java/org/thingsboard/server/controller/AssetProfileControllerTest.java
  56. 2
      application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java
  57. 2
      application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java
  58. 2
      application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java
  59. 2
      application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java
  60. 2
      application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java
  61. 2
      application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java
  62. 2
      application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java
  63. 2
      application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java
  64. 2
      application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java
  65. 2
      application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java
  66. 2
      application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java
  67. 25
      application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java
  68. 13
      application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java
  69. 12
      application/src/test/java/org/thingsboard/server/edge/DeviceProfileEdgeTest.java
  70. 148
      application/src/test/java/org/thingsboard/server/edge/EdgeStatsIntegrationTest.java
  71. 4
      application/src/test/java/org/thingsboard/server/edge/TelemetryEdgeTest.java
  72. 77
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationArgumentEntryTest.java
  73. 141
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java
  74. 4
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesAggregationCalculatedFieldStateTest.java
  75. 98
      application/src/test/java/org/thingsboard/server/service/edge/EdgeMsgConstructorUtilsTest.java
  76. 133
      application/src/test/java/org/thingsboard/server/service/edge/EdgeStatsTest.java
  77. 2
      application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java
  78. 66
      application/src/test/java/org/thingsboard/server/transport/lwm2m/ota/AbstractOtaLwM2MIntegrationTest.java
  79. 51
      application/src/test/java/org/thingsboard/server/transport/lwm2m/ota/sql/Ota5LwM2MIntegrationTest.java
  80. 50
      application/src/test/java/org/thingsboard/server/transport/lwm2m/ota/sql/Ota9LwM2MIntegrationTest.java
  81. 5
      application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java
  82. 5
      common/data/src/main/java/org/thingsboard/server/exception/DataValidationException.java
  83. 4
      common/data/src/main/java/org/thingsboard/server/exception/EntitiesLimitExceededException.java
  84. 51
      common/data/src/main/java/org/thingsboard/server/exception/ThingsboardRuntimeException.java
  85. 1
      common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeRpcClient.java
  86. 1
      common/proto/src/main/proto/queue.proto
  87. 3
      common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttSslHandlerProvider.java
  88. 31
      common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/TbMqttSslTransportComponent.java
  89. 6
      common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/TbMqttTransportComponent.java
  90. 2
      dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java
  91. 2
      dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java
  92. 2
      dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java
  93. 2
      dao/src/main/java/org/thingsboard/server/dao/component/BaseComponentDescriptorService.java
  94. 2
      dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java
  95. 2
      dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java
  96. 2
      dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsServiceImpl.java
  97. 2
      dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java
  98. 2
      dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java
  99. 2
      dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java
  100. 3
      dao/src/main/java/org/thingsboard/server/dao/edge/PostgresEdgeEventService.java

95
application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java

@ -41,6 +41,7 @@ import org.thingsboard.server.common.msg.CalculatedFieldStatePartitionRestoreMsg
import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.util.ProtoUtils;
import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto;
import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto;
@ -53,6 +54,7 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState;
import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry;
import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry;
import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState;
import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState;
import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry;
@ -75,6 +77,7 @@ import java.util.stream.Collectors;
import static org.thingsboard.server.common.data.DataConstants.REEVALUATION_MSG;
import static org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration.PROPAGATION_CONFIG_ARGUMENT;
import static org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry.getValueForTsRecord;
import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createStateByType;
/**
@ -243,7 +246,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
}
if (state instanceof PropagationCalculatedFieldState propagationState) {
PropagationArgumentEntry entry = new PropagationArgumentEntry();
entry.setAdded(msg.getRelatedEntityId());
entry.setAdded(List.of(msg.getRelatedEntityId()));
updatedArgs = propagationState.update(Map.of(PROPAGATION_CONFIG_ARGUMENT, entry), ctx);
}
if (CollectionsUtil.isEmpty(updatedArgs)) {
@ -422,19 +425,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
if (state == null) {
state = createState(ctx);
justRestored = true;
} else if (ctx.shouldFetchRelationQueryDynamicArgumentsFromDb(state)) {
log.debug("[{}][{}] Going to update dynamic arguments for CF.", entityId, ctx.getCfId());
try {
Map<String, ArgumentEntry> dynamicArgsFromDb = cfService.fetchDynamicArgsFromDb(ctx, entityId);
dynamicArgsFromDb.forEach(newArgValues::putIfAbsent);
if (ctx.getCfType() == CalculatedFieldType.GEOFENCING) {
var geofencingState = (GeofencingCalculatedFieldState) state;
geofencingState.updateLastDynamicArgumentsRefreshTs();
}
} catch (Exception e) {
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build();
}
} else if (ctx.shouldFetchEntityRelations(state)) {
} else if (ctx.shouldFetchRelatedEntities(state)) {
log.debug("[{}][{}] Going to update related entities for CF.", entityId, ctx.getCfId());
try {
if (state instanceof RelatedEntitiesAggregationCalculatedFieldState relatedEntitiesState) {
@ -448,6 +439,11 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
justRestored = true;
}
}
if (state instanceof GeofencingCalculatedFieldState geofencingCalculatedFieldState) {
Map<String, ArgumentEntry> dynamicArgsFromDb = cfService.fetchDynamicArgsFromDb(ctx, entityId);
dynamicArgsFromDb.forEach(newArgValues::putIfAbsent);
geofencingCalculatedFieldState.updateScheduledRefreshTs();
}
} catch (Exception e) {
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build();
}
@ -477,9 +473,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
state.setCtx(ctx, actorCtx);
state.init(false);
if (ctx.getCfType() == CalculatedFieldType.GEOFENCING && ctx.isRelationQueryDynamicArguments()) {
if (ctx.getCfType() == CalculatedFieldType.GEOFENCING && ctx.isCfHasRelationPathQuerySource()) {
GeofencingCalculatedFieldState geofencingState = (GeofencingCalculatedFieldState) state;
geofencingState.updateLastDynamicArgumentsRefreshTs();
geofencingState.updateScheduledRefreshTs();
}
Map<String, ArgumentEntry> arguments = fetchArguments(ctx);
@ -588,30 +584,61 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
if (!relatedEntityArgs.isEmpty() || !args.isEmpty()) {
for (TsKvProto item : data) {
ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null);
Set<String> argNames = relatedEntityArgs.get(key);
if (argNames != null) {
argNames.forEach(argName -> {
arguments.put(argName, new SingleValueArgumentEntry(originator, item));
});
}
argNames = args.get(key);
if (argNames != null) {
argNames.forEach(argName -> {
arguments.put(argName, new SingleValueArgumentEntry(item));
});
}
SingleValueArgumentEntry relatedArgIncoming = new SingleValueArgumentEntry(originator, item);
mapLatest(relatedArgIncoming, relatedEntityArgs.get(key), arguments);
SingleValueArgumentEntry incoming = new SingleValueArgumentEntry(item);
mapLatest(incoming, args.get(key), arguments);
key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null);
argNames = args.get(key);
if (argNames != null) {
argNames.forEach(argName -> {
arguments.put(argName, new SingleValueArgumentEntry(item));
});
}
mapRolling(item, args.get(key), arguments);
}
}
return arguments;
}
private void mapLatest(SingleValueArgumentEntry incoming,
Set<String> argNames,
Map<String, ArgumentEntry> arguments) {
if (argNames != null) {
argNames.forEach(argName -> arguments.compute(argName, (name, existing) -> {
if (existing == null) {
return incoming;
}
existing.updateEntry(incoming);
return existing;
}));
}
}
private void mapRolling(TsKvProto item,
Set<String> argNames,
Map<String, ArgumentEntry> arguments) {
if (argNames != null) {
Double recordValue = getValueForTsRecord(ProtoUtils.fromProto(item.getKv()));
argNames.forEach(argName -> arguments.compute(argName, (name, existing) -> {
if (existing instanceof TsRollingArgumentEntry rolling) {
if (recordValue != null) {
rolling.getTsRecords().put(item.getTs(), recordValue);
}
return rolling;
}
TsRollingArgumentEntry rolling = new TsRollingArgumentEntry();
if (recordValue != null) {
rolling.getTsRecords().put(item.getTs(), recordValue);
}
if (existing instanceof SingleValueArgumentEntry single) {
Double existingValue = getValueForTsRecord(single.getKvEntryValue());
if (existingValue != null) {
rolling.getTsRecords().put(single.getTs(), existingValue);
}
}
return rolling;
}));
}
}
private Map<String, ArgumentEntry> mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List<AttributeValueProto> attrDataList) {
return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), Collections.emptyMap(), scope, attrDataList);
}

4
application/src/main/java/org/thingsboard/server/controller/BaseController.java

@ -150,8 +150,6 @@ import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.domain.DomainService;
import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.exception.EntitiesLimitExceededException;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.job.JobService;
import org.thingsboard.server.dao.mobile.MobileAppBundleService;
@ -175,6 +173,8 @@ import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.dao.widget.WidgetTypeService;
import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.exception.EntitiesLimitExceededException;
import org.thingsboard.server.exception.ThingsboardErrorResponseHandler;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;

2
application/src/main/java/org/thingsboard/server/controller/EdgeController.java

@ -54,7 +54,7 @@ import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportReques
import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportResult;
import org.thingsboard.server.common.msg.edge.FromEdgeSyncResponse;
import org.thingsboard.server.config.annotations.ApiOperation;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.queue.util.TbCoreComponent;

1
application/src/main/java/org/thingsboard/server/exception/ThingsboardErrorResponseHandler.java

@ -52,7 +52,6 @@ import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.msg.tools.MaxPayloadSizeExceededException;
import org.thingsboard.server.common.msg.tools.TbRateLimitsException;
import org.thingsboard.server.dao.exception.EntitiesLimitExceededException;
import org.thingsboard.server.service.security.exception.AuthMethodNotSupportedException;
import org.thingsboard.server.service.security.exception.JwtExpiredTokenException;
import org.thingsboard.server.service.security.exception.UserPasswordExpiredException;

24
application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java

@ -22,17 +22,18 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.gson.JsonElement;
import com.google.gson.JsonParser;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.function.TriConsumer;
import org.thingsboard.common.util.DonAsynchron;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.rule.engine.api.AttributesSaveRequest;
import org.thingsboard.rule.engine.api.AttributesSaveRequest.Strategy;
import org.thingsboard.rule.engine.api.TimeseriesSaveRequest;
import org.thingsboard.server.actors.calculatedField.MultipleTbCallback;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.adaptor.JsonConverter;
import org.thingsboard.server.common.data.AttributeScope;
@ -80,7 +81,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.function.Function;
@ -391,6 +391,24 @@ public abstract class AbstractCalculatedFieldProcessingService {
return new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, endTs, 0, limit, Aggregation.NONE);
}
protected void handlePropagationResults(PropagationCalculatedFieldResult propagationResult, TbCallback callback,
TriConsumer<EntityId, TelemetryCalculatedFieldResult, TbCallback> telemetryResultHandler) {
List<EntityId> propagationEntityIds = propagationResult.getEntityIds();
if (propagationEntityIds.isEmpty()) {
callback.onSuccess();
return;
}
if (propagationEntityIds.size() == 1) {
EntityId propagationEntityId = propagationEntityIds.get(0);
telemetryResultHandler.accept(propagationEntityId, propagationResult.getResult(), callback);
return;
}
MultipleTbCallback multipleTbCallback = new MultipleTbCallback(propagationEntityIds.size(), callback);
for (var propagationEntityId : propagationEntityIds) {
telemetryResultHandler.accept(propagationEntityId, propagationResult.getResult(), multipleTbCallback);
}
}
protected void sendMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbCallback callback, TbMsg msg) {
try {
clusterService.pushMsgToRuleEngine(tenantId, entityId, msg, new TbQueueCallback() {
@ -413,7 +431,7 @@ public abstract class AbstractCalculatedFieldProcessingService {
protected void saveTelemetryResult(TenantId tenantId, EntityId entityId, String cfName, TelemetryCalculatedFieldResult cfResult, List<CalculatedFieldId> cfIds, TbCallback callback) {
OutputType type = cfResult.getType();
JsonElement jsonResult = JsonParser.parseString(Objects.requireNonNull(cfResult.stringValue()));
JsonElement jsonResult = cfResult.toJsonElement();
log.trace("[{}][{}] Saving CF result: {}", tenantId, entityId, jsonResult);
switch (type) {

4
application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java

@ -27,9 +27,11 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntry;
import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry;
import java.util.List;
import java.util.Map;
import java.util.Optional;
public interface CalculatedFieldProcessingService {
@ -37,6 +39,8 @@ public interface CalculatedFieldProcessingService {
Map<String, ArgumentEntry> fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId);
Optional<PropagationArgumentEntry> fetchPropagationArgumentFromDb(CalculatedFieldCtx ctx, EntityId entityId);
List<EntityId> fetchRelatedEntities(CalculatedFieldCtx ctx, EntityId entityId);
Map<String, ArgumentEntry> fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map<String, Argument> arguments);

7
application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java

@ -15,11 +15,14 @@
*/
package org.thingsboard.server.service.cf;
import com.google.gson.JsonElement;
import com.google.gson.JsonParser;
import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.msg.TbMsg;
import java.util.List;
import java.util.Objects;
public interface CalculatedFieldResult {
@ -29,4 +32,8 @@ public interface CalculatedFieldResult {
boolean isEmpty();
default JsonElement toJsonElement() {
return JsonParser.parseString(Objects.requireNonNull(stringValue()));
}
}

39
application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java

@ -17,13 +17,13 @@ package org.thingsboard.server.service.cf;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.function.TriConsumer;
import org.springframework.stereotype.Service;
import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg;
import org.thingsboard.server.actors.calculatedField.MultipleTbCallback;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Argument;
import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric;
import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration;
@ -50,6 +50,7 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import org.thingsboard.server.service.cf.ctx.state.aggregation.single.AggIntervalEntry;
import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry;
import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService;
import java.util.ArrayList;
@ -57,6 +58,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
@ -94,11 +96,18 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF
@Override
public Map<String, ArgumentEntry> fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId) {
return switch (ctx.getCfType()) {
case GEOFENCING -> resolveArgumentFutures(fetchGeofencingCalculatedFieldArguments(ctx, entityId, true, System.currentTimeMillis()));
case PROPAGATION -> resolveArgumentFutures(Map.of(PROPAGATION_CONFIG_ARGUMENT, fetchPropagationCalculatedFieldArgument(ctx, entityId)));
default -> Collections.emptyMap();
};
return ctx.getCfType() == CalculatedFieldType.GEOFENCING ?
resolveArgumentFutures(fetchGeofencingCalculatedFieldArguments(ctx, entityId, true, System.currentTimeMillis())) :
Collections.emptyMap();
}
@Override
public Optional<PropagationArgumentEntry> fetchPropagationArgumentFromDb(CalculatedFieldCtx ctx, EntityId entityId) {
if (ctx.getCfType() != CalculatedFieldType.PROPAGATION) {
return Optional.empty();
}
return Optional.of((PropagationArgumentEntry)
resolveArgumentValue(PROPAGATION_CONFIG_ARGUMENT, fetchPropagationCalculatedFieldArgument(ctx, entityId)));
}
@Override
@ -169,24 +178,6 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF
sendMsgToRuleEngine(tenantId, entityId, callback, result.toTbMsg(entityId, cfName, cfIds));
}
private void handlePropagationResults(PropagationCalculatedFieldResult propagationResult, TbCallback callback,
TriConsumer<EntityId, TelemetryCalculatedFieldResult, TbCallback> telemetryResultHandler) {
List<EntityId> propagationEntityIds = propagationResult.getEntityIds();
if (propagationEntityIds.isEmpty()) {
callback.onSuccess();
return;
}
if (propagationEntityIds.size() == 1) {
EntityId propagationEntityId = propagationEntityIds.get(0);
telemetryResultHandler.accept(propagationEntityId, propagationResult.getResult(), callback);
return;
}
MultipleTbCallback multipleTbCallback = new MultipleTbCallback(propagationEntityIds.size(), callback);
for (var propagationEntityId : propagationEntityIds) {
telemetryResultHandler.accept(propagationEntityId, propagationResult.getResult(), multipleTbCallback);
}
}
@Override
public void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List<CalculatedFieldEntityCtxId> linkedCalculatedFields, TbCallback callback) {
Map<TopicPartitionInfo, List<CalculatedFieldEntityCtxId>> unicasts = new HashMap<>();

10
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java

@ -63,7 +63,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState,
this.ctx = ctx;
this.actorCtx = actorCtx;
this.requiredArguments = ctx.getArgNames();
this.readinessStatus = checkReadiness(requiredArguments, arguments);
this.readinessStatus = checkReadiness();
}
@Override
@ -108,7 +108,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState,
if (updatedArguments == null) {
return Collections.emptyMap();
}
readinessStatus = checkReadiness(requiredArguments, arguments);
readinessStatus = checkReadiness();
return updatedArguments;
}
@ -183,13 +183,13 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState,
return latestTs;
}
protected ReadinessStatus checkReadiness(List<String> requiredArguments, Map<String, ArgumentEntry> currentArguments) {
if (currentArguments == null) {
protected ReadinessStatus checkReadiness() {
if (arguments == null) {
return ReadinessStatus.from(requiredArguments);
}
List<String> emptyArguments = null;
for (String requiredArgumentKey : requiredArguments) {
ArgumentEntry argumentEntry = currentArguments.get(requiredArgumentKey);
ArgumentEntry argumentEntry = arguments.get(requiredArgumentKey);
if (argumentEntry == null || argumentEntry.isEmpty()) {
if (emptyArguments == null) {
emptyArguments = new ArrayList<>();

51
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java

@ -63,8 +63,7 @@ import org.thingsboard.server.dao.util.TimeUtils;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto;
import org.thingsboard.server.service.cf.CalculatedFieldProcessingService;
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState;
import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState;
import org.thingsboard.server.service.cf.ctx.state.geofencing.ScheduledRefreshSupported;
import org.thingsboard.server.service.telemetry.AlarmSubscriptionService;
import java.io.Closeable;
@ -122,14 +121,14 @@ public class CalculatedFieldCtx implements Closeable {
private long maxSingleValueArgumentSize;
private long intermediateAggregationIntervalMillis;
private boolean relationQueryDynamicArguments;
private boolean cfHasRelationPathQuerySource;
private List<String> mainEntityGeofencingArgumentNames;
private List<String> linkedEntityAndCurrentOwnerGeofencingArgumentNames;
private List<String> relatedEntityArgumentNames;
private long scheduledUpdateIntervalMillis;
private long cfCheckReevaluationInterval;
private long alarmReevaluationInterval;
private long cfCheckReevaluationIntervalMillis;
private long alarmReevaluationIntervalMillis;
private Argument propagationArgument;
private boolean applyExpressionForResolvedArguments;
@ -161,10 +160,11 @@ public class CalculatedFieldCtx implements Closeable {
if (refId == null) {
if (CalculatedFieldType.RELATED_ENTITIES_AGGREGATION.equals(cfType)) {
relatedEntityArguments.compute(refKey, (key, existingNames) -> CollectionsUtil.addToSet(existingNames, entry.getKey()));
cfHasRelationPathQuerySource = true;
continue;
}
if (entry.getValue().hasRelationQuerySource()) {
relationQueryDynamicArguments = true;
cfHasRelationPathQuerySource = true;
continue;
}
if (entry.getValue().hasOwnerSource()) {
@ -201,7 +201,7 @@ public class CalculatedFieldCtx implements Closeable {
if (calculatedField.getConfiguration() instanceof PropagationCalculatedFieldConfiguration propagationConfig) {
propagationArgument = propagationConfig.toPropagationArgument();
applyExpressionForResolvedArguments = propagationConfig.isApplyExpressionToResolvedArguments();
relationQueryDynamicArguments = true;
cfHasRelationPathQuerySource = true;
}
}
if (calculatedField.getConfiguration() instanceof ScheduledUpdateSupportedCalculatedFieldConfiguration scheduledConfig) {
@ -246,8 +246,7 @@ public class CalculatedFieldCtx implements Closeable {
boolean requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation();
if (calculatedField.getConfiguration() instanceof AlarmCalculatedFieldConfiguration) {
if (requiresScheduledReevaluation) {
long reevaluationIntervalMillis = TimeUnit.SECONDS.toMillis(alarmReevaluationInterval);
if (now - lastReevaluationTs >= reevaluationIntervalMillis) {
if (now - lastReevaluationTs >= alarmReevaluationIntervalMillis) {
lastReevaluationTs = now;
return true;
}
@ -306,8 +305,8 @@ public class CalculatedFieldCtx implements Closeable {
this.maxStateSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes) * 1024;
this.maxSingleValueArgumentSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxSingleValueArgumentSizeInKBytes) * 1024;
this.intermediateAggregationIntervalMillis = TimeUnit.SECONDS.toMillis(apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getIntermediateAggregationIntervalInSecForCF));
this.cfCheckReevaluationInterval = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getCfReevaluationCheckInterval);
this.alarmReevaluationInterval = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getAlarmsReevaluationInterval);
this.cfCheckReevaluationIntervalMillis = TimeUnit.SECONDS.toMillis(apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getCfReevaluationCheckInterval));
this.alarmReevaluationIntervalMillis = TimeUnit.SECONDS.toMillis(apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getAlarmsReevaluationInterval));
}
public double evaluateSimpleExpression(Expression expression, CalculatedFieldState state) {
@ -757,38 +756,20 @@ public class CalculatedFieldCtx implements Closeable {
return scheduledUpdateIntervalMillis == DISABLED_INTERVAL_VALUE;
}
public boolean shouldFetchRelationQueryDynamicArgumentsFromDb(CalculatedFieldState state) {
if (!relationQueryDynamicArguments) {
public boolean shouldFetchRelatedEntities(CalculatedFieldState state) {
if (!cfHasRelationPathQuerySource) {
return false;
}
return switch (cfType) {
case PROPAGATION -> true;
case GEOFENCING -> {
if (isScheduledUpdateDisabled()) {
yield false;
}
var geofencingState = (GeofencingCalculatedFieldState) state;
if (geofencingState.getLastDynamicArgumentsRefreshTs() == DEFAULT_LAST_UPDATE_TS) {
yield true;
}
yield geofencingState.getLastDynamicArgumentsRefreshTs() <
System.currentTimeMillis() - scheduledUpdateIntervalMillis;
}
default -> false;
};
}
public boolean shouldFetchEntityRelations(CalculatedFieldState state) {
if (!(state instanceof RelatedEntitiesAggregationCalculatedFieldState relatedEntitiesAggState)) {
if (isScheduledUpdateDisabled()) {
return false;
}
if (isScheduledUpdateDisabled()) {
if (!(state instanceof ScheduledRefreshSupported scheduledRefreshSupported)) {
return false;
}
if (relatedEntitiesAggState.getLastRelatedEntitiesRefreshTs() == DEFAULT_LAST_UPDATE_TS) {
if (scheduledRefreshSupported.getLastScheduledRefreshTs() == DEFAULT_LAST_UPDATE_TS) {
return true;
}
return relatedEntitiesAggState.getLastRelatedEntitiesRefreshTs() < System.currentTimeMillis() - scheduledUpdateIntervalMillis;
return scheduledRefreshSupported.getLastScheduledRefreshTs() < System.currentTimeMillis() - scheduledUpdateIntervalMillis;
}
@Override

15
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java

@ -110,7 +110,13 @@ public interface CalculatedFieldState extends Closeable {
private static final String MISSING_PROPAGATION_TARGETS_ERROR = "No entities found via 'Propagation path to related entities'. " +
"Verify the configured relation type and direction.";
private static final String MISSING_PROPAGATION_TARGETS_AND_ARGUMENTS_ERROR = MISSING_PROPAGATION_TARGETS_ERROR + " Missing arguments to propagate: ";
private static final ReadinessStatus READY = new ReadinessStatus(true, null);
public static final String MISSING_AGGREGATION_ENTITIES_ERROR = "No entities found via 'Aggregation path to related entities'. " +
"Verify the configured relation type and direction.";
public static final ReadinessStatus READY = new ReadinessStatus(true, null);
public static ReadinessStatus notReady(String errorMsg) {
return new ReadinessStatus(false, errorMsg);
}
public static ReadinessStatus from(List<String> emptyOrMissingArguments) {
if (CollectionsUtil.isEmpty(emptyOrMissingArguments)) {
@ -118,13 +124,12 @@ public interface CalculatedFieldState extends Closeable {
}
boolean propagationCtxIsEmpty = emptyOrMissingArguments.remove(PROPAGATION_CONFIG_ARGUMENT);
if (!propagationCtxIsEmpty) {
return new ReadinessStatus(false, MISSING_REQUIRED_ARGUMENTS_ERROR + String.join(", ", emptyOrMissingArguments));
return notReady(MISSING_REQUIRED_ARGUMENTS_ERROR + String.join(", ", emptyOrMissingArguments));
}
if (emptyOrMissingArguments.isEmpty()) {
return new ReadinessStatus(false, MISSING_PROPAGATION_TARGETS_ERROR);
return notReady(MISSING_PROPAGATION_TARGETS_ERROR);
}
return new ReadinessStatus(false, MISSING_PROPAGATION_TARGETS_AND_ARGUMENTS_ERROR +
String.join(", ", emptyOrMissingArguments));
return notReady(MISSING_PROPAGATION_TARGETS_AND_ARGUMENTS_ERROR + String.join(", ", emptyOrMissingArguments));
}
}

32
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java

@ -122,20 +122,11 @@ public class TsRollingArgumentEntry implements ArgumentEntry, HasLatestTs {
}
private void addTsRecord(Long ts, KvEntry value) {
try {
switch (value.getDataType()) {
case LONG -> value.getLongValue().ifPresent(aLong -> tsRecords.put(ts, aLong.doubleValue()));
case DOUBLE -> value.getDoubleValue().ifPresent(aDouble -> tsRecords.put(ts, aDouble));
case BOOLEAN -> value.getBooleanValue().ifPresent(aBoolean -> tsRecords.put(ts, aBoolean ? 1.0 : 0.0));
case STRING -> value.getStrValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString)));
case JSON -> value.getJsonValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString)));
}
} catch (Exception e) {
tsRecords.put(ts, Double.NaN);
log.debug("Invalid value '{}' for time series rolling arguments. Only numeric values are supported.", value.getValue());
} finally {
cleanupExpiredRecords();
Double recordValue = getValueForTsRecord(value);
if (recordValue != null) {
tsRecords.put(ts, recordValue);
}
cleanupExpiredRecords();
}
private void addTsRecord(Long ts, double value) {
@ -150,4 +141,19 @@ public class TsRollingArgumentEntry implements ArgumentEntry, HasLatestTs {
tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - timeWindow);
}
public static Double getValueForTsRecord(KvEntry value) {
try {
return switch (value.getDataType()) {
case LONG -> value.getLongValue().map(Long::doubleValue).orElse(null);
case DOUBLE -> value.getDoubleValue().orElse(null);
case BOOLEAN -> value.getBooleanValue().map(b -> b ? 1.0 : 0.0).orElse(null);
case STRING -> value.getStrValue().map(Double::parseDouble).orElse(null);
case JSON -> value.getJsonValue().map(Double::parseDouble).orElse(null);
};
} catch (Exception e) {
log.debug("Invalid value '{}' for time series rolling arguments. Only numeric values are supported.", value.getValue());
return Double.NaN;
}
}
}

40
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java

@ -41,6 +41,7 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType;
import org.thingsboard.server.service.cf.ctx.state.BaseCalculatedFieldState;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import org.thingsboard.server.service.cf.ctx.state.aggregation.function.AggEntry;
import org.thingsboard.server.service.cf.ctx.state.geofencing.ScheduledRefreshSupported;
import java.util.ArrayList;
import java.util.HashMap;
@ -52,16 +53,17 @@ import java.util.stream.Collectors;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx.DISABLED_INTERVAL_VALUE;
import static org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState.ReadinessStatus.MISSING_AGGREGATION_ENTITIES_ERROR;
@Slf4j
@Getter
public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculatedFieldState {
public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculatedFieldState implements ScheduledRefreshSupported {
@Setter
@Getter
private long lastArgsRefreshTs = DEFAULT_LAST_UPDATE_TS;
@Setter
@Getter
private long lastMetricsEvalTs = DEFAULT_LAST_UPDATE_TS;
@Setter
private long lastRelatedEntitiesRefreshTs = DEFAULT_LAST_UPDATE_TS;
private long deduplicationIntervalMs = DISABLED_INTERVAL_VALUE;
private Map<String, AggMetric> metrics;
@ -103,13 +105,24 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat
@Override
public void reset() { // must reset everything dependent on arguments
super.reset();
resetScheduledRefreshTs();
lastArgsRefreshTs = DEFAULT_LAST_UPDATE_TS;
lastMetricsEvalTs = DEFAULT_LAST_UPDATE_TS;
lastRelatedEntitiesRefreshTs = DEFAULT_LAST_UPDATE_TS;
metrics = null;
}
public void updateLastRelatedEntitiesRefreshTs() {
@Override
public void resetScheduledRefreshTs() {
lastRelatedEntitiesRefreshTs = DEFAULT_LAST_UPDATE_TS;
}
@Override
public long getLastScheduledRefreshTs() {
return lastRelatedEntitiesRefreshTs;
}
@Override
public void updateScheduledRefreshTs() {
lastRelatedEntitiesRefreshTs = System.currentTimeMillis();
}
@ -127,7 +140,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat
public List<EntityId> checkRelatedEntities(List<EntityId> relatedEntities) {
Map<EntityId, Map<String, ArgumentEntry>> entityInputs = prepareInputs();
findOutdatedEntities(entityInputs, relatedEntities).forEach(this::cleanupEntityData);
updateLastRelatedEntitiesRefreshTs();
updateScheduledRefreshTs();
return findMissingEntities(entityInputs, relatedEntities);
}
@ -165,6 +178,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat
});
lastMetricsEvalTs = DEFAULT_LAST_UPDATE_TS;
lastArgsRefreshTs = System.currentTimeMillis();
readinessStatus = checkReadiness();
}
public void scheduleReevaluation() {
@ -276,4 +290,18 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat
record EntityArgument(EntityInfo entity, JsonNode entityArguments) {}
@Override
protected ReadinessStatus checkReadiness() {
if (arguments == null) {
return ReadinessStatus.notReady(MISSING_AGGREGATION_ENTITIES_ERROR);
}
for (String requiredArgumentKey : requiredArguments) {
ArgumentEntry argumentEntry = arguments.get(requiredArgumentKey);
if (argumentEntry == null || argumentEntry.isEmpty()) {
return ReadinessStatus.notReady(MISSING_AGGREGATION_ENTITIES_ERROR);
}
}
return ReadinessStatus.READY;
}
}

4
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java

@ -205,7 +205,7 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt
handleExpiredInterval(intervalEntry, args, results);
expiredIntervals.add(intervalEntry);
} else if (now - startTs >= intervalEntry.getIntervalDuration()) {
handleActiveInterval(ctx.getCfCheckReevaluationInterval(), intervalEntry, args, results);
handleActiveInterval(ctx.getCfCheckReevaluationIntervalMillis(), intervalEntry, args, results);
if (watermarkDuration == 0) {
expiredIntervals.add(intervalEntry);
}
@ -288,7 +288,7 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt
}
if (!metricsNode.isEmpty()) {
ObjectNode resultNode = JacksonUtil.newObjectNode();
resultNode.put("ts", interval.getEndTs() - 1);
resultNode.put("ts", interval.getStartTs());
resultNode.set("values", metricsNode);
result.add(resultNode);

2
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/alarm/AlarmCalculatedFieldState.java

@ -94,8 +94,6 @@ public class AlarmCalculatedFieldState extends BaseCalculatedFieldState {
private Alarm currentAlarm;
private boolean initialFetchDone;
// TODO: deprecate device profile node, describe the differences and improvements
public AlarmCalculatedFieldState(EntityId entityId) {
super(entityId);
}

19
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/GeofencingCalculatedFieldState.java

@ -21,8 +21,6 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.geo.Coordinates;
@ -52,11 +50,9 @@ import static org.thingsboard.server.common.data.cf.configuration.geofencing.Ent
import static org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingPresenceStatus.INSIDE;
import static org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingPresenceStatus.OUTSIDE;
@Getter
@Setter
@Slf4j
@EqualsAndHashCode(callSuper = true)
public class GeofencingCalculatedFieldState extends BaseCalculatedFieldState {
public class GeofencingCalculatedFieldState extends BaseCalculatedFieldState implements ScheduledRefreshSupported {
private long lastDynamicArgumentsRefreshTs = DEFAULT_LAST_UPDATE_TS;
@ -147,10 +143,21 @@ public class GeofencingCalculatedFieldState extends BaseCalculatedFieldState {
@Override
public void reset() {
super.reset();
resetScheduledRefreshTs();
}
@Override
public void resetScheduledRefreshTs() {
lastDynamicArgumentsRefreshTs = DEFAULT_LAST_UPDATE_TS;
}
public void updateLastDynamicArgumentsRefreshTs() {
@Override
public long getLastScheduledRefreshTs() {
return lastDynamicArgumentsRefreshTs;
}
@Override
public void updateScheduledRefreshTs() {
lastDynamicArgumentsRefreshTs = System.currentTimeMillis();
}

26
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/geofencing/ScheduledRefreshSupported.java

@ -0,0 +1,26 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.cf.ctx.state.geofencing;
public interface ScheduledRefreshSupported {
void resetScheduledRefreshTs();
long getLastScheduledRefreshTs();
void updateScheduledRefreshTs();
}

46
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationArgumentEntry.java

@ -22,6 +22,8 @@ import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry;
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@ -30,10 +32,11 @@ import java.util.Set;
public class PropagationArgumentEntry implements ArgumentEntry {
private Set<EntityId> entityIds;
private transient EntityId added;
private transient List<EntityId> added;
private transient EntityId removed;
private boolean forceResetPrevious;
private transient boolean forceResetPrevious;
private transient boolean ignoreRemovedEntities;
public PropagationArgumentEntry() {
this.entityIds = new HashSet<>();
@ -57,27 +60,44 @@ public class PropagationArgumentEntry implements ArgumentEntry {
@Override
public boolean updateEntry(ArgumentEntry entry) {
if (!(entry instanceof PropagationArgumentEntry propagationArgumentEntry)) {
if (!(entry instanceof PropagationArgumentEntry updated)) {
throw new IllegalArgumentException("Unsupported argument entry type for propagation argument entry: " + entry.getType());
}
if (propagationArgumentEntry.getAdded() != null) {
boolean updated = entityIds.add(propagationArgumentEntry.getAdded());
if (updated) {
added = propagationArgumentEntry.getAdded();
}
return updated;
if (updated.getAdded() != null) {
return checkAdded(updated.getAdded());
}
if (updated.getRemoved() != null) {
return entityIds.remove(updated.getRemoved());
}
if (propagationArgumentEntry.getRemoved() != null) {
return entityIds.remove(propagationArgumentEntry.getRemoved());
if (updated.isIgnoreRemovedEntities()) {
Set<EntityId> updatedIds = updated.getEntityIds();
if (updatedIds.isEmpty()) {
entityIds.clear();
return false;
}
entityIds.retainAll(updatedIds);
return checkAdded(updatedIds);
}
if (propagationArgumentEntry.isEmpty()) {
if (updated.isEmpty()) {
entityIds.clear();
return true;
}
entityIds = propagationArgumentEntry.getEntityIds();
entityIds = updated.getEntityIds();
return true;
}
private boolean checkAdded(Collection<EntityId> updatedIds) {
for (EntityId id : updatedIds) {
if (entityIds.add(id)) {
if (added == null) {
added = new ArrayList<>();
}
added.add(id);
}
}
return added != null && !added.isEmpty();
}
@Override
public boolean isEmpty() {
return entityIds.isEmpty();

67
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/propagation/PropagationCalculatedFieldState.java

@ -25,6 +25,8 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Output;
import org.thingsboard.server.common.data.cf.configuration.OutputType;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.util.CollectionsUtil;
import org.thingsboard.server.service.cf.CalculatedFieldProcessingService;
import org.thingsboard.server.service.cf.CalculatedFieldResult;
import org.thingsboard.server.service.cf.PropagationCalculatedFieldResult;
import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult;
@ -41,6 +43,8 @@ import static org.thingsboard.server.common.data.cf.configuration.PropagationCal
public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState {
private CalculatedFieldProcessingService cfProcessingService;
public PropagationCalculatedFieldState(EntityId entityId) {
super(entityId);
}
@ -49,14 +53,30 @@ public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState
public void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx) {
this.ctx = ctx;
this.actorCtx = actorCtx;
this.cfProcessingService = ctx.getCfProcessingService();
this.requiredArguments = new ArrayList<>(ctx.getArgNames());
requiredArguments.add(PROPAGATION_CONFIG_ARGUMENT);
this.readinessStatus = checkReadiness(requiredArguments, arguments);
this.readinessStatus = checkReadiness();
if (ctx.isApplyExpressionForResolvedArguments()) {
this.tbelExpression = ctx.getTbelExpressions().get(ctx.getExpression());
}
}
@Override
public void init(boolean restored) {
super.init(restored);
if (restored) {
cfProcessingService.fetchPropagationArgumentFromDb(ctx, entityId).ifPresent(fromDb -> {
fromDb.setIgnoreRemovedEntities(true);
var updatedArgs = update(Map.of(PROPAGATION_CONFIG_ARGUMENT, fromDb), ctx);
if (updatedArgs.isEmpty()) {
return;
}
ctx.scheduleReevaluation(0L, actorCtx);
});
}
}
@Override
public CalculatedFieldType getType() {
return CalculatedFieldType.PROPAGATION;
@ -68,9 +88,10 @@ public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState
if (!(argumentEntry instanceof PropagationArgumentEntry propagationArgumentEntry)) {
return Futures.immediateFuture(PropagationCalculatedFieldResult.builder().build());
}
boolean newEntityAdded = propagationArgumentEntry.getAdded() != null;
List<EntityId> entityIds;
if (propagationArgumentEntry.getAdded() != null) {
entityIds = List.of(propagationArgumentEntry.getAdded());
if (newEntityAdded) {
entityIds = propagationArgumentEntry.getAdded();
propagationArgumentEntry.setAdded(null);
} else {
if (propagationArgumentEntry.getEntityIds().isEmpty()) {
@ -86,13 +107,43 @@ public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState
.build(),
MoreExecutors.directExecutor());
}
if (newEntityAdded || CollectionsUtil.isEmpty(updatedArgs)) {
updatedArgs = arguments;
}
return Futures.immediateFuture(PropagationCalculatedFieldResult.builder()
.entityIds(entityIds)
.result(toTelemetryResult(ctx))
.result(toTelemetryResult(ctx, updatedArgs))
.build());
}
private TelemetryCalculatedFieldResult toTelemetryResult(CalculatedFieldCtx ctx) {
@Override
protected ReadinessStatus checkReadiness() {
if (ctx.isApplyExpressionForResolvedArguments() || arguments == null) {
return super.checkReadiness();
}
boolean propagationNotEmpty = false;
boolean hasOtherNonEmpty = false;
List<String> emptyArguments = null;
for (String requiredArgumentKey : requiredArguments) {
ArgumentEntry argumentEntry = arguments.get(requiredArgumentKey);
if (argumentEntry == null || argumentEntry.isEmpty()) {
if (emptyArguments == null) {
emptyArguments = new ArrayList<>();
}
emptyArguments.add(requiredArgumentKey);
} else if (PROPAGATION_CONFIG_ARGUMENT.equals(requiredArgumentKey)) {
propagationNotEmpty = true;
} else {
hasOtherNonEmpty = true;
}
}
if (propagationNotEmpty && hasOtherNonEmpty) {
return ReadinessStatus.READY;
}
return ReadinessStatus.from(emptyArguments);
}
private TelemetryCalculatedFieldResult toTelemetryResult(CalculatedFieldCtx ctx, Map<String, ArgumentEntry> updatedArgs) {
Output output = ctx.getOutput();
TelemetryCalculatedFieldResult.TelemetryCalculatedFieldResultBuilder telemetryCfBuilder =
TelemetryCalculatedFieldResult.builder()
@ -100,12 +151,14 @@ public class PropagationCalculatedFieldState extends ScriptCalculatedFieldState
.type(output.getType())
.scope(output.getScope());
ObjectNode valuesNode = JacksonUtil.newObjectNode();
arguments.forEach((outputKey, argumentEntry) -> {
updatedArgs.forEach((outputKey, argumentEntry) -> {
if (argumentEntry instanceof PropagationArgumentEntry) {
return;
}
if (argumentEntry instanceof SingleValueArgumentEntry singleArgumentEntry) {
JacksonUtil.addKvEntry(valuesNode, singleArgumentEntry.getKvEntryValue(), outputKey);
if (!singleArgumentEntry.isEmpty()) {
JacksonUtil.addKvEntry(valuesNode, singleArgumentEntry.getKvEntryValue(), outputKey);
}
return;
}
throw new IllegalArgumentException("Unsupported argument type: " + argumentEntry.getType() + " detected for argument: " + outputKey + ". " +

1
application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java

@ -206,7 +206,6 @@ public class EdgeContextComponent {
@Autowired
private Optional<EdgeStatsCounterService> statsCounterService;
// processors
@Autowired
private AlarmProcessor alarmProcessor;

133
application/src/main/java/org/thingsboard/server/service/edge/EdgeMsgConstructorUtils.java

@ -21,6 +21,7 @@ import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.JsonPrimitive;
import com.google.gson.reflect.TypeToken;
import lombok.extern.slf4j.Slf4j;
@ -52,6 +53,7 @@ import org.thingsboard.server.common.data.asset.AssetProfile;
import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.domain.DomainInfo;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.edge.EdgeEvent;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.id.AiModelId;
import org.thingsboard.server.common.data.id.AssetId;
@ -76,6 +78,7 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.id.WidgetTypeId;
import org.thingsboard.server.common.data.id.WidgetsBundleId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.notification.rule.NotificationRule;
import org.thingsboard.server.common.data.notification.targets.NotificationTarget;
import org.thingsboard.server.common.data.notification.template.NotificationTemplate;
@ -88,6 +91,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.common.transport.util.JsonUtils;
import org.thingsboard.server.gen.edge.v1.AiModelUpdateMsg;
import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg;
import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg;
@ -127,11 +131,16 @@ import org.thingsboard.server.gen.edge.v1.WidgetTypeUpdateMsg;
import org.thingsboard.server.gen.edge.v1.WidgetsBundleUpdateMsg;
import org.thingsboard.server.gen.transport.TransportProtos;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
@Slf4j
public class EdgeMsgConstructorUtils {
@ -670,4 +679,128 @@ public class EdgeMsgConstructorUtils {
.setIdLSB(aiModelId.getId().getLeastSignificantBits()).build();
}
public static List<EdgeEvent> mergeAndFilterDownlinkDuplicates(List<EdgeEvent> edgeEvents) {
try {
edgeEvents = removeDownlinkDuplicates(edgeEvents);
List<AttrUpdateMsg> attrUpdateMsgs = new ArrayList<>();
for (EdgeEvent edgeEvent : edgeEvents) {
if (EdgeEventActionType.ATTRIBUTES_UPDATED.equals(edgeEvent.getAction())) {
attrUpdateMsgs.add(new AttrUpdateMsg(edgeEvent.getEntityId(), edgeEvent.getBody()));
}
}
Map<UUID, Map<String, Long>> latestTsByEntityAndKey = computeLatestTsByEntityAndKey(attrUpdateMsgs);
List<EdgeEvent> result = new ArrayList<>();
for (EdgeEvent edgeEvent : edgeEvents) {
if (!EdgeEventActionType.ATTRIBUTES_UPDATED.equals(edgeEvent.getAction())) {
result.add(edgeEvent);
continue;
}
Map<String, Long> latestByKey = latestTsByEntityAndKey.get(edgeEvent.getEntityId());
JsonNode filteredBody = filterAttributesBody(edgeEvent.getBody(), latestByKey);
if (filteredBody == null) {
continue;
}
result.add(createFilteredEdgeEvent(edgeEvent, filteredBody));
}
result.sort(Comparator.comparingLong(EdgeEvent::getSeqId));
return result;
} catch (Exception e) {
log.warn("Can't merge downlink duplicates, edgeEvents [{}]", edgeEvents, e);
return edgeEvents;
}
}
private static AttrsTs extractAttributes(JsonNode body) {
if (body == null) {
return new AttrsTs(0L, List.of());
}
String bodyStr = JacksonUtil.toString(body);
var jsonObject = JsonParser.parseString(bodyStr).getAsJsonObject();
long ts = jsonObject.get("ts").getAsLong();
var kv = jsonObject.getAsJsonObject("kv");
List<AttributeKvEntry> attrs = JsonConverter.convertToAttributes(
JsonUtils.getJsonObject(
JsonConverter.convertToAttributesProto(kv).getKvList()
), ts);
return new AttrsTs(ts, attrs);
}
private static JsonNode filterAttributesBody(JsonNode body, Map<String, Long> latestByKey) {
if (body == null || latestByKey == null || latestByKey.isEmpty()) {
return null;
}
String bodyStr = JacksonUtil.toString(body);
JsonObject jsonObject = JsonParser.parseString(bodyStr).getAsJsonObject();
long ts = jsonObject.get("ts").getAsLong();
JsonObject kv = jsonObject.getAsJsonObject("kv");
for (Iterator<Map.Entry<String, JsonElement>> it = kv.entrySet().iterator(); it.hasNext(); ) {
Map.Entry<String, JsonElement> e = it.next();
Long latestTs = latestByKey.get(e.getKey());
if (latestTs == null || !latestTs.equals(ts)) {
it.remove();
}
}
if (kv.isEmpty()) {
return null;
}
return JacksonUtil.toJsonNode(jsonObject.toString());
}
private static Map<UUID, Map<String, Long>> computeLatestTsByEntityAndKey(List<AttrUpdateMsg> attrUpdateMsgs) {
Map<UUID, Map<String, Long>> latestTsByEntityAndKey = new HashMap<>();
for (AttrUpdateMsg attrUpdateMsg : attrUpdateMsgs) {
UUID entityId = attrUpdateMsg.entityId();
AttrsTs attrsTs = extractAttributes(attrUpdateMsg.body());
Map<String, Long> map = latestTsByEntityAndKey.computeIfAbsent(entityId, id -> new HashMap<>());
long ts = attrsTs.ts();
for (AttributeKvEntry attr : attrsTs.attrs()) {
map.merge(attr.getKey(), ts, Math::max);
}
}
return latestTsByEntityAndKey;
}
private static EdgeEvent createFilteredEdgeEvent(EdgeEvent edgeEvent, JsonNode filteredBody) {
EdgeEvent filtered = new EdgeEvent();
filtered.setSeqId(edgeEvent.getSeqId());
filtered.setTenantId(edgeEvent.getTenantId());
filtered.setEdgeId(edgeEvent.getEdgeId());
filtered.setAction(edgeEvent.getAction());
filtered.setEntityId(edgeEvent.getEntityId());
filtered.setUid(edgeEvent.getUid());
filtered.setType(edgeEvent.getType());
filtered.setBody(filteredBody);
return filtered;
}
private static List<EdgeEvent> removeDownlinkDuplicates(List<EdgeEvent> edgeEvents) {
Set<EventKey> seen = new HashSet<>();
return edgeEvents.stream()
.filter(e -> seen.add(new EventKey(
e.getTenantId(),
e.getAction(),
e.getEntityId(),
e.getType().name(),
(e.getBody() != null ? e.getBody().toString() : "null"))))
.collect(Collectors.toList());
}
private record EventKey(TenantId tenantId,
EdgeEventActionType action,
UUID entityId,
String type,
String body) {
}
private record AttrsTs(long ts, List<AttributeKvEntry> attrs) {
}
private record AttrUpdateMsg(UUID entityId, JsonNode body) {
}
}

17
application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java

@ -117,7 +117,7 @@ public abstract class EdgeGrpcSession implements Closeable {
private static final int MAX_DOWNLINK_ATTEMPTS = 3;
private static final String RATE_LIMIT_REACHED = "Rate limit reached";
protected static final ConcurrentLinkedQueue<EdgeEvent> highPriorityQueue = new ConcurrentLinkedQueue<>();
protected final ConcurrentLinkedQueue<EdgeEvent> highPriorityQueue = new ConcurrentLinkedQueue<>();
protected UUID sessionId;
private BiConsumer<EdgeId, EdgeGrpcSession> sessionOpenListener;
@ -306,7 +306,8 @@ public abstract class EdgeGrpcSession implements Closeable {
if (isConnected() && !pageData.getData().isEmpty()) {
if (fetcher instanceof GeneralEdgeEventFetcher) {
long queueSize = pageData.getTotalElements() - ((long) pageLink.getPageSize() * pageLink.getPage());
ctx.getStatsCounterService().ifPresent(statsCounterService -> statsCounterService.setDownlinkMsgsLag(edge.getTenantId(), edge.getId(), queueSize));
ctx.getStatsCounterService().ifPresent(statsCounterService ->
statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_LAG, tenantId, edge.getId(), queueSize));
}
log.trace("[{}][{}][{}] event(s) are going to be processed.", tenantId, edge.getId(), pageData.getData().size());
List<DownlinkMsg> downlinkMsgsPack = convertToDownlinkMsgsPack(pageData.getData());
@ -504,7 +505,8 @@ public abstract class EdgeGrpcSession implements Closeable {
ctx.getRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId())
.customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg)
.error("Failed to deliver messages after " + MAX_DOWNLINK_ATTEMPTS + " attempts").build());
ctx.getStatsCounterService().ifPresent(statsCounterService -> statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_PERMANENTLY_FAILED, edge.getTenantId(), edge.getId(), copy.size()));
ctx.getStatsCounterService().ifPresent(statsCounterService ->
statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_PERMANENTLY_FAILED, edge.getTenantId(), edge.getId(), copy.size()));
stopCurrentSendDownlinkMsgsTask(false);
}
} else {
@ -544,7 +546,8 @@ public abstract class EdgeGrpcSession implements Closeable {
try {
if (msg.getSuccess()) {
sessionState.getPendingMsgsMap().remove(msg.getDownlinkMsgId());
ctx.getStatsCounterService().ifPresent(statsCounterService -> statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_PUSHED, edge.getTenantId(), edge.getId(), 1));
ctx.getStatsCounterService().ifPresent(statsCounterService ->
statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_PUSHED, edge.getTenantId(), edge.getId(), 1));
log.debug("[{}][{}][{}] Msg has been processed successfully! Msg Id: [{}], Msg: {}", tenantId, edge.getId(), sessionId, msg.getDownlinkMsgId(), msg);
} else {
log.debug("[{}][{}][{}] Msg processing failed! Msg Id: [{}], Error msg: {}", tenantId, edge.getId(), sessionId, msg.getDownlinkMsgId(), msg.getErrorMsg());
@ -651,7 +654,8 @@ public abstract class EdgeGrpcSession implements Closeable {
protected List<DownlinkMsg> convertToDownlinkMsgsPack(List<EdgeEvent> edgeEvents) {
List<DownlinkMsg> result = new ArrayList<>();
for (EdgeEvent edgeEvent : edgeEvents) {
List<EdgeEvent> filtered = EdgeMsgConstructorUtils.mergeAndFilterDownlinkDuplicates(edgeEvents);
for (EdgeEvent edgeEvent : filtered) {
log.trace("[{}][{}] converting edge event to downlink msg [{}]", tenantId, edge.getId(), edgeEvent);
DownlinkMsg downlinkMsg = null;
try {
@ -813,7 +817,8 @@ public abstract class EdgeGrpcSession implements Closeable {
}
}
highPriorityQueue.add(edgeEvent);
ctx.getStatsCounterService().ifPresent(statsCounterService -> statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edge.getTenantId(), edgeEvent.getEdgeId(), 1));
ctx.getStatsCounterService().ifPresent(statsCounterService ->
statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edge.getTenantId(), edgeEvent.getEdgeId(), 1));
}
protected ListenableFuture<List<Void>> processUplinkMsg(UplinkMsg uplinkMsg) {

1
application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java

@ -92,6 +92,7 @@ public class EdgeSyncCursor {
fetchers.add(new NotificationTargetEdgeEventFetcher(ctx.getNotificationTargetService()));
fetchers.add(new NotificationRuleEdgeEventFetcher(ctx.getNotificationRuleService()));
fetchers.add(new OtaPackagesEdgeEventFetcher(ctx.getOtaPackageService()));
// sync device profiles twice to update software and hardware fields
fetchers.add(new DeviceProfilesEdgeEventFetcher(ctx.getDeviceProfileService()));
fetchers.add(new TenantResourcesEdgeEventFetcher(ctx.getResourceService()));
}

3
application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeEventService.java

@ -52,7 +52,8 @@ public class KafkaEdgeEventService extends BaseEdgeEventService {
TopicPartitionInfo tpi = topicService.getEdgeEventNotificationsTopic(edgeEvent.getTenantId(), edgeEvent.getEdgeId());
ToEdgeEventNotificationMsg msg = ToEdgeEventNotificationMsg.newBuilder().setEdgeEventMsg(ProtoUtils.toProto(edgeEvent)).build();
producerProvider.getTbEdgeEventsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null);
statsCounterService.ifPresent(statsCounterService -> statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edgeEvent.getTenantId(), edgeEvent.getEdgeId(), 1));
statsCounterService.ifPresent(statsCounterService ->
statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edgeEvent.getTenantId(), edgeEvent.getEdgeId(), 1));
return Futures.immediateFuture(null);
}

8
application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java

@ -82,16 +82,18 @@ public class KafkaEdgeGrpcSession extends EdgeGrpcSession {
edgeEvents.add(edgeEvent);
}
List<DownlinkMsg> downlinkMsgsPack = convertToDownlinkMsgsPack(edgeEvents);
boolean isInterrupted = true;
try {
boolean isInterrupted = sendDownlinkMsgsPack(downlinkMsgsPack).get();
isInterrupted = sendDownlinkMsgsPack(downlinkMsgsPack).get();
if (isInterrupted) {
log.debug("[{}][{}] Send downlink messages task was interrupted", tenantId, edge.getId());
} else {
consumer.commit();
}
} catch (Exception e) {
log.error("[{}][{}] Failed to process downlink messages", tenantId, edge.getId(), e);
}
if (!isInterrupted) {
consumer.commit();
}
}
@Override

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/ai/AiModelEdgeProcessor.java

@ -29,7 +29,7 @@ import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.data.id.AiModelId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.AiModelUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java

@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/profile/AssetProfileEdgeProcessor.java

@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/cf/CalculatedFieldEdgeProcessor.java

@ -36,7 +36,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.CalculatedFieldUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java

@ -29,7 +29,7 @@ import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DashboardId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java

@ -44,7 +44,7 @@ import org.thingsboard.server.common.msg.TbMsgDataType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponseActorMsg;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DeviceRpcCallMsg;
import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/profile/DeviceProfileEdgeProcessor.java

@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java

@ -32,7 +32,7 @@ import org.thingsboard.server.common.data.id.EntityViewId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;
import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/resource/ResourceEdgeProcessor.java

@ -27,7 +27,7 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;
import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java

@ -31,7 +31,7 @@ import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java

@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;
import org.thingsboard.server.gen.edge.v1.UpdateMsgType;

13
application/src/main/java/org/thingsboard/server/service/edge/stats/EdgeStatsService.java

@ -54,7 +54,7 @@ import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_P
import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_TMP_FAILED;
@TbCoreComponent
@ConditionalOnProperty(prefix = "edges.stats", name = "enabled", havingValue = "true", matchIfMissing = false)
@ConditionalOnProperty(prefix = "edges.stats", name = "enabled", havingValue = "true")
@RequiredArgsConstructor
@Service
@Slf4j
@ -70,7 +70,6 @@ public class EdgeStatsService {
@Value("${edges.stats.report-interval-millis:600000}")
private long reportIntervalMillis;
@Scheduled(
fixedDelayString = "${edges.stats.report-interval-millis:600000}",
initialDelayString = "${edges.stats.report-interval-millis:600000}"
@ -80,13 +79,13 @@ public class EdgeStatsService {
long now = System.currentTimeMillis();
long ts = now - (now % reportIntervalMillis);
Map<EdgeId, MsgCounters> countersByEdge = statsCounterService.getCounterByEdge();
Map<EdgeId, Long> lagByEdgeId = kafkaAdmin.isPresent() ? getEdgeLagByEdgeId(countersByEdge) : Collections.emptyMap();
Map<EdgeId, MsgCounters> countersByEdgeSnapshot = new HashMap<>(statsCounterService.getCounterByEdge());
Map<EdgeId, MsgCounters> countersByEdgeSnapshot = new HashMap<>(statsCounterService.getMsgCountersByEdge());
boolean isKafkaStats = kafkaAdmin.isPresent();
Map<EdgeId, Long> lagByEdgeId = isKafkaStats ? getLagByEdgeId(countersByEdgeSnapshot) : Collections.emptyMap();
countersByEdgeSnapshot.forEach((edgeId, counters) -> {
TenantId tenantId = counters.getTenantId();
if (kafkaAdmin.isPresent()) {
if (isKafkaStats) {
counters.getMsgsLag().set(lagByEdgeId.getOrDefault(edgeId, 0L));
}
List<TsKvEntry> statsEntries = List.of(
@ -102,7 +101,7 @@ public class EdgeStatsService {
});
}
private Map<EdgeId, Long> getEdgeLagByEdgeId(Map<EdgeId, MsgCounters> countersByEdge) {
private Map<EdgeId, Long> getLagByEdgeId(Map<EdgeId, MsgCounters> countersByEdge) {
Map<EdgeId, String> edgeToTopicMap = countersByEdge.entrySet().stream()
.collect(Collectors.toMap(
Map.Entry::getKey,

2
application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java

@ -103,7 +103,7 @@ import org.thingsboard.server.dao.device.DeviceConnectivityConfiguration;
import org.thingsboard.server.dao.device.DeviceCredentialsService;
import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.mobile.MobileAppDao;
import org.thingsboard.server.dao.notification.NotificationSettingsService;
import org.thingsboard.server.dao.notification.NotificationTargetService;

2
application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java

@ -39,7 +39,7 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.dao.dashboard.DashboardService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService;
import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.resource.ResourceService;

13
application/src/main/java/org/thingsboard/server/service/install/SqlEntityDatabaseSchemaService.java

@ -22,12 +22,13 @@ import org.springframework.stereotype.Service;
@Service
@Profile("install")
@Slf4j
public class SqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaService
implements EntityDatabaseSchemaService {
public class SqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaService implements EntityDatabaseSchemaService {
public static final String SCHEMA_ENTITIES_SQL = "schema-entities.sql";
public static final String SCHEMA_ENTITIES_IDX_SQL = "schema-entities-idx.sql";
public static final String SCHEMA_ENTITIES_IDX_PSQL_ADDON_SQL = "schema-entities-idx-psql-addon.sql";
public static final String SCHEMA_VIEWS_AND_FUNCTIONS_SQL = "schema-views-and-functions.sql";
public static final String SCHEMA_VIEWS_SQL = "schema-views.sql";
public static final String SCHEMA_FUNCTIONS_SQL = "schema-functions.sql";
public SqlEntityDatabaseSchemaService() {
super(SCHEMA_ENTITIES_SQL, SCHEMA_ENTITIES_IDX_SQL);
@ -49,8 +50,10 @@ public class SqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaSer
@Override
public void createOrUpdateViewsAndFunctions() throws Exception {
log.info("Installing SQL DataBase schema views and functions: " + SCHEMA_VIEWS_AND_FUNCTIONS_SQL);
executeQueryFromFile(SCHEMA_VIEWS_AND_FUNCTIONS_SQL);
log.info("Installing SQL DataBase schema views: " + SCHEMA_VIEWS_SQL);
executeQueryFromFile(SCHEMA_VIEWS_SQL);
log.info("Installing SQL DataBase schema functions: " + SCHEMA_FUNCTIONS_SQL);
executeQueryFromFile(SCHEMA_FUNCTIONS_SQL);
}
}

2
application/src/main/java/org/thingsboard/server/service/security/AccessValidator.java

@ -69,7 +69,7 @@ import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.ota.OtaPackageService;
import org.thingsboard.server.dao.resource.ResourceService;

2
application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/DefaultJwtSettingsValidator.java

@ -21,7 +21,7 @@ import org.apache.commons.lang3.StringUtils;
import org.bouncycastle.util.Arrays;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.security.model.JwtSettings;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import java.util.Base64;
import java.util.Optional;

2
application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java

@ -29,11 +29,11 @@ import org.thingsboard.server.common.data.security.model.mfa.account.AccountTwoF
import org.thingsboard.server.common.data.security.model.mfa.account.TwoFaAccountConfig;
import org.thingsboard.server.common.data.security.model.mfa.provider.TwoFaProviderConfig;
import org.thingsboard.server.common.data.security.model.mfa.provider.TwoFaProviderType;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.service.ConstraintValidator;
import org.thingsboard.server.dao.settings.AdminSettingsDao;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.dao.user.UserAuthSettingsDao;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.service.security.auth.mfa.TwoFactorAuthService;
import java.util.Comparator;

2
application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java

@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.security.model.SecuritySettings;
import org.thingsboard.server.common.data.security.model.UserPasswordPolicy;
import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.settings.SecuritySettingsService;
import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.queue.util.TbCoreComponent;

2
application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java

@ -50,7 +50,7 @@ import org.thingsboard.server.common.data.security.model.SecuritySettings;
import org.thingsboard.server.common.data.security.model.UserPasswordPolicy;
import org.thingsboard.server.common.data.security.model.mfa.PlatformTwoFaSettings;
import org.thingsboard.server.dao.audit.AuditLogService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.dao.settings.SecuritySettingsService;
import org.thingsboard.server.dao.user.UserService;

2
application/src/main/java/org/thingsboard/server/service/security/system/SystemSecurityService.java

@ -24,7 +24,7 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.security.model.UserPasswordPolicy;
import org.thingsboard.server.common.data.security.model.mfa.PlatformTwoFaSettings;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.service.security.model.SecurityUser;
public interface SystemSecurityService {

2
application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java

@ -31,7 +31,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.common.data.util.ThrowingRunnable;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.TbLogEntityActionService;

18
application/src/main/java/org/thingsboard/server/service/system/SystemPatchApplier.java

@ -16,7 +16,9 @@
package org.thingsboard.server.service.system;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.base.Charsets;
import com.google.common.hash.Hashing;
import com.google.common.io.Resources;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -34,6 +36,7 @@ import org.thingsboard.server.service.install.update.DefaultDataUpdateService;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
@ -53,6 +56,8 @@ import java.util.stream.Stream;
@RequiredArgsConstructor
public class SystemPatchApplier {
private static final String SCHEMA_VIEWS_SQL = "sql/schema-views.sql";
private static final long ADVISORY_LOCK_ID = 7536891047216478431L;
private final JdbcTemplate jdbcTemplate;
@ -86,6 +91,9 @@ public class SystemPatchApplier {
}
try {
updateSqlViews();
log.info("Updated sql database views");
int updated = updateWidgetTypes();
log.info("Updated {} widget types", updated);
@ -124,6 +132,16 @@ public class SystemPatchApplier {
&& packageVersion.maintenance == dbVersion.maintenance && packageVersion.patch > dbVersion.patch;
}
private void updateSqlViews() {
try {
URL schemaViewsUrl = Resources.getResource(SCHEMA_VIEWS_SQL);
String sql = Resources.toString(schemaViewsUrl, Charsets.UTF_8);
jdbcTemplate.execute(sql);
} catch (IOException e) {
throw new RuntimeException("Unable to update database views from schema-views.sql", e);
}
}
private int updateWidgetTypes() {
AtomicInteger updated = new AtomicInteger();
Path widgetTypesDir = installScripts.getWidgetTypesDir();

2
application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java

@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.protobuf.ByteString;
import org.thingsboard.server.exception.EntitiesLimitExceededException;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor;
@ -76,7 +77,6 @@ import org.thingsboard.server.dao.device.provision.ProvisionFailedException;
import org.thingsboard.server.dao.device.provision.ProvisionRequest;
import org.thingsboard.server.dao.device.provision.ProvisionResponse;
import org.thingsboard.server.dao.device.provision.ProvisionResponseStatus;
import org.thingsboard.server.dao.exception.EntitiesLimitExceededException;
import org.thingsboard.server.dao.ota.OtaPackageService;
import org.thingsboard.server.dao.queue.QueueService;
import org.thingsboard.server.dao.relation.RelationService;

25
application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java

@ -33,6 +33,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ArgumentIntervalProt
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto;
import org.thingsboard.server.gen.transport.TransportProtos.EntityIdProto;
import org.thingsboard.server.gen.transport.TransportProtos.GeofencingArgumentProto;
import org.thingsboard.server.gen.transport.TransportProtos.GeofencingZoneProto;
import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto;
@ -61,6 +62,7 @@ import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgume
import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalculatedFieldState;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.TreeMap;
@ -108,10 +110,15 @@ public class CalculatedFieldUtils {
case SINGLE_VALUE -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry));
case TS_ROLLING -> builder.addRollingValueArguments(toRollingArgumentProto(argName, (TsRollingArgumentEntry) argEntry));
case GEOFENCING -> builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry));
case PROPAGATION -> builder.addAllPropagationEntityIds(toPropagationEntityIdsProto((PropagationArgumentEntry) argEntry));
case RELATED_ENTITIES -> {
RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry;
relatedEntitiesArgumentEntry.getEntityInputs()
.forEach((entityId, entry) -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) entry)));
Map<EntityId, ArgumentEntry> entityInputs = relatedEntitiesArgumentEntry.getEntityInputs();
if (entityInputs.isEmpty()) {
builder.addSingleValueArguments(SingleValueArgumentProto.newBuilder().setArgName(argName).build());
} else {
entityInputs.forEach((entityId, entry) -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) entry)));
}
}
case ENTITY_AGGREGATION -> {
EntityAggregationArgumentEntry entityAggregationArgumentEntry = (EntityAggregationArgumentEntry) argEntry;
@ -136,6 +143,10 @@ public class CalculatedFieldUtils {
return builder.build();
}
private static List<EntityIdProto> toPropagationEntityIdsProto(PropagationArgumentEntry argEntry) {
return argEntry.getEntityIds().stream().map(ProtoUtils::toProto).collect(Collectors.toList());
}
private static AlarmRuleStateProto toAlarmRuleStateProto(AlarmRuleState ruleState) {
return AlarmRuleStateProto.newBuilder()
.setSeverity(Optional.ofNullable(ruleState.getSeverity()).map(Enum::name).orElse(""))
@ -234,7 +245,10 @@ public class CalculatedFieldUtils {
Map<String, Map<EntityId, ArgumentEntry>> arguments = new HashMap<>();
proto.getSingleValueArgumentsList().forEach(argProto -> {
SingleValueArgumentEntry entry = fromSingleValueArgumentProto(argProto);
arguments.computeIfAbsent(argProto.getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry);
Map<EntityId, ArgumentEntry> entityInputs = arguments.computeIfAbsent(argProto.getArgName(), name -> new HashMap<>());
if (entry.getEntityId() != null) {
entityInputs.put(entry.getEntityId(), entry);
}
});
arguments.forEach((argName, entityInputs) -> {
relatedEntitiesAggState.getArguments().put(argName, new RelatedEntitiesArgumentEntry(entityInputs, false));
@ -268,7 +282,10 @@ public class CalculatedFieldUtils {
state.getArguments().put(argProto.getKey(), fromRollingArgumentProto(argProto)));
case GEOFENCING -> proto.getGeofencingArgumentsList().forEach(argProto ->
state.getArguments().put(argProto.getArgName(), fromGeofencingArgumentProto(argProto)));
case PROPAGATION -> state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, new PropagationArgumentEntry());
case PROPAGATION -> {
List<EntityId> propagationEntityIds = proto.getPropagationEntityIdsList().stream().map(ProtoUtils::fromProto).toList();
state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, new PropagationArgumentEntry(propagationEntityIds));
}
case ALARM -> {
AlarmCalculatedFieldState alarmState = (AlarmCalculatedFieldState) state;
AlarmStateProto alarmStateProto = proto.getAlarmState();

2
application/src/main/java/org/thingsboard/server/utils/LwM2mObjectModelUtils.java

@ -26,7 +26,7 @@ import org.thingsboard.server.common.data.lwm2m.LwM2mInstance;
import org.thingsboard.server.common.data.lwm2m.LwM2mObject;
import org.thingsboard.server.common.data.lwm2m.LwM2mResourceObserve;
import org.thingsboard.server.common.data.util.TbDDFFileParser;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import java.io.ByteArrayInputStream;
import java.io.IOException;

135
application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java

@ -1089,7 +1089,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
// Telemetry on device
doPost("/api/plugins/telemetry/DEVICE/" + device.getUuidId() + "/timeseries/unusedScope",
JacksonUtil.toJsonNode("{\"temperature\":12.5}")).andExpect(status().isOk());
JacksonUtil.toJsonNode("{\"temperature\":12.5, \"humidity\":85}")).andExpect(status().isOk());
// --- Build CF: PROPAGATION with expression ---
CalculatedField cf = new CalculatedField();
@ -1102,11 +1102,14 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
cfg.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE));
cfg.setApplyExpressionToResolvedArguments(true);
Argument arg = new Argument();
arg.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null));
cfg.setArguments(Map.of("t", arg));
Argument arg1 = new Argument();
arg1.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null));
cfg.setExpression("{\"testResult\": t * 2}");
Argument arg2 = new Argument();
arg2.setRefEntityKey(new ReferencedEntityKey("humidity", ArgumentType.TS_LATEST, null));
cfg.setArguments(Map.of("t", arg1, "h", arg2));
cfg.setExpression("return { testResult: (t + h) / 2};");
AttributesOutput output = new AttributesOutput();
output.setScope(AttributeScope.SERVER_SCOPE);
@ -1125,8 +1128,8 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
ArrayNode attrs2 = getServerAttributes(asset2.getId(), "testResult");
assertThat(attrs1).isNotNull();
assertThat(attrs2).isNotNull();
assertThat(attrs1.get(0).get("value").asDouble()).isEqualTo(25.0);
assertThat(attrs2.get(0).get("value").asDouble()).isEqualTo(25.0);
assertThat(attrs1.get(0).get("value").asDouble()).isEqualTo(48.75);
assertThat(attrs2.get(0).get("value").asDouble()).isEqualTo(48.75);
});
String deleteUrl = String.format("/api/v2/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s",
@ -1148,7 +1151,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
ArrayNode attrs2 = getServerAttributes(asset2.getId(), "testResult");
assertThat(attrs1).isNullOrEmpty();
assertThat(attrs2).isNotNull();
assertThat(attrs2.get(0).get("value").asDouble()).isEqualTo(50);
assertThat(attrs2.get(0).get("value").asDouble()).isEqualTo(55);
});
}
@ -1167,7 +1170,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
// Telemetry on device
long ts = System.currentTimeMillis() - 300000L;
postTelemetry(device.getId(), String.format("{\"ts\": %s, \"values\": {\"temperature\":12.5}}", ts));
postTelemetry(device.getId(), String.format("{\"ts\": %s, \"values\": {\"temperature\":12.5, \"humidity\":85}}", ts));
// --- Build CF: PROPAGATION without expression ---
CalculatedField cf = new CalculatedField();
@ -1180,9 +1183,12 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
cfg.setRelation(new RelationPathLevel(EntitySearchDirection.TO, EntityRelation.CONTAINS_TYPE));
cfg.setApplyExpressionToResolvedArguments(false); // arguments-only mode
Argument arg = new Argument();
arg.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null));
cfg.setArguments(Map.of("temperatureComputed", arg));
Argument arg1 = new Argument();
arg1.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null));
Argument arg2 = new Argument();
arg2.setRefEntityKey(new ReferencedEntityKey("humidity", ArgumentType.TS_LATEST, null));
cfg.setArguments(Map.of("temperatureComputed", arg1, "humidityComputed", arg2));
TimeSeriesOutput output = new TimeSeriesOutput();
output.setStrategy(new TimeSeriesImmediateOutputStrategy(0, true, true, true, true));
@ -1197,14 +1203,18 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
.atMost(TIMEOUT, TimeUnit.SECONDS)
.pollInterval(POLL_INTERVAL, TimeUnit.SECONDS)
.untilAsserted(() -> {
ObjectNode telemetry1 = getLatestTelemetry(asset1.getId(), "temperatureComputed");
ObjectNode telemetry2 = getLatestTelemetry(asset2.getId(), "temperatureComputed");
ObjectNode telemetry1 = getLatestTelemetry(asset1.getId(), "temperatureComputed,humidityComputed");
ObjectNode telemetry2 = getLatestTelemetry(asset2.getId(), "temperatureComputed,humidityComputed");
assertThat(telemetry1).isNotNull();
assertThat(telemetry2).isNotNull();
assertThat(telemetry1.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts));
assertThat(telemetry1.get("temperatureComputed").get(0).get("value").asDouble()).isEqualTo(12.5);
assertThat(telemetry1.get("humidityComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts));
assertThat(telemetry1.get("humidityComputed").get(0).get("value").asDouble()).isEqualTo(85);
assertThat(telemetry2.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts));
assertThat(telemetry2.get("temperatureComputed").get(0).get("value").asDouble()).isEqualTo(12.5);
assertThat(telemetry2.get("humidityComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts));
assertThat(telemetry2.get("humidityComputed").get(0).get("value").asDouble()).isEqualTo(85);
});
String deleteUrl = String.format("/api/v2/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s",
@ -1212,10 +1222,10 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
EntityRelation.CONTAINS_TYPE, device.getId().getId(), EntityType.DEVICE
);
doDelete(deleteUrl).andExpect(status().isOk());
doDelete("/api/plugins/telemetry/ASSET/" + asset1.getId() + "/timeseries/delete?keys=temperatureComputed&deleteAllDataForKeys=true").andExpect(status().isOk());
doDelete("/api/plugins/telemetry/ASSET/" + asset1.getId() + "/timeseries/delete?keys=temperatureComputed,humidityComputed&deleteAllDataForKeys=true").andExpect(status().isOk());
// Update telemetry on device
long newTs = System.currentTimeMillis() - 300000L;
// Update telemetry on the device
long newTs = ts + 300000L;
postTelemetry(device.getId(), String.format("{\"ts\": %s, \"values\": {\"temperature\":25}}", newTs));
// --- Assert propagated calculation (arguments-only mode after update) ---
@ -1223,13 +1233,18 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
.atMost(TIMEOUT, TimeUnit.SECONDS)
.pollInterval(POLL_INTERVAL, TimeUnit.SECONDS)
.untilAsserted(() -> {
ObjectNode telemetry1 = getLatestTelemetry(asset1.getId(), "temperatureComputed");
ObjectNode telemetry2 = getLatestTelemetry(asset2.getId(), "temperatureComputed");
ObjectNode telemetry1 = getLatestTelemetry(asset1.getId(), "temperatureComputed,humidityComputed");
ObjectNode telemetry2 = getLatestTelemetry(asset2.getId(), "temperatureComputed,humidityComputed");
assertThat(telemetry1).isNotNull();
assertThat(telemetry2).isNotNull();
assertThat(telemetry1.get("temperatureComputed").get(0).get("value")).isEqualTo(NullNode.instance);
assertThat(telemetry1.get("humidityComputed").get(0).get("value")).isEqualTo(NullNode.instance);
assertThat(telemetry2.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(newTs));
assertThat(telemetry2.get("temperatureComputed").get(0).get("value").asDouble()).isEqualTo(25);
// TS for humidity is not updated -> expected
assertThat(telemetry2.get("humidityComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(ts));
assertThat(telemetry2.get("humidityComputed").get(0).get("value").asDouble()).isEqualTo(85);
});
Asset asset3 = createAsset("Propagated Asset 3", null);
@ -1241,10 +1256,12 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
.atMost(TIMEOUT, TimeUnit.SECONDS)
.pollInterval(POLL_INTERVAL, TimeUnit.SECONDS)
.untilAsserted(() -> {
ObjectNode telemetry = getLatestTelemetry(asset3.getId(), "temperatureComputed");
ObjectNode telemetry = getLatestTelemetry(asset3.getId(), "temperatureComputed,humidityComputed");
assertThat(telemetry).isNotNull();
assertThat(telemetry.get("temperatureComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(newTs));
assertThat(telemetry.get("temperatureComputed").get(0).get("value").asDouble()).isEqualTo(25);
assertThat(telemetry.get("humidityComputed").get(0).get("ts").asText()).isEqualTo(Long.toString(newTs));
assertThat(telemetry.get("humidityComputed").get(0).get("value").asDouble()).isEqualTo(85);
});
}
@ -1298,6 +1315,84 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
});
}
@Test
public void testCalculatedFieldWhenBatchOfTelemetrySent() throws Exception {
Device testDevice = createDevice("Test device", "1234567890");
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"a\":5}"));
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"b\":10}"));
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"b\":20}"));
CalculatedField calculatedField = new CalculatedField();
calculatedField.setEntityId(testDevice.getId());
calculatedField.setType(CalculatedFieldType.SCRIPT);
calculatedField.setName("Script CF");
calculatedField.setDebugSettings(DebugSettings.all());
SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration();
ReferencedEntityKey refEntityKeyA = new ReferencedEntityKey("a", ArgumentType.TS_LATEST, null);
Argument argumentA = new Argument();
argumentA.setRefEntityKey(refEntityKeyA);
Argument argumentB = new Argument();
ReferencedEntityKey refEntityKeyB = new ReferencedEntityKey("b", ArgumentType.TS_ROLLING, null);
argumentB.setTimeWindow(TimeUnit.MINUTES.toMillis(10));
argumentB.setLimit(1000);
argumentB.setRefEntityKey(refEntityKeyB);
config.setArguments(Map.of("a", argumentA, "b", argumentB));
config.setExpression("""
return {
"latestA": a,
"avgB": b.avg
};
""");
config.setOutput(new TimeSeriesOutput());
calculatedField.setConfiguration(config);
doPost("/api/calculatedField", calculatedField, CalculatedField.class);
await().alias("create CF -> perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS)
.pollInterval(POLL_INTERVAL, TimeUnit.SECONDS)
.untilAsserted(() -> {
ObjectNode result = getLatestTelemetry(testDevice.getId(), "latestA", "avgB");
assertThat(result).isNotNull();
assertThat(result.get("latestA").get(0).get("value").asText()).isEqualTo("5");
assertThat(result.get("avgB").get(0).get("value").asText()).isEqualTo("15.0");
});
long now = System.currentTimeMillis();
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("""
[{
"ts": %s,
"values": {
"a": 6,
"b": 100
}
}, {
"ts": %s,
"values": {
"a": 7,
"b": 200
}
}, {
"ts": %s,
"values": {
"a": 8,
"b": 300
}
}]""", now - TimeUnit.MINUTES.toMillis(2), now, now - TimeUnit.MINUTES.toMillis(5))));
await().alias("update telemetry -> recalculate state").atMost(TIMEOUT, TimeUnit.SECONDS)
.pollInterval(POLL_INTERVAL, TimeUnit.SECONDS)
.untilAsserted(() -> {
ObjectNode result = getLatestTelemetry(testDevice.getId(), "latestA", "avgB");
assertThat(result).isNotNull();
assertThat(result.get("latestA").get(0).get("value").asText()).isEqualTo("7");
assertThat(result.get("avgB").get(0).get("value").asText()).isEqualTo("126.0");
});
}
@Test
public void testSimpleCalculatedFieldWhenSkipRuleEngineOutputProcessing() throws Exception {
Device testDevice = createDevice("Test device", "1234567890");

2
application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java

@ -1209,7 +1209,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
Map<CalculatedFieldId, CalculatedFieldState> statesMap = (Map<CalculatedFieldId, CalculatedFieldState>) ReflectionTestUtils.getField(processor, "states");
Awaitility.await("CF state for entity actor ready to refresh dynamic arguments").atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> {
CalculatedFieldState calculatedFieldState = statesMap.get(cfId);
boolean isReady = calculatedFieldState != null && ((GeofencingCalculatedFieldState) calculatedFieldState).getLastDynamicArgumentsRefreshTs() <
boolean isReady = calculatedFieldState != null && ((GeofencingCalculatedFieldState) calculatedFieldState).getLastScheduledRefreshTs() <
System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(scheduledUpdateInterval);
log.warn("entityId {}, cfId {}, state ready to refresh == {}", entityId, cfId, isReady);
return isReady;

2
application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java

@ -50,7 +50,7 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.asset.AssetDao;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.service.DaoSqlTest;

2
application/src/test/java/org/thingsboard/server/controller/AssetProfileControllerTest.java

@ -42,7 +42,7 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.asset.AssetProfileDao;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.ArrayList;

2
application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java

@ -44,7 +44,7 @@ import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.customer.CustomerDao;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.ArrayList;

2
application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java

@ -51,7 +51,7 @@ import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.dashboard.DashboardDao;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.ArrayList;

2
application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java

@ -72,7 +72,7 @@ import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportColumn
import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportRequest;
import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportResult;
import org.thingsboard.server.dao.device.DeviceDao;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.service.DaoSqlTest;

2
application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java

@ -52,7 +52,7 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.device.DeviceProfileDao;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.ArrayList;

2
application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java

@ -72,7 +72,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.security.model.JwtSettings;
import org.thingsboard.server.dao.edge.EdgeDao;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.edge.imitator.EdgeImitator;

2
application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java

@ -63,7 +63,7 @@ import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.dao.entityview.EntityViewDao;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.service.DaoSqlTest;

2
application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java

@ -37,7 +37,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest;
import java.nio.ByteBuffer;

2
application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java

@ -47,7 +47,7 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.rule.RuleChainDao;
import org.thingsboard.server.dao.service.DaoSqlTest;

2
application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java

@ -48,7 +48,7 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.widget.WidgetType;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.ArrayList;

2
application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java

@ -50,7 +50,7 @@ import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.settings.StarredDashboardInfo;
import org.thingsboard.server.common.data.settings.UserDashboardsInfo;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.dao.user.UserDao;

2
application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java

@ -29,7 +29,7 @@ import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.ArrayList;

25
application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java

@ -22,6 +22,7 @@ import com.google.protobuf.AbstractMessage;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.MessageLite;
import lombok.extern.slf4j.Slf4j;
import org.awaitility.Awaitility;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@ -102,6 +103,7 @@ import org.thingsboard.server.gen.edge.v1.UserUpdateMsg;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.TreeMap;
import java.util.UUID;
@ -738,4 +740,27 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest {
return rpc;
}
protected void verifyEdgeDisconnected() {
verifyEdgeActiveFlag(false);
}
protected void verifyEdgeConnected() {
verifyEdgeActiveFlag(true);
}
private void verifyEdgeActiveFlag(boolean value) {
Awaitility.await()
.atMost(TIMEOUT, TimeUnit.SECONDS)
.until(() -> {
List<Map<String, Object>> values = doGetAsyncTyped("/api/plugins/telemetry/EDGE/" + edge.getId() +
"/values/attributes/SERVER_SCOPE", new TypeReference<>() {});
Optional<Map<String, Object>> activeAttrOpt = values.stream().filter(att -> att.get("key").equals("active")).findFirst();
if (activeAttrOpt.isEmpty()) {
return false;
}
Map<String, Object> activeAttr = activeAttrOpt.get();
return Boolean.toString(value).equals(activeAttr.get("value").toString());
});
}
}

13
application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java

@ -893,18 +893,7 @@ public class DeviceEdgeTest extends AbstractEdgeTest {
ObjectNode attributes = JacksonUtil.newObjectNode();
attributes.put("active", true);
doPost("/api/plugins/telemetry/EDGE/" + edge.getId() + "/attributes/" + DataConstants.SERVER_SCOPE, attributes);
Awaitility.await()
.atMost(TIMEOUT, TimeUnit.SECONDS)
.until(() -> {
List<Map<String, Object>> values = doGetAsyncTyped("/api/plugins/telemetry/EDGE/" + edge.getId() +
"/values/attributes/SERVER_SCOPE", new TypeReference<>() {});
Optional<Map<String, Object>> activeAttrOpt = values.stream().filter(att -> att.get("key").equals("active")).findFirst();
if (activeAttrOpt.isEmpty()) {
return false;
}
Map<String, Object> activeAttr = activeAttrOpt.get();
return "true".equals(activeAttr.get("value").toString());
});
verifyEdgeConnected();
}
}

12
application/src/test/java/org/thingsboard/server/edge/DeviceProfileEdgeTest.java

@ -151,14 +151,20 @@ public class DeviceProfileEdgeTest extends AbstractEdgeTest {
// delete profile when edge is offline
edgeImitator.disconnect();
verifyEdgeDisconnected();
doDelete("/api/deviceProfile/" + deviceProfile.getUuidId())
.andExpect(status().isOk());
edgeImitator.connect();
// 25 sync message
// + 2 RuleChain and RuleChainMetadata
// + 1 delete DeviceProfile
// + 1 RuleChain Added
// + 1 RuleChainMetadata Added
// + 1 DeviceProfile Delete
edgeImitator.expectMessageAmount(SYNC_MESSAGE_COUNT + 3);
edgeImitator.connect();
verifyEdgeConnected();
Assert.assertTrue(edgeImitator.waitForMessages());
latestMessage = edgeImitator.getLatestMessage();

148
application/src/test/java/org/thingsboard/server/edge/EdgeStatsIntegrationTest.java

@ -0,0 +1,148 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.edge;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.extern.slf4j.Slf4j;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.edge.EdgeEvent;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.dao.edge.stats.EdgeStatsCounterService;
import org.thingsboard.server.dao.edge.stats.EdgeStatsKey;
import org.thingsboard.server.dao.edge.stats.MsgCounters;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.service.edge.stats.EdgeStatsService;
import java.time.Duration;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import static org.awaitility.Awaitility.await;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_ADDED;
import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_PERMANENTLY_FAILED;
import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_PUSHED;
import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_TMP_FAILED;
@DaoSqlTest
@Slf4j
public class EdgeStatsIntegrationTest extends AbstractEdgeTest {
private static final String STATISTICS_DEVICE_PROFILE = "STATISTICS";
private static final long EXPECTED_MSGS_ADDED = 6L;
private static final long EXPECTED_MSGS_PUSHED = 6L;
private static final long EXPECTED_MSGS_PERMANENTLY_FAILED = 0L;
private static final long EXPECTED_MSGS_TMP_FAILED = 0L;
@Autowired
private EdgeStatsService edgeStatsService;
@Autowired
private EdgeStatsCounterService statsCounterService;
@Test
public void testReportStats() throws Exception {
// GIVEN
simulateEdgeEventsAddedDownlinkPushed();
// Await Edge Counters Updated
await().atMost(10, TimeUnit.SECONDS).pollInterval(Duration.ofMillis(200)).untilAsserted(() -> {
MsgCounters counters = statsCounterService.getMsgCountersByEdge().get(edge.getId());
assertEquals(EXPECTED_MSGS_ADDED, counters.getMsgsAdded().get());
assertEquals(EXPECTED_MSGS_PUSHED, counters.getMsgsPushed().get());
assertEquals(EXPECTED_MSGS_PERMANENTLY_FAILED, counters.getMsgsPermanentlyFailed().get());
assertEquals(EXPECTED_MSGS_TMP_FAILED, counters.getMsgsTmpFailed().get());
});
Thread.sleep(1000);
// WHEN
edgeStatsService.reportStats();
// THEN
await().atMost(10, TimeUnit.SECONDS).pollInterval(Duration.ofMillis(200)).untilAsserted(() -> {
List<TsKvEntry> actualStats = fetchLatestStats();
assertEquals(EXPECTED_MSGS_ADDED, getStatsLongValue(actualStats, DOWNLINK_MSGS_ADDED));
assertEquals(EXPECTED_MSGS_PUSHED, getStatsLongValue(actualStats, DOWNLINK_MSGS_PUSHED));
assertEquals(EXPECTED_MSGS_PERMANENTLY_FAILED, getStatsLongValue(actualStats, DOWNLINK_MSGS_PERMANENTLY_FAILED));
assertEquals(EXPECTED_MSGS_TMP_FAILED, getStatsLongValue(actualStats, DOWNLINK_MSGS_TMP_FAILED));
});
}
private long getStatsLongValue(List<TsKvEntry> stats, EdgeStatsKey key) {
return stats.stream().filter(e -> e.getKey().equals(key.getKey())).findFirst().get().getLongValue().orElse(0L);
}
private List<TsKvEntry> fetchLatestStats() throws ExecutionException, InterruptedException {
return tsService.findLatest(
tenantId,
edge.getId(),
Arrays.stream(EdgeStatsKey.values()).map(EdgeStatsKey::getKey).toList()).get();
}
private void simulateEdgeEventsAddedDownlinkPushed() throws InterruptedException, ExecutionException {
statsCounterService.clear(edge.getId());
// Save device and assign to edge
// 2 DOWNLINK_MSGS_ADDED, EdgeEvents: [{DEVICE_PROFILE: ADDED}, {DEVICE: ASSIGNED_TO_EDGE}]
// 2 DOWNLINK_MSGS_PUSHED, Downlinks: [{deviceProfileUpdateMsg}, {deviceUpdateMsg, deviceProfileUpdateMsg, deviceCredentialsUpdateMsg}]
edgeImitator.expectMessageAmount(4);
Device savedDevice = saveDevice("StatisticDevice", STATISTICS_DEVICE_PROFILE);
doPost("/api/edge/" + edge.getUuidId() + "/device/" + savedDevice.getUuidId(), Device.class);
Assert.assertTrue(edgeImitator.waitForMessages());
// Save asset and assign to edge
// 1 DOWNLINK_MSGS_ADDED, EdgeEvents: [{ASSET: ASSIGNED_TO_EDGE}]
// 1 DOWNLINK_MSGS_PUSHED, Downlinks: [{assetUpdateMsg, assetProfileUpdateMsg}]
edgeImitator.expectMessageAmount(2);
Asset savedAsset = saveAsset("Edge Asset");
doPost("/api/edge/" + edge.getUuidId()
+ "/asset/" + savedAsset.getUuidId(), Asset.class);
Assert.assertTrue(edgeImitator.waitForMessages());
// Create customer and assign edge to the customer
// 2 DOWNLINK_MSGS_ADDED, EdgeEvents: [{CUSTOMER: ADDED}, {EDGE: ASSIGNED_TO_CUSTOMER}]
// 2 DOWNLINK_MSGS_PUSHED, Downlinks: [{customerUpdateMsg}, {edgeConfiguration}]
edgeImitator.expectMessageAmount(2);
Customer customer = new Customer();
customer.setTitle("Edge Customer");
Customer savedCustomer = doPost("/api/customer", customer, Customer.class);
doPost("/api/customer/" + savedCustomer.getUuidId()
+ "/edge/" + edge.getUuidId(), Edge.class);
Assert.assertTrue(edgeImitator.waitForMessages());
// Send device telemetry downlink for the device
// 1 DOWNLINK_MSGS_ADDED, EdgeEvents: [{DEVICE: TIMESERIES_UPDATED}]
// 1 DOWNLINK_MSGS_PUSHED, Downlinks: [{entityData}]
edgeImitator.expectMessageAmount(1);
String timeseriesData = "{\"data\":{\"temperature\":25},\"ts\":" + System.currentTimeMillis() + "}";
JsonNode timeseriesEntityData = JacksonUtil.toJsonNode(timeseriesData);
EdgeEvent edgeEvent = constructEdgeEvent(tenantId, edge.getId(), EdgeEventActionType.TIMESERIES_UPDATED, savedDevice.getId().getId(), EdgeEventType.DEVICE, timeseriesEntityData);
edgeEventService.saveAsync(edgeEvent).get();
Assert.assertTrue(edgeImitator.waitForMessages());
}
}

4
application/src/test/java/org/thingsboard/server/edge/TelemetryEdgeTest.java

@ -223,8 +223,10 @@ public class TelemetryEdgeTest extends AbstractEdgeTest {
device.getId().getId(), EdgeEventType.DEVICE, timeseriesEntityData);
edgeEventService.saveAsync(failedEdgeEvent).get();
// add unique body to avoid merge and filter by device id in edge service (mergeAndFilterDownlinkDuplicates)
JsonNode uniqueBody = JacksonUtil.toJsonNode("{\"idx\":" + idx + "}");
EdgeEvent successEdgeEvent = constructEdgeEvent(tenantId, edge.getId(), EdgeEventActionType.UPDATED,
device.getId().getId(), EdgeEventType.DEVICE, null);
device.getId().getId(), EdgeEventType.DEVICE, uniqueBody);
edgeEventService.saveAsync(successEdgeEvent).get();
}

77
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationArgumentEntryTest.java

@ -104,19 +104,19 @@ public class PropagationArgumentEntryTest {
@Test
void testUpdateEntryWhenAdded() {
var added = new PropagationArgumentEntry();
added.setAdded(ENTITY_3_ID);
added.setAdded(List.of(ENTITY_3_ID));
boolean changed = entry.updateEntry(added);
assertThat(changed).isTrue();
assertThat(entry.getEntityIds()).containsExactlyInAnyOrder(ENTITY_1_ID, ENTITY_2_ID, ENTITY_3_ID);
assertThat(entry.getAdded()).isEqualTo(ENTITY_3_ID);
assertThat(entry.getAdded()).isEqualTo(List.of(ENTITY_3_ID));
}
@Test
void testUpdateEntryWhenAddedExistingEntity() {
var added = new PropagationArgumentEntry();
added.setAdded(ENTITY_2_ID);
added.setAdded(List.of(ENTITY_2_ID));
boolean changed = entry.updateEntry(added);
@ -149,6 +149,77 @@ public class PropagationArgumentEntryTest {
assertThat(entry.getRemoved()).isNull();
}
@Test
void testUpdateEntryWhenPartitionStateRestoreAddsMissingIds() {
var restore = new PropagationArgumentEntry(List.of(ENTITY_1_ID, ENTITY_2_ID, ENTITY_3_ID));
restore.setIgnoreRemovedEntities(true);
boolean changed = entry.updateEntry(restore);
assertThat(changed).isTrue();
assertThat(entry.getEntityIds()).containsExactlyInAnyOrder(ENTITY_1_ID, ENTITY_2_ID, ENTITY_3_ID);
assertThat(entry.getAdded()).containsExactly(ENTITY_3_ID);
assertThat(entry.getRemoved()).isNull();
assertThat(entry.isIgnoreRemovedEntities()).isFalse();
}
@Test
void testUpdateEntryWhenPartitionStateRestoreRemovesStaleIds() {
var restore = new PropagationArgumentEntry(List.of(ENTITY_1_ID));
restore.setIgnoreRemovedEntities(true);
boolean changed = entry.updateEntry(restore);
assertThat(changed).isFalse(); // expected no change, since we consider the removal of stale ids as no-op
assertThat(entry.getEntityIds()).containsExactlyInAnyOrder(ENTITY_1_ID);
assertThat(entry.getAdded()).isNull();
assertThat(entry.getRemoved()).isNull();
assertThat(entry.isIgnoreRemovedEntities()).isFalse();
}
@Test
void testUpdateEntryWhenPartitionStateRestoreAddsAndRemoves() {
var restore = new PropagationArgumentEntry(List.of(ENTITY_1_ID, ENTITY_3_ID));
restore.setIgnoreRemovedEntities(true);
boolean changed = entry.updateEntry(restore);
assertThat(changed).isTrue();
assertThat(entry.getEntityIds()).containsExactlyInAnyOrder(ENTITY_1_ID, ENTITY_3_ID);
assertThat(entry.getAdded()).containsExactly(ENTITY_3_ID);
assertThat(entry.getRemoved()).isNull();
assertThat(entry.isIgnoreRemovedEntities()).isFalse();
}
@Test
void testUpdateEntryWhenPartitionStateRestoreNoChanges() {
var restore = new PropagationArgumentEntry(List.of(ENTITY_1_ID, ENTITY_2_ID));
restore.setIgnoreRemovedEntities(true);
boolean changed = entry.updateEntry(restore);
assertThat(changed).isFalse();
assertThat(entry.getEntityIds()).containsExactlyInAnyOrder(ENTITY_1_ID, ENTITY_2_ID);
assertThat(entry.getAdded()).isNull();
assertThat(entry.getRemoved()).isNull();
assertThat(entry.isIgnoreRemovedEntities()).isFalse();
}
@Test
void testUpdateEntryWhenPartitionStateRestoreEmptySet() {
var restore = new PropagationArgumentEntry(List.of());
restore.setIgnoreRemovedEntities(true);
boolean changed = entry.updateEntry(restore);
assertThat(changed).isFalse(); // expected no change, since we consider the removal of stale ids as no-op
assertThat(entry.getEntityIds()).isEmpty();
assertThat(entry.getAdded()).isNull();
assertThat(entry.getRemoved()).isNull();
assertThat(entry.isIgnoreRemovedEntities()).isFalse();
}
@Test
@SuppressWarnings("unchecked")
void testToTbelCfArgWithValues() {

141
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/PropagationCalculatedFieldStateTest.java

@ -47,6 +47,7 @@ import org.thingsboard.server.common.data.relation.EntitySearchDirection;
import org.thingsboard.server.common.data.relation.RelationPathLevel;
import org.thingsboard.server.common.stats.DefaultStatsFactory;
import org.thingsboard.server.dao.usagerecord.ApiLimitService;
import org.thingsboard.server.service.cf.CalculatedFieldProcessingService;
import org.thingsboard.server.service.cf.PropagationCalculatedFieldResult;
import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult;
import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry;
@ -57,12 +58,17 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.stream.Stream;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration.PROPAGATION_CONFIG_ARGUMENT;
@ -70,17 +76,26 @@ import static org.thingsboard.server.common.data.cf.configuration.PropagationCal
public class PropagationCalculatedFieldStateTest {
private static final String TEMPERATURE_ARGUMENT_NAME = "t";
private static final String HUMIDITY_ARGUMENT_NAME = "h";
private static final String TEST_RESULT_EXPRESSION_KEY = "testResult";
private static final double TEMPERATURE_VALUE = 12.5;
private static final double HUMIDITY_VALUE = 85;
private static final PropagationArgumentEntry EMPTY_PROPAGATION_ARGUMENT = new PropagationArgumentEntry(Collections.emptyList());
private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("6c3513cb-85e7-4510-8746-1ba01859a8ce"));
private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("be960a50-c029-4698-b2ec-c56a543c561c"));
private final AssetId ASSET_ID_1 = new AssetId(UUID.fromString("d26f0e5b-7d7d-4a61-9f5e-08ab97b30734"));
private final AssetId ASSET_ID_2 = new AssetId(UUID.fromString("1933a317-4df5-4d36-9800-68aded74579b"));
private final SingleValueArgumentEntry singleValueArgEntry =
private final SingleValueArgumentEntry EMPTY_SINGLE_VALUE_ARGUMENT = new SingleValueArgumentEntry();
private final SingleValueArgumentEntry temperatureArgumentEntry =
new SingleValueArgumentEntry(System.currentTimeMillis(), new DoubleDataEntry("temperature", TEMPERATURE_VALUE), 99L);
private final SingleValueArgumentEntry humidityArgumentEntry =
new SingleValueArgumentEntry(System.currentTimeMillis(), new DoubleDataEntry("humidity", HUMIDITY_VALUE), 99L);
private final PropagationArgumentEntry propagationArgEntry =
new PropagationArgumentEntry(new ArrayList<>(List.of(ASSET_ID_2, ASSET_ID_1)));
@ -96,15 +111,19 @@ public class PropagationCalculatedFieldStateTest {
@MockitoBean
private ActorSystemContext actorSystemContext;
@MockitoBean
private CalculatedFieldProcessingService cfProcessingService;
@BeforeEach
void setUp() {
when(actorSystemContext.getTbelInvokeService()).thenReturn(tbelInvokeService);
when(actorSystemContext.getApiLimitService()).thenReturn(apiLimitService);
when(actorSystemContext.getCalculatedFieldProcessingService()).thenReturn(cfProcessingService);
when(apiLimitService.getLimit(any(), any())).thenReturn(1000L);
}
void initCtxAndState(boolean applyExpressionToResolvedArguments) {
ctx = new CalculatedFieldCtx(getCalculatedField(applyExpressionToResolvedArguments), actorSystemContext);
ctx = spy(new CalculatedFieldCtx(getCalculatedField(applyExpressionToResolvedArguments), actorSystemContext));
ctx.init();
state = new PropagationCalculatedFieldState(ctx.getEntityId());
@ -121,7 +140,7 @@ public class PropagationCalculatedFieldStateTest {
@Test
void testInitAddsRequiredArgument() {
initCtxAndState(false);
assertThat(state.getRequiredArguments()).containsExactlyInAnyOrder(TEMPERATURE_ARGUMENT_NAME, PROPAGATION_CONFIG_ARGUMENT);
assertThat(state.getRequiredArguments()).containsExactlyInAnyOrder(TEMPERATURE_ARGUMENT_NAME, HUMIDITY_ARGUMENT_NAME, PROPAGATION_CONFIG_ARGUMENT);
}
@Test
@ -133,7 +152,7 @@ public class PropagationCalculatedFieldStateTest {
private static Stream<ArgumentEntry> provideInvalidPropagationArgs() {
return Stream.of(
null,
new PropagationArgumentEntry(Collections.emptyList())
EMPTY_PROPAGATION_ARGUMENT
);
}
@ -143,7 +162,8 @@ public class PropagationCalculatedFieldStateTest {
initCtxAndState(false);
Map<String, ArgumentEntry> args = new HashMap<>();
args.put(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry); // Valid user arg
args.put(TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry); // Valid user arg
args.put(HUMIDITY_ARGUMENT_NAME, humidityArgumentEntry); // Valid user arg
if (propagationEntry != null) {
args.put(PROPAGATION_CONFIG_ARGUMENT, propagationEntry);
@ -155,19 +175,54 @@ public class PropagationCalculatedFieldStateTest {
}
@Test
void testIsReadyWhenPropagationArgHasEntities() {
void testIsReadyWithoutExpressionWhenAllArgumentsAreNotEmpty() {
initCtxAndState(false);
state.update(Map.of(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry, PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry), ctx);
Map<String, ArgumentEntry> updatedArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, humidityArgumentEntry,
PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry
);
state.update(updatedArgs, ctx);
assertThat(state.isReady()).isTrue();
assertThat(state.getReadinessStatus().errorMsg()).isNull();
}
@Test
void testIsReadyWithoutExpressionWhenAtLeastOneArgumentIsNotEmpty() {
initCtxAndState(false);
Map<String, ArgumentEntry> updatedArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, EMPTY_SINGLE_VALUE_ARGUMENT,
PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry);
state.update(updatedArgs, ctx);
assertThat(state.isReady()).isTrue();
assertThat(state.getReadinessStatus().errorMsg()).isNull();
}
@Test
void testIsNotReadyWithExpressionWhenAtLeastOneArgumentIsEmpty() {
initCtxAndState(true);
Map<String, ArgumentEntry> updatedArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, EMPTY_SINGLE_VALUE_ARGUMENT,
PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry);
state.update(updatedArgs, ctx);
assertThat(state.isReady()).isFalse();
assertThat(state.getReadinessStatus().errorMsg()).isEqualTo("Required arguments are missing: h");
}
@Test
void testPerformCalculationWithEmptyPropagationArg() throws Exception {
initCtxAndState(false);
state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, new PropagationArgumentEntry(Collections.emptyList()));
Map<String, ArgumentEntry> initArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, humidityArgumentEntry,
PROPAGATION_CONFIG_ARGUMENT, EMPTY_PROPAGATION_ARGUMENT);
state.update(initArgs, ctx);
assertThat(state.isReady()).isFalse();
// test empty propagation argument calculation
PropagationCalculatedFieldResult result = performCalculation();
assertThat(result).isNotNull();
@ -178,8 +233,12 @@ public class PropagationCalculatedFieldStateTest {
@Test
void testPerformCalculationWithArgumentsOnlyMode() throws Exception {
initCtxAndState(false);
state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry);
state.getArguments().put(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry);
Map<String, ArgumentEntry> initArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, EMPTY_SINGLE_VALUE_ARGUMENT,
PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry);
state.update(initArgs, ctx);
assertThat(state.isReady()).isTrue();
PropagationCalculatedFieldResult propagationResult = performCalculation();
@ -193,7 +252,7 @@ public class PropagationCalculatedFieldStateTest {
assertThat(result.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE);
ObjectNode expectedNode = JacksonUtil.newObjectNode();
JacksonUtil.addKvEntry(expectedNode, singleValueArgEntry.getKvEntryValue(), TEMPERATURE_ARGUMENT_NAME);
JacksonUtil.addKvEntry(expectedNode, temperatureArgumentEntry.getKvEntryValue(), TEMPERATURE_ARGUMENT_NAME);
assertThat(result.getResult()).isEqualTo(expectedNode);
}
@ -201,9 +260,13 @@ public class PropagationCalculatedFieldStateTest {
@Test
void testPerformCalculationWithExpressionResultMode() throws Exception {
initCtxAndState(true);
state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry);
state.getArguments().put(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry);
Map<String, ArgumentEntry> initArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, humidityArgumentEntry,
PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry
);
state.update(initArgs, ctx);
assertThat(state.isReady()).isTrue();
PropagationCalculatedFieldResult propagationResult = performCalculation();
assertThat(propagationResult).isNotNull();
@ -216,7 +279,7 @@ public class PropagationCalculatedFieldStateTest {
assertThat(result.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE);
ObjectNode expectedNode = JacksonUtil.newObjectNode();
expectedNode.put(TEST_RESULT_EXPRESSION_KEY, TEMPERATURE_VALUE * 2);
expectedNode.put(TEST_RESULT_EXPRESSION_KEY, (TEMPERATURE_VALUE + HUMIDITY_VALUE) / 2);
assertThat(result.getResult()).isEqualTo(expectedNode);
}
@ -224,18 +287,52 @@ public class PropagationCalculatedFieldStateTest {
@Test
void testPropagationWithUpdatedPropagationArgument() throws ExecutionException, InterruptedException {
initCtxAndState(false);
state.getArguments().put(PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry);
state.getArguments().put(TEMPERATURE_ARGUMENT_NAME, singleValueArgEntry);
Map<String, ArgumentEntry> initArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, EMPTY_SINGLE_VALUE_ARGUMENT,
PROPAGATION_CONFIG_ARGUMENT, propagationArgEntry
);
state.update(initArgs, ctx);
assertThat(state.isReady()).isTrue();
AssetId newEntityId = new AssetId(UUID.fromString("83e2c962-eeae-4708-984e-e6a24760f9c3"));
PropagationArgumentEntry propagationArgumentEntry = new PropagationArgumentEntry();
propagationArgumentEntry.setAdded(newEntityId);
propagationArgumentEntry.setAdded(List.of(newEntityId));
Map<String, ArgumentEntry> updated = state.update(Map.of(PROPAGATION_CONFIG_ARGUMENT, propagationArgumentEntry), ctx);
assertThat(updated).isNotNull().containsEntry(PROPAGATION_CONFIG_ARGUMENT, propagationArgumentEntry);
PropagationCalculatedFieldResult propagationCalculatedFieldResult = performCalculation(updated);
assertThat(propagationCalculatedFieldResult).isNotNull();
assertThat(propagationCalculatedFieldResult.getEntityIds()).isNotNull().containsExactly(newEntityId);
assertThat(propagationCalculatedFieldResult.getResult()).isNotNull();
assertThat(propagationCalculatedFieldResult.getResult().getResult()).isNotNull();
assertThat(propagationCalculatedFieldResult.getResult().getResult()).isEqualTo(JacksonUtil.newObjectNode().put(TEMPERATURE_ARGUMENT_NAME, TEMPERATURE_VALUE));
}
@Test
void testPropapagationStateInitWithRestoredSetToFalse() {
initCtxAndState(false);
verify(cfProcessingService, never()).fetchPropagationArgumentFromDb(any(), any());
verify(ctx, never()).scheduleReevaluation(anyLong(), any());
}
@Test
void testPropapagationStateInitWithRestoredSetToTrue() {
initCtxAndState(false);
Map<String, ArgumentEntry> initArgs = Map.of(
TEMPERATURE_ARGUMENT_NAME, temperatureArgumentEntry,
HUMIDITY_ARGUMENT_NAME, humidityArgumentEntry,
PROPAGATION_CONFIG_ARGUMENT, new PropagationArgumentEntry(Collections.emptyList())
);
state.update(initArgs, ctx);
assertThat(state.isReady()).isFalse();
when(cfProcessingService.fetchPropagationArgumentFromDb(any(), any())).thenReturn(Optional.of(propagationArgEntry));
state.init(true);
verify(cfProcessingService).fetchPropagationArgumentFromDb(ctx, state.getEntityId());
verify(ctx).scheduleReevaluation(0L, state.getActorCtx());
}
private CalculatedField getCalculatedField(boolean applyExpressionToResolvedArguments) {
@ -260,8 +357,12 @@ public class PropagationCalculatedFieldStateTest {
ReferencedEntityKey tempKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null);
temperatureArg.setRefEntityKey(tempKey);
config.setArguments(Map.of(TEMPERATURE_ARGUMENT_NAME, temperatureArg));
config.setExpression("{" + TEST_RESULT_EXPRESSION_KEY + ": " + TEMPERATURE_ARGUMENT_NAME + " * 2}");
Argument humidityArg = new Argument();
ReferencedEntityKey humidityKey = new ReferencedEntityKey("humidity", ArgumentType.TS_LATEST, null);
humidityArg.setRefEntityKey(humidityKey);
config.setArguments(Map.of(TEMPERATURE_ARGUMENT_NAME, temperatureArg, HUMIDITY_ARGUMENT_NAME, humidityArg));
config.setExpression("return { " + TEST_RESULT_EXPRESSION_KEY + ": (" + TEMPERATURE_ARGUMENT_NAME + " + " + HUMIDITY_ARGUMENT_NAME + ") / 2};");
AttributesOutput output = new AttributesOutput();
output.setScope(AttributeScope.SERVER_SCOPE);

4
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesAggregationCalculatedFieldStateTest.java

@ -113,8 +113,10 @@ public class RelatedEntitiesAggregationCalculatedFieldStateTest {
}
@Test
void testIsReadyReturnFalseWhenNoArgumentsSet() {
void testIsReadyWhenNoRelatedEntities() {
assertThat(state.isReady()).isFalse();
assertThat(state.getReadinessStatus().errorMsg())
.isEqualTo("No entities found via 'Aggregation path to related entities'. Verify the configured relation type and direction.");
}
@Test

98
application/src/test/java/org/thingsboard/server/service/edge/EdgeMsgConstructorUtilsTest.java

@ -15,10 +15,12 @@
*/
package org.thingsboard.server.service.edge;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
@ -36,6 +38,10 @@ import org.thingsboard.rule.engine.rest.TbSendRestApiCallReplyNode;
import org.thingsboard.rule.engine.telemetry.TbCalculatedFieldsNode;
import org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode;
import org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode;
import org.thingsboard.server.common.data.edge.EdgeEvent;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.gen.edge.v1.EdgeVersion;
@ -44,6 +50,7 @@ import org.thingsboard.server.gen.edge.v1.UpdateMsgType;
import java.lang.reflect.Constructor;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Stream;
import static org.thingsboard.server.service.edge.EdgeMsgConstructorUtils.EXCLUDED_NODES_BY_EDGE_VERSION;
@ -155,4 +162,95 @@ public class EdgeMsgConstructorUtilsTest {
String.format("For EdgeVersion '%s', ruleNode '%s' should not be included.", edgeVersion, ruleNode.getType()));
}
@Test
@DisplayName("mergeDownlinkDuplicates: latest per attribute key is retained and duplicates removed")
public void testMergeDownlinkDuplicates() {
UUID deviceId = UUID.randomUUID();
UUID assetId = UUID.randomUUID();
TenantId tenantId = TenantId.fromUUID(UUID.randomUUID());
var deviceAttrUpdate1 = createEdgeEvent(tenantId, 1, EdgeEventActionType.ATTRIBUTES_UPDATED,
deviceId, EdgeEventType.DEVICE, createAttrBody(1_000L, "{\"a\":1,\"b\":1,\"d\":1}"));
var deviceAttrUpdate2 = createEdgeEvent(tenantId, 2, EdgeEventActionType.ATTRIBUTES_UPDATED,
deviceId, EdgeEventType.DEVICE, createAttrBody(2_000L, "{\"a\":2,\"b\":2,\"c\":2}"));
var deviceAttrUpdate3 = createEdgeEvent(tenantId, 3, EdgeEventActionType.ATTRIBUTES_UPDATED,
deviceId, EdgeEventType.DEVICE, createAttrBody(3_000L, "{\"a\":3,\"d\":3}"));
var deviceUpdate = createEdgeEvent(tenantId, 4, EdgeEventActionType.UPDATED,
deviceId, EdgeEventType.DEVICE, null);
var deviceUpdateDup = createEdgeEvent(tenantId, 5, EdgeEventActionType.UPDATED,
deviceId, EdgeEventType.DEVICE, null);
var assetAttrUpdate1 = createEdgeEvent(tenantId, 6, EdgeEventActionType.ATTRIBUTES_UPDATED,
assetId, EdgeEventType.ASSET, createAttrBody(6_000L, "{\"a\":6,\"d\":6}"));
var assetAttrUpdate2 = createEdgeEvent(tenantId, 7, EdgeEventActionType.ATTRIBUTES_UPDATED,
assetId, EdgeEventType.ASSET, createAttrBody(7_000L, "{\"a\":7,\"b\":7,\"c\":7}"));
var assetAttrUpdate3 = createEdgeEvent(tenantId, 8, EdgeEventActionType.ATTRIBUTES_UPDATED,
assetId, EdgeEventType.ASSET, createAttrBody(8_000L, "{\"a\":8,\"d\":8}"));
List<EdgeEvent> input = List.of(deviceAttrUpdate1, deviceAttrUpdate2, deviceAttrUpdate3,
deviceUpdate, deviceUpdateDup,
assetAttrUpdate1, assetAttrUpdate2, assetAttrUpdate3);
List<EdgeEvent> merged = EdgeMsgConstructorUtils.mergeAndFilterDownlinkDuplicates(input);
Assertions.assertEquals(5, merged.size());
EdgeEvent deviceMergedAttrBC = merged.get(0);
Assertions.assertEquals(2, deviceMergedAttrBC.getSeqId());
Assertions.assertEquals(deviceId, deviceMergedAttrBC.getEntityId());
Assertions.assertEquals(2_000L, deviceMergedAttrBC.getBody().get("ts").asLong());
Assertions.assertEquals(2, getIntValue(deviceMergedAttrBC.getBody(), "b"));
Assertions.assertEquals(2, getIntValue(deviceMergedAttrBC.getBody(), "c"));
Assertions.assertNull(getIntValue(deviceMergedAttrBC.getBody(), "a"));
EdgeEvent deviceMergedAttrAD = merged.get(1);
Assertions.assertEquals(3, deviceMergedAttrAD.getSeqId());
Assertions.assertEquals(deviceId, deviceMergedAttrAD.getEntityId());
Assertions.assertEquals(3_000L, deviceMergedAttrAD.getBody().get("ts").asLong());
Assertions.assertEquals(3, getIntValue(deviceMergedAttrAD.getBody(), "a"));
Assertions.assertEquals(3, getIntValue(deviceMergedAttrAD.getBody(), "d"));
EdgeEvent mergedDeviceUpdate = merged.get(2);
Assertions.assertEquals(4, mergedDeviceUpdate.getSeqId());
Assertions.assertEquals(EdgeEventActionType.UPDATED, mergedDeviceUpdate.getAction());
EdgeEvent assetMergedAttrBC = merged.get(3);
Assertions.assertEquals(7, assetMergedAttrBC.getSeqId());
Assertions.assertEquals(assetId, assetMergedAttrBC.getEntityId());
Assertions.assertEquals(7_000L, assetMergedAttrBC.getBody().get("ts").asLong());
Assertions.assertEquals(7, getIntValue(assetMergedAttrBC.getBody(), "b"));
Assertions.assertEquals(7, getIntValue(assetMergedAttrBC.getBody(), "c"));
Assertions.assertNull(getIntValue(assetMergedAttrBC.getBody(), "a"));
EdgeEvent assetMergedAttrAD = merged.get(4);
Assertions.assertEquals(8, assetMergedAttrAD.getSeqId());
Assertions.assertEquals(assetId, assetMergedAttrAD.getEntityId());
Assertions.assertEquals(8_000L, assetMergedAttrAD.getBody().get("ts").asLong());
Assertions.assertEquals(8, getIntValue(assetMergedAttrAD.getBody(), "a"));
Assertions.assertEquals(8, getIntValue(assetMergedAttrAD.getBody(), "d"));
}
private Integer getIntValue(JsonNode body, String key) {
return body.get("kv").get(key) != null ? body.get("kv").get(key).asInt() : null;
}
private static JsonNode createAttrBody(long ts, String kvJson) {
return JacksonUtil.toJsonNode("{\"ts\":" + ts + ",\"kv\":" + kvJson + "}");
}
private static EdgeEvent createEdgeEvent(TenantId tenantId,
long seqId,
EdgeEventActionType action,
UUID entityId,
EdgeEventType type,
JsonNode body) {
EdgeEvent edgeEvent = new EdgeEvent();
edgeEvent.setSeqId(seqId);
edgeEvent.setTenantId(tenantId);
edgeEvent.setAction(action);
edgeEvent.setEntityId(entityId);
edgeEvent.setType(type);
edgeEvent.setBody(body);
return edgeEvent;
}
}

133
application/src/test/java/org/thingsboard/server/service/edge/EdgeStatsTest.java

@ -21,6 +21,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.test.util.ReflectionTestUtils;
@ -44,9 +45,11 @@ import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_ADDED;
@ -58,6 +61,16 @@ import static org.thingsboard.server.dao.edge.stats.EdgeStatsKey.DOWNLINK_MSGS_T
@ExtendWith(MockitoExtension.class)
public class EdgeStatsTest {
private static final int TTL_DAYS = 30;
private static final long REPORT_INTERVAL_MILLIS = 600_000L;
private static final long EXPECTED_MSGS_ADDED = 5L;
private static final long EXPECTED_MSGS_PUSHED = 3L;
private static final long EXPECTED_MSGS_PERMANENTLY_FAILED = 1L;
private static final long EXPECTED_MSGS_TMP_FAILED = 0L;
private static final long EXPECTED_MSGS_LAG = 10L;
private static final long EXPECTED_MSGS_KAFKA_LAG = 15L;
@Mock
private TimeseriesService tsService;
@Mock
@ -66,108 +79,100 @@ public class EdgeStatsTest {
private EdgeStatsCounterService statsCounterService;
private EdgeStatsService edgeStatsService;
@Captor
private ArgumentCaptor<List<TsKvEntry>> captor;
private final TenantId tenantId = TenantId.fromUUID(UUID.randomUUID());
private final EdgeId edgeId = new EdgeId(UUID.randomUUID());
@BeforeEach
void setUp() {
edgeStatsService = new EdgeStatsService(
edgeStatsService = createEdgeStatsService(Optional.empty());
}
private EdgeStatsService createEdgeStatsService(Optional<KafkaAdmin> kafkaAdmin) {
EdgeStatsService service = new EdgeStatsService(
tsService,
statsCounterService,
topicService,
Optional.empty()
kafkaAdmin
);
ReflectionTestUtils.setField(edgeStatsService, "edgesStatsTtlDays", 30);
ReflectionTestUtils.setField(edgeStatsService, "reportIntervalMillis", 600_000L);
ReflectionTestUtils.setField(service, "edgesStatsTtlDays", TTL_DAYS);
ReflectionTestUtils.setField(service, "reportIntervalMillis", REPORT_INTERVAL_MILLIS);
return service;
}
@Test
public void testReportStatsSavesTelemetry() {
// given
// GIVEN
setupCounters();
// WHEN
edgeStatsService.reportStats();
// THEN
Map<String, Long> counters = verifyCounters();
Assertions.assertEquals(EXPECTED_MSGS_LAG, counters.get(DOWNLINK_MSGS_LAG.getKey()).longValue());
}
@Test
public void testReportStatsWithKafkaLag() {
// GIVEN
setupCounters();
setupKafkaLag();
// WHEN
edgeStatsService.reportStats();
// THEN
Map<String, Long> valuesByKey = verifyCounters();
Assertions.assertEquals(EXPECTED_MSGS_KAFKA_LAG, valuesByKey.get(DOWNLINK_MSGS_LAG.getKey()));
}
private void setupCounters() {
MsgCounters counters = new MsgCounters(tenantId);
counters.getMsgsAdded().set(5);
counters.getMsgsPushed().set(3);
counters.getMsgsPermanentlyFailed().set(1);
counters.getMsgsTmpFailed().set(0);
counters.getMsgsLag().set(10);
counters.getMsgsAdded().set(EXPECTED_MSGS_ADDED);
counters.getMsgsPushed().set(EXPECTED_MSGS_PUSHED);
counters.getMsgsPermanentlyFailed().set(EXPECTED_MSGS_PERMANENTLY_FAILED);
counters.getMsgsTmpFailed().set(EXPECTED_MSGS_TMP_FAILED);
counters.getMsgsLag().set(EXPECTED_MSGS_LAG);
ConcurrentHashMap<EdgeId, MsgCounters> countersByEdge = new ConcurrentHashMap<>();
countersByEdge.put(edgeId, counters);
when(statsCounterService.getCounterByEdge()).thenReturn(countersByEdge);
when(statsCounterService.getMsgCountersByEdge()).thenReturn(countersByEdge);
ArgumentCaptor<List<TsKvEntry>> captor = ArgumentCaptor.forClass(List.class);
when(tsService.save(eq(tenantId), eq(edgeId), captor.capture(), anyLong()))
.thenReturn(Futures.immediateFuture(mock(TimeseriesSaveResult.class)));
}
// when
edgeStatsService.reportStats();
private Map<String, Long> verifyCounters() {
verify(tsService, times(1)).save(eq(tenantId), eq(edgeId), anyList(), anyLong());
verify(statsCounterService, times(1)).clear(edgeId);
// then
List<TsKvEntry> entries = captor.getValue();
Assertions.assertEquals(5, entries.size());
Map<String, Long> valuesByKey = entries.stream()
.collect(Collectors.toMap(TsKvEntry::getKey, e -> e.getLongValue().orElse(-1L)));
Assertions.assertEquals(5L, valuesByKey.get(DOWNLINK_MSGS_ADDED.getKey()).longValue());
Assertions.assertEquals(3L, valuesByKey.get(DOWNLINK_MSGS_PUSHED.getKey()).longValue());
Assertions.assertEquals(1L, valuesByKey.get(DOWNLINK_MSGS_PERMANENTLY_FAILED.getKey()).longValue());
Assertions.assertEquals(0L, valuesByKey.get(DOWNLINK_MSGS_TMP_FAILED.getKey()).longValue());
Assertions.assertEquals(10L, valuesByKey.get(DOWNLINK_MSGS_LAG.getKey()).longValue());
verify(statsCounterService).clear(edgeId);
Assertions.assertEquals(EXPECTED_MSGS_ADDED, valuesByKey.get(DOWNLINK_MSGS_ADDED.getKey()).longValue());
Assertions.assertEquals(EXPECTED_MSGS_PUSHED, valuesByKey.get(DOWNLINK_MSGS_PUSHED.getKey()).longValue());
Assertions.assertEquals(EXPECTED_MSGS_PERMANENTLY_FAILED, valuesByKey.get(DOWNLINK_MSGS_PERMANENTLY_FAILED.getKey()).longValue());
Assertions.assertEquals(EXPECTED_MSGS_TMP_FAILED, valuesByKey.get(DOWNLINK_MSGS_TMP_FAILED.getKey()).longValue());
return valuesByKey;
}
@Test
public void testReportStatsWithKafkaLag() {
// given
MsgCounters counters = new MsgCounters(tenantId);
counters.getMsgsAdded().set(2);
counters.getMsgsPushed().set(2);
counters.getMsgsPermanentlyFailed().set(0);
counters.getMsgsTmpFailed().set(1);
counters.getMsgsLag().set(0);
ConcurrentHashMap<EdgeId, MsgCounters> countersByEdge = new ConcurrentHashMap<>();
countersByEdge.put(edgeId, counters);
// mocks
when(statsCounterService.getCounterByEdge()).thenReturn(countersByEdge);
private void setupKafkaLag() {
String topic = "edge-topic";
TopicPartitionInfo partitionInfo = new TopicPartitionInfo(topic, tenantId, 0, false);
when(topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId)).thenReturn(partitionInfo);
KafkaAdmin kafkaAdmin = mock(KafkaAdmin.class);
when(kafkaAdmin.getTotalLagForGroupsBulk(Set.of(topic)))
.thenReturn(Map.of(topic, 15L));
ArgumentCaptor<List<TsKvEntry>> captor = ArgumentCaptor.forClass(List.class);
when(tsService.save(eq(tenantId), eq(edgeId), captor.capture(), anyLong()))
.thenReturn(Futures.immediateFuture(mock(TimeseriesSaveResult.class)));
edgeStatsService = new EdgeStatsService(
tsService,
statsCounterService,
topicService,
Optional.of(kafkaAdmin)
);
ReflectionTestUtils.setField(edgeStatsService, "edgesStatsTtlDays", 30);
ReflectionTestUtils.setField(edgeStatsService, "reportIntervalMillis", 600_000L);
// when
edgeStatsService.reportStats();
// then
List<TsKvEntry> entries = captor.getValue();
Map<String, Long> valuesByKey = entries.stream()
.collect(Collectors.toMap(TsKvEntry::getKey, e -> e.getLongValue().orElse(-1L)));
.thenReturn(Map.of(topic, EXPECTED_MSGS_KAFKA_LAG));
Assertions.assertEquals(15L, valuesByKey.get(DOWNLINK_MSGS_LAG.getKey()));
verify(statsCounterService).clear(edgeId);
edgeStatsService = createEdgeStatsService(Optional.of(kafkaAdmin));
}
}

2
application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java

@ -60,7 +60,7 @@ import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.controller.AbstractControllerTest;
import org.thingsboard.server.dao.ai.AiModelService;
import org.thingsboard.server.dao.dashboard.DashboardService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.service.DaoSqlTest;

66
application/src/test/java/org/thingsboard/server/transport/lwm2m/ota/AbstractOtaLwM2MIntegrationTest.java

@ -16,14 +16,10 @@
package org.thingsboard.server.transport.lwm2m.ota;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.github.dockerjava.zerodep.shaded.org.apache.commons.codec.binary.Hex;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.leshan.core.ResponseCode;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.test.web.servlet.request.MockMultipartHttpServletRequestBuilder;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.OtaPackageInfo;
import org.thingsboard.server.common.data.id.DeviceProfileId;
@ -35,20 +31,13 @@ import org.thingsboard.server.transport.lwm2m.AbstractLwM2MIntegrationTest;
import java.util.Comparator;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import static org.thingsboard.rest.client.utils.RestJsonConverter.toTimeseries;
import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE;
import static org.thingsboard.server.common.data.ota.OtaPackageType.SOFTWARE;
import static org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService.OTA_INFO_19_FILE_CHECKSUM256;
import static org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService.OTA_INFO_19_FILE_NAME;
import static org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService.OTA_INFO_19_FILE_SIZE;
import static org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService.OTA_INFO_19_TITLE;
import static org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService.OTA_INFO_19_VERSION;
@Slf4j
@DaoSqlTest
@ -57,7 +46,6 @@ public abstract class AbstractOtaLwM2MIntegrationTest extends AbstractLwM2MInteg
protected static final String CLIENT_ENDPOINT_WITHOUT_FW_INFO = "WithoutFirmwareInfoDevice";
protected static final String CLIENT_ENDPOINT_OTA5 = "Ota5_Device";
protected static final String CLIENT_ENDPOINT_OTA9 = "Ota9_Device";
protected static final String CLIENT_ENDPOINT_OTA9_19 = "Ota9_Device_19";
protected List<OtaPackageUpdateStatus> expectedStatuses;
protected final String OBSERVE_ATTRIBUTES_WITH_PARAMS_OTA5 =
@ -88,37 +76,6 @@ public abstract class AbstractOtaLwM2MIntegrationTest extends AbstractLwM2MInteg
" \"attributeLwm2m\": {}\n" +
" }";
protected final String OBSERVE_ATTRIBUTES_WITH_PARAMS_OTA5_19 =
" {\n" +
" \"keyName\": {\n" +
" \"/5_1.2/0/3\": \"state\",\n" +
" \"/5_1.2/0/5\": \"updateResult\",\n" +
" \"/5_1.2/0/6\": \"pkgname\",\n" +
" \"/5_1.2/0/7\": \"pkgversion\",\n" +
" \"/5_1.2/0/9\": \"firmwareUpdateDeliveryMethod\",\n" +
" \"/19_1.1/0/0\": \"dataRead\"\n" +
" },\n" +
" \"observe\": [\n" +
" \"/5_1.2/0/3\",\n" +
" \"/5_1.2/0/5\",\n" +
" \"/5_1.2/0/6\",\n" +
" \"/5_1.2/0/7\",\n" +
" \"/5_1.2/0/9\",\n" +
" \"/19_1.1/0/0\"\n" +
" ],\n" +
" \"attribute\": [],\n" +
" \"telemetry\": [\n" +
" \"/5_1.2/0/3\",\n" +
" \"/5_1.2/0/5\",\n" +
" \"/5_1.2/0/6\",\n" +
" \"/5_1.2/0/7\",\n" +
" \"/5_1.2/0/9\",\n" +
" \"/19_1.1/0/0\"\n" +
" ],\n" +
" \"attributeLwm2m\": {}\n" +
" }";
public static final String CLIENT_LWM2M_SETTINGS_19 =
" {\n" +
" \"useObject19ForOtaInfo\": true,\n" +
@ -247,27 +204,4 @@ public abstract class AbstractOtaLwM2MIntegrationTest extends AbstractLwM2MInteg
log.warn("{}", statuses);
return statuses.containsAll(expectedStatuses);
}
protected void resultReadOtaParams_19(String resourceIdVer, OtaPackageInfo otaPackageInfo) throws Exception {
String actualResult = sendRPCById(resourceIdVer);
ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class);
assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText());
String valStr = rpcActualResult.get("value").asText();
String start = "{ id=0 value=";
String valHexDec = valStr.substring(valStr.indexOf(start) + start.length(), (valStr.indexOf("}")));
String valNode = new String(Hex.decodeHex((valHexDec).toCharArray()));
ObjectNode actualResultVal = JacksonUtil.fromString(valNode, ObjectNode.class);
assert actualResultVal != null;
assertEquals(otaPackageInfo.getTitle(), actualResultVal.get(OTA_INFO_19_TITLE).asText());
assertEquals(otaPackageInfo.getVersion(), actualResultVal.get(OTA_INFO_19_VERSION).asText());
assertEquals(otaPackageInfo.getChecksum(), actualResultVal.get(OTA_INFO_19_FILE_CHECKSUM256).asText());
assertEquals(otaPackageInfo.getFileName(), actualResultVal.get(OTA_INFO_19_FILE_NAME).asText());
assertEquals(Optional.of(otaPackageInfo.getDataSize()), Optional.of((long) actualResultVal.get(OTA_INFO_19_FILE_SIZE).asInt()));
}
private String sendRPCById(String path) throws Exception {
String setRpcRequest = "{\"method\": \"Read\", \"params\": {\"id\": \"" + path + "\"}}";
return doPostAsync("/api/plugins/rpc/twoway/" + lwM2MTestClient.getDeviceIdStr(), setRpcRequest, String.class, status().isOk());
}
}

51
application/src/test/java/org/thingsboard/server/transport/lwm2m/ota/sql/Ota5LwM2MIntegrationTest.java

@ -21,7 +21,6 @@ import org.junit.Assert;
import org.junit.Test;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.OtaPackageInfo;
import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MDeviceCredentials;
import org.thingsboard.server.common.data.device.profile.Lwm2mDeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.kv.KvEntry;
@ -46,10 +45,7 @@ import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.QUEU
import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.UPDATED;
import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.UPDATING;
import static org.thingsboard.server.dao.service.OtaPackageServiceTest.TARGET_FW_VERSION;
import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.BINARY_APP_DATA_CONTAINER;
import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.NONE;
import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_0;
import static org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService.FW_INSTANCE_ID;
@Slf4j
public class Ota5LwM2MIntegrationTest extends AbstractOtaLwM2MIntegrationTest {
@ -111,51 +107,4 @@ public class Ota5LwM2MIntegrationTest extends AbstractOtaLwM2MIntegrationTest {
.until(() -> getFwSwStateTelemetryFromAPI(device.getId().getId(), "fw_state"), this::predicateForStatuses);
log.warn("Object5: Got the ts: {}", ts);
}
/**
* ObjectId = 19/65533/0
* {
* "title" : "My firmware",
* "version" : "fw.v.1.5.0-update",
* "checksum" : "4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a",
* "fileSize" : 1,
* "fileName" : "filename.txt"
* }
* to base64
* /5/0/5 -> Update Result (Res); 5/0/3 -> State;
* => ((Res>=0 && Res<=9) && State=0)
* => Write to Package/Write to Package URI -> DOWNLOADING ((Res>=0 && Res<=9) && State=1)
* => Download Finished -> DOWNLOADED ((Res==0 || Res=8) && State=2)
* => Executable resource Update is triggered / Initiate Firmware Update -> UPDATING (Res=0 && State=3)
* => Update Successful [Res==1]
* => Start / Res=0 -> "IDLE" ....
* @throws Exception
*/
@Test
public void testFirmwareUpdateByObject5WithObject19_Ok() throws Exception {
Lwm2mDeviceProfileTransportConfiguration transportConfiguration = getTransportConfiguration19(OBSERVE_ATTRIBUTES_WITH_PARAMS_OTA5_19, getBootstrapServerCredentialsNoSec(NONE));
DeviceProfile deviceProfile = createLwm2mDeviceProfile("profileFor" + this.CLIENT_ENDPOINT_OTA5 + "19_Ok", transportConfiguration);
String endpoint = this.CLIENT_ENDPOINT_OTA5 + "19_Ok";
LwM2MDeviceCredentials deviceCredentials = getDeviceCredentialsNoSec(createNoSecClientCredentials(endpoint));
final Device device = createLwm2mDevice(deviceCredentials, endpoint, deviceProfile.getId());
createNewClient(SECURITY_NO_SEC, null, false, endpoint, device.getId().getId().toString());
awaitObserveReadAll(6, device.getId().getId().toString());
OtaPackageInfo otaPackageInfo = createFirmware(TARGET_FW_VERSION, deviceProfile.getId());
device.setFirmwareId(otaPackageInfo.getId());
final Device savedDevice = doPost("/api/device", device, Device.class);
assertThat(savedDevice).as("saved device").isNotNull();
assertThat(getDeviceFromAPI(device.getId().getId())).as("fetched device").isEqualTo(savedDevice);
expectedStatuses = Arrays.asList(QUEUED, INITIATED, DOWNLOADING, DOWNLOADED, UPDATING, UPDATED);
List<TsKvEntry> ts = await("await on timeseries for FW")
.atMost(TIMEOUT, TimeUnit.SECONDS)
.until(() -> getFwSwStateTelemetryFromAPI(device.getId().getId(), "fw_state"), this::predicateForStatuses);
String ver_Id_19 = lwM2MTestClient.getLeshanClient().getObjectTree().getModel().getObjectModel(BINARY_APP_DATA_CONTAINER).version;
String resourceIdVer = "/" + BINARY_APP_DATA_CONTAINER + "_" + ver_Id_19 + "/" + FW_INSTANCE_ID + "/" + RESOURCE_ID_0;
resultReadOtaParams_19(resourceIdVer, otaPackageInfo);
log.warn("Object5 with Object19: Got the ts: {}", ts);
}
}

50
application/src/test/java/org/thingsboard/server/transport/lwm2m/ota/sql/Ota9LwM2MIntegrationTest.java

@ -19,7 +19,6 @@ import lombok.extern.slf4j.Slf4j;
import org.junit.Test;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.OtaPackageInfo;
import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MDeviceCredentials;
import org.thingsboard.server.common.data.device.profile.Lwm2mDeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.kv.TsKvEntry;
@ -36,10 +35,7 @@ import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.INIT
import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.QUEUED;
import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.UPDATED;
import static org.thingsboard.server.common.data.ota.OtaPackageUpdateStatus.VERIFIED;
import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.BINARY_APP_DATA_CONTAINER;
import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.NONE;
import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_0;
import static org.thingsboard.server.transport.lwm2m.server.ota.DefaultLwM2MOtaUpdateService.SW_INSTANCE_ID;
@Slf4j
public class Ota9LwM2MIntegrationTest extends AbstractOtaLwM2MIntegrationTest {
@ -51,7 +47,8 @@ public class Ota9LwM2MIntegrationTest extends AbstractOtaLwM2MIntegrationTest {
* => PKG integrity verified -> DELIVERED (Res=3 (Successfully Downloaded and package integrity verified) && State=3) -> INSTALLED;
* => Install -> INSTALLED (Res=2 SW successfully installed) && State=4) -> Start
*
* */
*
*/
@Test
public void testSoftwareUpdateByObject9() throws Exception {
String clientEndpoint = this.CLIENT_ENDPOINT_OTA9;
@ -75,47 +72,4 @@ public class Ota9LwM2MIntegrationTest extends AbstractOtaLwM2MIntegrationTest {
.until(() -> getFwSwStateTelemetryFromAPI(device.getId().getId(), "sw_state"), this::predicateForStatuses);
log.warn("Object9: Got the ts: {}", ts);
}
/**
* ObjectId = 19/65534/0
* {
* "title" : "My sw",
* "version" : "v1.0.19",
* "checksum" : "4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a",
* "fileSize" : 1,
* "fileName" : "filename.txt"
* }
* => Start -> INITIAL (State=0) -> DOWNLOAD STARTED;
* => PKG / URI Write -> DOWNLOAD STARTED (Res=1 (Downloading) && State=1) -> DOWNLOADED
* => PKG Written -> DOWNLOADED (Res=1 Initial && State=2) -> DELIVERED;
* => PKG integrity verified -> DELIVERED (Res=3 (Successfully Downloaded and package integrity verified) && State=3) -> INSTALLED;
* => Install -> INSTALLED (Res=2 SW successfully installed) && State=4) -> Start
*
* */
@Test
public void testSoftwareUpdateByObject9WithObject19_Ok() throws Exception {
String clientEndpoint = this.CLIENT_ENDPOINT_OTA9_19;
Lwm2mDeviceProfileTransportConfiguration transportConfiguration = getTransportConfiguration19(OBSERVE_ATTRIBUTES_WITH_PARAMS_OTA9_19, getBootstrapServerCredentialsNoSec(NONE));
DeviceProfile deviceProfile = createLwm2mDeviceProfile("profileFor" + clientEndpoint, transportConfiguration);
LwM2MDeviceCredentials deviceCredentials = getDeviceCredentialsNoSec(createNoSecClientCredentials(clientEndpoint));
final Device device = createLwm2mDevice(deviceCredentials, clientEndpoint, deviceProfile.getId());
createNewClient(SECURITY_NO_SEC, null, false, clientEndpoint, device.getId().getId().toString());
awaitObserveReadAll(5, device.getId().getId().toString());
OtaPackageInfo otaPackageInfo = createSoftware(deviceProfile.getId(), "v1.0.19");
device.setSoftwareId(otaPackageInfo.getId());
final Device savedDevice = doPost("/api/device", device, Device.class); //sync call
assertThat(savedDevice).as("saved device").isNotNull();
assertThat(getDeviceFromAPI(device.getId().getId())).as("fetched device").isEqualTo(savedDevice);
expectedStatuses = List.of(
QUEUED, INITIATED, DOWNLOADING, DOWNLOADING, DOWNLOADING, DOWNLOADED, VERIFIED, UPDATED);
List<TsKvEntry> ts = await("await on timeseries")
.atMost(TIMEOUT, TimeUnit.SECONDS)
.until(() -> getFwSwStateTelemetryFromAPI(device.getId().getId(), "sw_state"), this::predicateForStatuses);
String ver_Id_19 = lwM2MTestClient.getLeshanClient().getObjectTree().getModel().getObjectModel(BINARY_APP_DATA_CONTAINER).version;
String resourceIdVer = "/" + BINARY_APP_DATA_CONTAINER + "_" + ver_Id_19 + "/" + SW_INSTANCE_ID + "/" + RESOURCE_ID_0;
resultReadOtaParams_19(resourceIdVer, otaPackageInfo);
log.warn("Object9: Got the ts: {}", ts);
}
}

5
application/src/test/java/org/thingsboard/server/utils/CalculatedFieldUtilsTest.java

@ -119,7 +119,7 @@ class CalculatedFieldUtilsTest {
}
@Test
void toProtoAndFromProto_shouldCreatePropagationStateWithEmptyPropagationArgument() {
void toProtoAndFromProto_shouldCreatePropagationStateWithNotEmptyPropagationArgument() {
// given
CalculatedFieldEntityCtxId stateId = mock(CalculatedFieldEntityCtxId.class);
given(stateId.tenantId()).willReturn(TENANT_ID);
@ -158,8 +158,7 @@ class CalculatedFieldUtilsTest {
assertThat(propagationState.getEntityId()).isEqualTo(DEVICE_ID);
assertThat(propagationState.getArguments()).isNotNull();
assertThat(propagationState.getArguments().get(PROPAGATION_CONFIG_ARGUMENT)).isNotNull();
assertThat(propagationState.getArguments().get(PROPAGATION_CONFIG_ARGUMENT).isEmpty()).isTrue();
assertThat(propagationState.getArguments().get(PROPAGATION_CONFIG_ARGUMENT)).isEqualTo(propagationArgumentEntry);
assertThat(propagationState.getArguments().get("state")).isNotNull().isEqualTo(singleValueArgumentEntry);
assertThat(propagationState.getRequiredArguments()).isNull();
assertThat(propagationState.getReadinessStatus()).isNull();

5
dao/src/main/java/org/thingsboard/server/dao/exception/DataValidationException.java → common/data/src/main/java/org/thingsboard/server/exception/DataValidationException.java

@ -13,12 +13,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.exception;
package org.thingsboard.server.exception;
public class DataValidationException extends RuntimeException {
private static final long serialVersionUID = 7659985660312721830L;
public DataValidationException(String message) {
super(message);
}
@ -26,4 +24,5 @@ public class DataValidationException extends RuntimeException {
public DataValidationException(String message, Throwable cause) {
super(message, cause);
}
}

4
dao/src/main/java/org/thingsboard/server/dao/exception/EntitiesLimitExceededException.java → common/data/src/main/java/org/thingsboard/server/exception/EntitiesLimitExceededException.java

@ -13,14 +13,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.exception;
package org.thingsboard.server.exception;
import lombok.Getter;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.TenantId;
public class EntitiesLimitExceededException extends DataValidationException {
private static final long serialVersionUID = -9211462514373279196L;
@Getter
private final TenantId tenantId;
@ -36,4 +35,5 @@ public class EntitiesLimitExceededException extends DataValidationException {
this.entityType = entityType;
this.limit = limit;
}
}

51
common/data/src/main/java/org/thingsboard/server/exception/ThingsboardRuntimeException.java

@ -0,0 +1,51 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.exception;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
public class ThingsboardRuntimeException extends RuntimeException {
private ThingsboardErrorCode errorCode;
public ThingsboardRuntimeException() {
super();
}
public ThingsboardRuntimeException(ThingsboardErrorCode errorCode) {
this.errorCode = errorCode;
}
public ThingsboardRuntimeException(String message, ThingsboardErrorCode errorCode) {
super(message);
this.errorCode = errorCode;
}
public ThingsboardRuntimeException(String message, Throwable cause, ThingsboardErrorCode errorCode) {
super(message, cause);
this.errorCode = errorCode;
}
public ThingsboardRuntimeException(Throwable cause, ThingsboardErrorCode errorCode) {
super(cause);
this.errorCode = errorCode;
}
public ThingsboardErrorCode getErrorCode() {
return errorCode;
}
}

1
common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeRpcClient.java

@ -41,4 +41,5 @@ public interface EdgeRpcClient {
void sendDownlinkResponseMsg(DownlinkResponseMsg downlinkResponseMsg);
int getServerMaxInboundMessageSize();
}

1
common/proto/src/main/proto/queue.proto

@ -935,6 +935,7 @@ message CalculatedFieldStateProto {
int64 lastArgsUpdateTs = 7;
int64 lastMetricsEvalTs = 8;
repeated ArgumentIntervalProto aggregationArguments = 9;
repeated EntityIdProto propagationEntityIds = 10;
}
//Used to report session state to tb-Service and persist this state in the cache on the tb-Service level.

3
common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttSslHandlerProvider.java

@ -20,7 +20,6 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
@ -48,7 +47,7 @@ import java.util.concurrent.TimeUnit;
@Slf4j
@Component("MqttSslHandlerProvider")
@ConditionalOnProperty(prefix = "transport.mqtt.ssl", value = "enabled", havingValue = "true", matchIfMissing = false)
@TbMqttSslTransportComponent
public class MqttSslHandlerProvider {
@Value("${transport.mqtt.ssl.protocol}")

31
common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/TbMqttSslTransportComponent.java

@ -0,0 +1,31 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.transport.mqtt;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
/**
* Same as @TbMqttTransportComponent with additional condition by `transport.mqtt.ssl.enabled == true`
*/
@Inherited
@Retention(RetentionPolicy.RUNTIME)
@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.mqtt.enabled:true}'=='true' && '${transport.mqtt.ssl.enabled:false}'=='true')")
public @interface TbMqttSslTransportComponent {}

6
common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/TbMqttTransportComponent.java

@ -21,7 +21,11 @@ import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
/**
* See also @TbMqttSslTransportComponent
*/
@Inherited
@Retention(RetentionPolicy.RUNTIME)
@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.mqtt.enabled}'=='true')")
@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.mqtt.enabled:true}'=='true')")
public @interface TbMqttTransportComponent {}

2
dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java

@ -64,7 +64,7 @@ import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.ConstraintValidator;
import org.thingsboard.server.dao.tenant.TenantService;

2
dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java

@ -37,7 +37,7 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.dao.entity.CachedVersionedEntityService;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.service.DataValidator;
import org.thingsboard.server.dao.service.PaginatedRemover;

2
dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java

@ -53,7 +53,7 @@ import org.thingsboard.server.dao.entity.EntityCountService;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.service.DataValidator;
import org.thingsboard.server.dao.service.PaginatedRemover;
import org.thingsboard.server.dao.sql.JpaExecutorService;

2
dao/src/main/java/org/thingsboard/server/dao/component/BaseComponentDescriptorService.java

@ -30,7 +30,7 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.plugin.ComponentDescriptor;
import org.thingsboard.server.common.data.plugin.ComponentScope;
import org.thingsboard.server.common.data.plugin.ComponentType;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.service.DataValidator;
import org.thingsboard.server.dao.service.Validator;

2
dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java

@ -45,7 +45,7 @@ import org.thingsboard.server.dao.entity.AbstractCachedEntityService;
import org.thingsboard.server.dao.entity.EntityCountService;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.service.DataValidator;
import org.thingsboard.server.dao.service.PaginatedRemover;

2
dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java

@ -50,7 +50,7 @@ import org.thingsboard.server.dao.entity.EntityCountService;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.dao.service.DataValidator;

2
dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsServiceImpl.java

@ -43,9 +43,9 @@ import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.msg.EncryptionUtil;
import org.thingsboard.server.dao.entity.AbstractCachedEntityService;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException;
import org.thingsboard.server.dao.service.validator.DeviceCredentialsDataValidator;
import org.thingsboard.server.exception.DataValidationException;
import java.util.Objects;

2
dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java

@ -46,11 +46,11 @@ import org.thingsboard.server.common.msg.EncryptionUtil;
import org.thingsboard.server.dao.entity.CachedVersionedEntityService;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.service.PaginatedRemover;
import org.thingsboard.server.dao.service.Validator;
import org.thingsboard.server.dao.service.validator.DeviceProfileDataValidator;
import org.thingsboard.server.exception.DataValidationException;
import java.io.ByteArrayInputStream;
import java.security.cert.Certificate;

2
dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java

@ -80,7 +80,7 @@ import org.thingsboard.server.dao.event.EventService;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.service.PaginatedRemover;
import org.thingsboard.server.dao.service.validator.DeviceDataValidator;

2
dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java

@ -66,7 +66,7 @@ import org.thingsboard.server.dao.entity.EntityCountService;
import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.exception.DataValidationException;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.service.DataValidator;

3
dao/src/main/java/org/thingsboard/server/dao/edge/PostgresEdgeEventService.java

@ -68,7 +68,8 @@ public class PostgresEdgeEventService extends BaseEdgeEventService {
Futures.addCallback(saveFuture, new FutureCallback<>() {
@Override
public void onSuccess(Void result) {
statsCounterService.ifPresent(statsCounterService -> statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edgeEvent.getTenantId(), edgeEvent.getEdgeId(), 1));
statsCounterService.ifPresent(statsCounterService ->
statsCounterService.recordEvent(EdgeStatsKey.DOWNLINK_MSGS_ADDED, edgeEvent.getTenantId(), edgeEvent.getEdgeId(), 1));
eventPublisher.publishEvent(SaveEntityEvent.builder()
.tenantId(edgeEvent.getTenantId())
.entityId(edgeEvent.getEdgeId())

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save