Browse Source

Merge branch 'master' of https://github.com/thingsboard/thingsboard into feature/kafka-settings

# Conflicts:
#	application/src/main/resources/thingsboard.yml
pull/4184/head
YevhenBondarenko 5 years ago
parent
commit
9f863116a0
  1. 11
      application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql
  2. 40
      application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java
  3. 9
      application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java
  4. 32
      application/src/main/java/org/thingsboard/server/controller/BaseController.java
  5. 6
      application/src/main/java/org/thingsboard/server/controller/CustomerController.java
  6. 7
      application/src/main/java/org/thingsboard/server/controller/TenantController.java
  7. 9
      application/src/main/java/org/thingsboard/server/controller/UserController.java
  8. 4
      application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
  9. 5
      application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java
  10. 1
      application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java
  11. 27
      application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java
  12. 6
      application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java
  13. 1
      application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java
  14. 2
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java
  15. 10
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
  16. 10
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java
  17. 39
      application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java
  18. 3
      application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java
  19. 5
      application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java
  20. 5
      application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java
  21. 49
      application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java
  22. 6
      application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java
  23. 2
      application/src/main/resources/logback.xml
  24. 20
      application/src/main/resources/thingsboard.yml
  25. 1
      common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java
  26. 2
      common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmCondition.java
  27. 30
      common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilter.java
  28. 26
      common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilterKey.java
  29. 23
      common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionKeyType.java
  30. 12
      common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java
  31. 2
      common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java
  32. 4
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java
  33. 4
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java
  34. 4
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java
  35. 37
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEvent.java
  36. 52
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java
  37. 37
      common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatisticConfig.java
  38. 184
      common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java
  39. 16
      common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java
  40. 11
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java
  41. 9
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java
  42. 7
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java
  43. 6
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java
  44. 18
      common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java
  45. 41
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java
  46. 6
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java
  47. 53
      common/transport/transport-api/src/test/java/JsonConverterTest.java
  48. 39
      dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCacheKey.java
  49. 39
      dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java
  50. 63
      dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesCacheWrapper.java
  51. 47
      dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java
  52. 193
      dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java
  53. 6
      rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java
  54. 2
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java
  55. 3
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java
  56. 19
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/CalculateDeltaNode.java
  57. 4
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/CalculateDeltaNodeConfiguration.java
  58. 3
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/azure/AzureIotHubSasCredentials.java
  59. 84
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java
  60. 7
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java
  61. 85
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DataSnapshot.java
  62. 67
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java
  63. 33
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java
  64. 8
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/SnapshotUpdate.java
  65. 344
      rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java
  66. 1
      tools/src/main/shell/keygen.properties
  67. 9
      tools/src/main/shell/server.keygen.sh
  68. 1
      transport/http/src/main/resources/tb-http-transport.yml
  69. 1
      ui-ngx/src/app/core/http/entity.service.ts
  70. 2
      ui-ngx/src/app/modules/home/components/alarm/alarm-table-config.ts
  71. 8
      ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts
  72. 6
      ui-ngx/src/app/modules/home/components/entity/entity-details-panel.component.ts
  73. 5
      ui-ngx/src/app/modules/home/components/filter/boolean-filter-predicate.component.html
  74. 2
      ui-ngx/src/app/modules/home/components/filter/boolean-filter-predicate.component.ts
  75. 1
      ui-ngx/src/app/modules/home/components/filter/complex-filter-predicate-dialog.component.html
  76. 1
      ui-ngx/src/app/modules/home/components/filter/complex-filter-predicate-dialog.component.ts
  77. 5
      ui-ngx/src/app/modules/home/components/filter/complex-filter-predicate.component.ts
  78. 5
      ui-ngx/src/app/modules/home/components/filter/filter-predicate-list.component.html
  79. 5
      ui-ngx/src/app/modules/home/components/filter/filter-predicate-list.component.ts
  80. 16
      ui-ngx/src/app/modules/home/components/filter/filter-predicate-value.component.html
  81. 27
      ui-ngx/src/app/modules/home/components/filter/filter-predicate-value.component.ts
  82. 4
      ui-ngx/src/app/modules/home/components/filter/filter-predicate.component.html
  83. 2
      ui-ngx/src/app/modules/home/components/filter/filter-predicate.component.ts
  84. 32
      ui-ngx/src/app/modules/home/components/filter/key-filter-dialog.component.html
  85. 33
      ui-ngx/src/app/modules/home/components/filter/key-filter-dialog.component.ts
  86. 1
      ui-ngx/src/app/modules/home/components/filter/key-filter-list.component.ts
  87. 5
      ui-ngx/src/app/modules/home/components/filter/numeric-filter-predicate.component.html
  88. 2
      ui-ngx/src/app/modules/home/components/filter/numeric-filter-predicate.component.ts
  89. 5
      ui-ngx/src/app/modules/home/components/filter/string-filter-predicate.component.html
  90. 2
      ui-ngx/src/app/modules/home/components/filter/string-filter-predicate.component.ts
  91. 4
      ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts
  92. 2
      ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html
  93. 4
      ui-ngx/src/app/shared/components/file-input.component.ts
  94. 12
      ui-ngx/src/app/shared/models/query/query.models.ts
  95. 7
      ui-ngx/src/assets/locale/locale.constant-en_US.json

11
application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql

@ -84,11 +84,12 @@ BEGIN
END IF;
END IF;
END IF;
END IF;
IF partition_to_delete IS NOT NULL THEN
RAISE NOTICE 'Partition to delete by max ttl: %', partition_to_delete;
EXECUTE format('DROP TABLE %I', partition_to_delete);
deleted := deleted + 1;
IF partition_to_delete IS NOT NULL THEN
RAISE NOTICE 'Partition to delete by max ttl: %', partition_to_delete;
EXECUTE format('DROP TABLE IF EXISTS %I', partition_to_delete);
partition_to_delete := NULL;
deleted := deleted + 1;
END IF;
END IF;
END LOOP;
END IF;

40
application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java

@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.alarm.Alarm;
import org.thingsboard.server.common.data.asset.Asset;
@ -278,7 +279,21 @@ class DefaultTbContext implements TbContext {
}
public TbMsg deviceCreatedMsg(Device device, RuleNodeId ruleNodeId) {
return entityActionMsg(device, device.getId(), ruleNodeId, DataConstants.ENTITY_CREATED);
RuleChainId ruleChainId = null;
String queueName = ServiceQueue.MAIN;
if (device.getDeviceProfileId() != null) {
DeviceProfile deviceProfile = mainCtx.getDeviceProfileCache().find(device.getDeviceProfileId());
if (deviceProfile == null) {
log.warn("[{}] Device profile is null!", device.getDeviceProfileId());
ruleChainId = null;
queueName = ServiceQueue.MAIN;
} else {
ruleChainId = deviceProfile.getDefaultRuleChainId();
String defaultQueueName = deviceProfile.getDefaultQueueName();
queueName = defaultQueueName != null ? defaultQueueName : ServiceQueue.MAIN;
}
}
return entityActionMsg(device, device.getId(), ruleNodeId, DataConstants.ENTITY_CREATED, queueName, ruleChainId);
}
public TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId) {
@ -286,12 +301,31 @@ class DefaultTbContext implements TbContext {
}
public TbMsg alarmActionMsg(Alarm alarm, RuleNodeId ruleNodeId, String action) {
return entityActionMsg(alarm, alarm.getId(), ruleNodeId, action);
RuleChainId ruleChainId = null;
String queueName = ServiceQueue.MAIN;
if (EntityType.DEVICE.equals(alarm.getOriginator().getEntityType())) {
DeviceId deviceId = new DeviceId(alarm.getOriginator().getId());
DeviceProfile deviceProfile = mainCtx.getDeviceProfileCache().get(getTenantId(), deviceId);
if (deviceProfile == null) {
log.warn("[{}] Device profile is null!", deviceId);
ruleChainId = null;
queueName = ServiceQueue.MAIN;
} else {
ruleChainId = deviceProfile.getDefaultRuleChainId();
String defaultQueueName = deviceProfile.getDefaultQueueName();
queueName = defaultQueueName != null ? defaultQueueName : ServiceQueue.MAIN;
}
}
return entityActionMsg(alarm, alarm.getId(), ruleNodeId, action, queueName, ruleChainId);
}
public <E, I extends EntityId> TbMsg entityActionMsg(E entity, I id, RuleNodeId ruleNodeId, String action) {
return entityActionMsg(entity, id, ruleNodeId, action, ServiceQueue.MAIN, null);
}
public <E, I extends EntityId> TbMsg entityActionMsg(E entity, I id, RuleNodeId ruleNodeId, String action, String queueName, RuleChainId ruleChainId) {
try {
return TbMsg.newMsg(action, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity)));
return TbMsg.newMsg(queueName, action, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity)), ruleChainId, null);
} catch (JsonProcessingException | IllegalArgumentException e) {
throw new RuntimeException("Failed to process " + id.getEntityType().name().toLowerCase() + " " + action + " msg: " + e);
}

9
application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java

@ -34,6 +34,7 @@ import org.thingsboard.server.actors.app.AppInitMsg;
import org.thingsboard.server.actors.stats.StatsActor;
import org.thingsboard.server.common.msg.queue.PartitionChangeMsg;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
@ -43,7 +44,7 @@ import java.util.concurrent.ScheduledExecutorService;
@Service
@Slf4j
public class DefaultActorService implements ActorService {
public class DefaultActorService extends TbApplicationEventListener<PartitionChangeEvent> implements ActorService {
public static final String APP_DISPATCHER_NAME = "app-dispatcher";
public static final String TENANT_DISPATCHER_NAME = "tenant-dispatcher";
@ -120,10 +121,10 @@ public class DefaultActorService implements ActorService {
appActor.tellWithHighPriority(new AppInitMsg());
}
@EventListener(PartitionChangeEvent.class)
public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
@Override
protected void onTbApplicationEvent(PartitionChangeEvent event) {
log.info("Received partition change event.");
this.appActor.tellWithHighPriority(new PartitionChangeMsg(partitionChangeEvent.getServiceQueueKey(), partitionChangeEvent.getPartitions()));
this.appActor.tellWithHighPriority(new PartitionChangeMsg(event.getServiceQueueKey(), event.getPartitions()));
}
@PreDestroy

32
application/src/main/java/org/thingsboard/server/controller/BaseController.java

@ -27,7 +27,22 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.thingsboard.server.common.data.*;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.Dashboard;
import org.thingsboard.server.common.data.DashboardInfo;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceInfo;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.EntityView;
import org.thingsboard.server.common.data.EntityViewInfo;
import org.thingsboard.server.common.data.HasName;
import org.thingsboard.server.common.data.HasTenantId;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantInfo;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.alarm.Alarm;
import org.thingsboard.server.common.data.alarm.AlarmInfo;
import org.thingsboard.server.common.data.asset.Asset;
@ -84,6 +99,7 @@ import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService;
import org.thingsboard.server.dao.oauth2.OAuth2Service;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.dao.tenant.TenantProfileService;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.user.UserService;
@ -95,7 +111,6 @@ import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.component.ComponentDiscoveryService;
import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.service.queue.TbClusterService;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.AccessControlService;
@ -123,6 +138,9 @@ public abstract class BaseController {
public static final String INCORRECT_TENANT_ID = "Incorrect tenantId ";
public static final String YOU_DON_T_HAVE_PERMISSION_TO_PERFORM_THIS_OPERATION = "You don't have permission to perform this operation!";
protected static final String DEFAULT_DASHBOARD = "defaultDashboardId";
protected static final String HOME_DASHBOARD = "homeDashboardId";
private static final ObjectMapper json = new ObjectMapper();
@Autowired
@ -858,4 +876,14 @@ public abstract class BaseController {
}
}
}
protected void processDashboardIdFromAdditionalInfo(ObjectNode additionalInfo, String requiredFields) throws ThingsboardException {
String dashboardId = additionalInfo.has(requiredFields) ? additionalInfo.get(requiredFields).asText() : null;
if(dashboardId != null && !dashboardId.equals("null")) {
if(dashboardService.findDashboardById(getTenantId(), new DashboardId(UUID.fromString(dashboardId))) == null) {
additionalInfo.remove(requiredFields);
}
}
}
}

6
application/src/main/java/org/thingsboard/server/controller/CustomerController.java

@ -55,7 +55,11 @@ public class CustomerController extends BaseController {
checkParameter(CUSTOMER_ID, strCustomerId);
try {
CustomerId customerId = new CustomerId(toUUID(strCustomerId));
return checkCustomerId(customerId, Operation.READ);
Customer customer = checkCustomerId(customerId, Operation.READ);
if(!customer.getAdditionalInfo().isNull()) {
processDashboardIdFromAdditionalInfo((ObjectNode) customer.getAdditionalInfo(), HOME_DASHBOARD);
}
return customer;
} catch (Exception e) {
throw handleException(e);
}

7
application/src/main/java/org/thingsboard/server/controller/TenantController.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.controller;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
@ -59,7 +60,11 @@ public class TenantController extends BaseController {
checkParameter("tenantId", strTenantId);
try {
TenantId tenantId = new TenantId(toUUID(strTenantId));
return checkTenantId(tenantId, Operation.READ);
Tenant tenant = checkTenantId(tenantId, Operation.READ);
if(!tenant.getAdditionalInfo().isNull()) {
processDashboardIdFromAdditionalInfo((ObjectNode) tenant.getAdditionalInfo(), HOME_DASHBOARD);
}
return tenant;
} catch (Exception e) {
throw handleException(e);
}

9
application/src/main/java/org/thingsboard/server/controller/UserController.java

@ -53,7 +53,6 @@ import org.thingsboard.server.service.security.model.token.JwtTokenFactory;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;
import org.thingsboard.server.service.security.system.SystemSecurityService;
import org.thingsboard.server.utils.MiscUtils;
import javax.servlet.http.HttpServletRequest;
@ -90,7 +89,12 @@ public class UserController extends BaseController {
checkParameter(USER_ID, strUserId);
try {
UserId userId = new UserId(toUUID(strUserId));
return checkUserId(userId, Operation.READ);
User user = checkUserId(userId, Operation.READ);
if(!user.getAdditionalInfo().isNull()) {
processDashboardIdFromAdditionalInfo((ObjectNode) user.getAdditionalInfo(), DEFAULT_DASHBOARD);
processDashboardIdFromAdditionalInfo((ObjectNode) user.getAdditionalInfo(), HOME_DASHBOARD);
}
return user;
} catch (Exception e) {
throw handleException(e);
}
@ -329,4 +333,5 @@ public class UserController extends BaseController {
throw handleException(e);
}
}
}

4
application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java

@ -187,6 +187,10 @@ public class ThingsboardInstallService {
databaseEntitiesUpgradeService.upgradeDatabase("3.2.0");
case "3.2.1":
log.info("Upgrading ThingsBoard from version 3.2.1 to 3.3.0 ...");
if (databaseTsUpgradeService != null) {
databaseTsUpgradeService.upgradeDatabase("3.2.1");
}
log.info("Updating system data...");
systemDataLoaderService.updateSystemWidgets();
break;

5
application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java

@ -54,6 +54,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
import org.thingsboard.server.queue.scheduler.SchedulerComponent;
import org.thingsboard.server.service.queue.TbClusterService;
import org.thingsboard.server.service.telemetry.InternalTelemetryService;
@ -78,7 +79,7 @@ import java.util.stream.Collectors;
@Slf4j
@Service
public class DefaultTbApiUsageStateService implements TbApiUsageStateService {
public class DefaultTbApiUsageStateService extends TbApplicationEventListener<PartitionChangeEvent> implements TbApiUsageStateService {
public static final String HOURLY = "Hourly";
public static final FutureCallback<Integer> VOID_CALLBACK = new FutureCallback<Integer>() {
@ -188,7 +189,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService {
}
@Override
public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
protected void onTbApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
if (partitionChangeEvent.getServiceType().equals(ServiceType.TB_CORE)) {
myTenantStates.entrySet().removeIf(entry -> !partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition());
otherTenantStates.entrySet().removeIf(entry -> partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition());

1
application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java

@ -50,6 +50,7 @@ public class CassandraTsDatabaseUpgradeService extends AbstractCassandraDatabase
break;
case "2.5.0":
case "3.1.1":
case "3.2.1":
break;
default:
throw new RuntimeException("Unable to upgrade Cassandra database, unsupported fromVersion: " + fromVersion);

27
application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java

@ -36,6 +36,9 @@ import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.alarm.AlarmSeverity;
import org.thingsboard.server.common.data.device.profile.AlarmCondition;
import org.thingsboard.server.common.data.device.profile.AlarmConditionFilter;
import org.thingsboard.server.common.data.device.profile.AlarmConditionFilterKey;
import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType;
import org.thingsboard.server.common.data.device.profile.AlarmRule;
import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration;
import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration;
@ -290,16 +293,16 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
AlarmCondition temperatureCondition = new AlarmCondition();
temperatureCondition.setSpec(new SimpleAlarmConditionSpec());
KeyFilter temperatureAlarmFlagAttributeFilter = new KeyFilter();
temperatureAlarmFlagAttributeFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperatureAlarmFlag"));
AlarmConditionFilter temperatureAlarmFlagAttributeFilter = new AlarmConditionFilter();
temperatureAlarmFlagAttributeFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.ATTRIBUTE, "temperatureAlarmFlag"));
temperatureAlarmFlagAttributeFilter.setValueType(EntityKeyValueType.BOOLEAN);
BooleanFilterPredicate temperatureAlarmFlagAttributePredicate = new BooleanFilterPredicate();
temperatureAlarmFlagAttributePredicate.setOperation(BooleanFilterPredicate.BooleanOperation.EQUAL);
temperatureAlarmFlagAttributePredicate.setValue(new FilterPredicateValue<>(Boolean.TRUE));
temperatureAlarmFlagAttributeFilter.setPredicate(temperatureAlarmFlagAttributePredicate);
KeyFilter temperatureTimeseriesFilter = new KeyFilter();
temperatureTimeseriesFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "temperature"));
AlarmConditionFilter temperatureTimeseriesFilter = new AlarmConditionFilter();
temperatureTimeseriesFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature"));
temperatureTimeseriesFilter.setValueType(EntityKeyValueType.NUMERIC);
NumericFilterPredicate temperatureTimeseriesFilterPredicate = new NumericFilterPredicate();
temperatureTimeseriesFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER);
@ -317,8 +320,8 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
AlarmCondition clearTemperatureCondition = new AlarmCondition();
clearTemperatureCondition.setSpec(new SimpleAlarmConditionSpec());
KeyFilter clearTemperatureTimeseriesFilter = new KeyFilter();
clearTemperatureTimeseriesFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "temperature"));
AlarmConditionFilter clearTemperatureTimeseriesFilter = new AlarmConditionFilter();
clearTemperatureTimeseriesFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature"));
clearTemperatureTimeseriesFilter.setValueType(EntityKeyValueType.NUMERIC);
NumericFilterPredicate clearTemperatureTimeseriesFilterPredicate = new NumericFilterPredicate();
clearTemperatureTimeseriesFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS_OR_EQUAL);
@ -340,16 +343,16 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
AlarmCondition humidityCondition = new AlarmCondition();
humidityCondition.setSpec(new SimpleAlarmConditionSpec());
KeyFilter humidityAlarmFlagAttributeFilter = new KeyFilter();
humidityAlarmFlagAttributeFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "humidityAlarmFlag"));
AlarmConditionFilter humidityAlarmFlagAttributeFilter = new AlarmConditionFilter();
humidityAlarmFlagAttributeFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.ATTRIBUTE, "humidityAlarmFlag"));
humidityAlarmFlagAttributeFilter.setValueType(EntityKeyValueType.BOOLEAN);
BooleanFilterPredicate humidityAlarmFlagAttributePredicate = new BooleanFilterPredicate();
humidityAlarmFlagAttributePredicate.setOperation(BooleanFilterPredicate.BooleanOperation.EQUAL);
humidityAlarmFlagAttributePredicate.setValue(new FilterPredicateValue<>(Boolean.TRUE));
humidityAlarmFlagAttributeFilter.setPredicate(humidityAlarmFlagAttributePredicate);
KeyFilter humidityTimeseriesFilter = new KeyFilter();
humidityTimeseriesFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "humidity"));
AlarmConditionFilter humidityTimeseriesFilter = new AlarmConditionFilter();
humidityTimeseriesFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "humidity"));
humidityTimeseriesFilter.setValueType(EntityKeyValueType.NUMERIC);
NumericFilterPredicate humidityTimeseriesFilterPredicate = new NumericFilterPredicate();
humidityTimeseriesFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS);
@ -368,8 +371,8 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
AlarmCondition clearHumidityCondition = new AlarmCondition();
clearHumidityCondition.setSpec(new SimpleAlarmConditionSpec());
KeyFilter clearHumidityTimeseriesFilter = new KeyFilter();
clearHumidityTimeseriesFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "humidity"));
AlarmConditionFilter clearHumidityTimeseriesFilter = new AlarmConditionFilter();
clearHumidityTimeseriesFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "humidity"));
clearHumidityTimeseriesFilter.setValueType(EntityKeyValueType.NUMERIC);
NumericFilterPredicate clearHumidityTimeseriesFilterPredicate = new NumericFilterPredicate();
clearHumidityTimeseriesFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER_OR_EQUAL);

6
application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java

@ -196,11 +196,17 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
}
break;
case "3.1.1":
case "3.2.1":
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
log.info("Load TTL functions ...");
loadSql(conn, LOAD_TTL_FUNCTIONS_SQL);
log.info("Load Drop Partitions functions ...");
loadSql(conn, LOAD_DROP_PARTITIONS_FUNCTIONS_SQL);
executeQuery(conn, "DROP PROCEDURE IF EXISTS cleanup_timeseries_by_ttl(character varying, bigint, bigint);");
executeQuery(conn, "DROP FUNCTION IF EXISTS delete_asset_records_from_ts_kv(character varying, character varying, bigint);");
executeQuery(conn, "DROP FUNCTION IF EXISTS delete_device_records_from_ts_kv(character varying, character varying, bigint);");
executeQuery(conn, "DROP FUNCTION IF EXISTS delete_customer_records_from_ts_kv(character varying, character varying, bigint);");
}
break;
default:

1
application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java

@ -178,6 +178,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
}
break;
case "3.1.1":
case "3.2.1":
break;
default:
throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);

2
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java

@ -145,7 +145,7 @@ public class DefaultTbClusterService implements TbClusterService {
tbMsg = transformMsg(tbMsg, deviceProfileCache.get(tenantId, new DeviceProfileId(entityId.getId())));
}
}
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId);
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tbMsg.getQueueName(), tenantId, entityId);
log.trace("PUSHING msg: {} to:{}", tbMsg, tpi);
ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())

10
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@ -151,12 +151,12 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
}
@Override
public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
if (partitionChangeEvent.getServiceType().equals(getServiceType())) {
log.info("Subscribing to partitions: {}", partitionChangeEvent.getPartitions());
this.mainConsumer.subscribe(partitionChangeEvent.getPartitions());
protected void onTbApplicationEvent(PartitionChangeEvent event) {
if (event.getServiceType().equals(getServiceType())) {
log.info("Subscribing to partitions: {}", event.getPartitions());
this.mainConsumer.subscribe(event.getPartitions());
this.usageStatsConsumer.subscribe(
partitionChangeEvent
event
.getPartitions()
.stream()
.map(tpi -> tpi.newByTopic(usageStatsConsumer.getTopic()))

10
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java

@ -140,11 +140,11 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
}
@Override
public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
if (partitionChangeEvent.getServiceType().equals(getServiceType())) {
ServiceQueue serviceQueue = partitionChangeEvent.getServiceQueueKey().getServiceQueue();
log.info("[{}] Subscribing to partitions: {}", serviceQueue.getQueue(), partitionChangeEvent.getPartitions());
consumers.get(serviceQueue.getQueue()).subscribe(partitionChangeEvent.getPartitions());
protected void onTbApplicationEvent(PartitionChangeEvent event) {
if (event.getServiceType().equals(getServiceType())) {
ServiceQueue serviceQueue = event.getServiceQueueKey().getServiceQueue();
log.info("[{}] Subscribing to partitions: {}", serviceQueue.getQueue(), event.getPartitions());
consumers.get(serviceQueue.getQueue()).subscribe(event.getPartitions());
}
}

39
application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java

@ -55,31 +55,22 @@ public class TbCoreConsumerStats {
public TbCoreConsumerStats(StatsFactory statsFactory) {
String statsKey = StatsType.CORE.getName();
this.totalCounter = statsFactory.createStatsCounter(statsKey, TOTAL_MSGS);
this.sessionEventCounter = statsFactory.createStatsCounter(statsKey, SESSION_EVENTS);
this.getAttributesCounter = statsFactory.createStatsCounter(statsKey, GET_ATTRIBUTE);
this.subscribeToAttributesCounter = statsFactory.createStatsCounter(statsKey, ATTRIBUTE_SUBSCRIBES);
this.subscribeToRPCCounter = statsFactory.createStatsCounter(statsKey, RPC_SUBSCRIBES);
this.toDeviceRPCCallResponseCounter = statsFactory.createStatsCounter(statsKey, TO_DEVICE_RPC_CALL_RESPONSES);
this.subscriptionInfoCounter = statsFactory.createStatsCounter(statsKey, SUBSCRIPTION_INFO);
this.claimDeviceCounter = statsFactory.createStatsCounter(statsKey, DEVICE_CLAIMS);
this.deviceStateCounter = statsFactory.createStatsCounter(statsKey, DEVICE_STATES);
this.subscriptionMsgCounter = statsFactory.createStatsCounter(statsKey, SUBSCRIPTION_MSGS);
this.toCoreNotificationsCounter = statsFactory.createStatsCounter(statsKey, TO_CORE_NOTIFICATIONS);
counters.add(totalCounter);
counters.add(sessionEventCounter);
counters.add(getAttributesCounter);
counters.add(subscribeToAttributesCounter);
counters.add(subscribeToRPCCounter);
counters.add(toDeviceRPCCallResponseCounter);
counters.add(subscriptionInfoCounter);
counters.add(claimDeviceCounter);
this.totalCounter = register(statsFactory.createStatsCounter(statsKey, TOTAL_MSGS));
this.sessionEventCounter = register(statsFactory.createStatsCounter(statsKey, SESSION_EVENTS));
this.getAttributesCounter = register(statsFactory.createStatsCounter(statsKey, GET_ATTRIBUTE));
this.subscribeToAttributesCounter = register(statsFactory.createStatsCounter(statsKey, ATTRIBUTE_SUBSCRIBES));
this.subscribeToRPCCounter = register(statsFactory.createStatsCounter(statsKey, RPC_SUBSCRIBES));
this.toDeviceRPCCallResponseCounter = register(statsFactory.createStatsCounter(statsKey, TO_DEVICE_RPC_CALL_RESPONSES));
this.subscriptionInfoCounter = register(statsFactory.createStatsCounter(statsKey, SUBSCRIPTION_INFO));
this.claimDeviceCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_CLAIMS));
this.deviceStateCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_STATES));
this.subscriptionMsgCounter = register(statsFactory.createStatsCounter(statsKey, SUBSCRIPTION_MSGS));
this.toCoreNotificationsCounter = register(statsFactory.createStatsCounter(statsKey, TO_CORE_NOTIFICATIONS));
}
counters.add(deviceStateCounter);
counters.add(subscriptionMsgCounter);
counters.add(toCoreNotificationsCounter);
private StatsCounter register(StatsCounter counter){
counters.add(counter);
return counter;
}
public void log(TransportProtos.TransportToDeviceActorMsg msg) {

3
application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java

@ -36,6 +36,7 @@ import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
@ -56,7 +57,7 @@ import java.util.function.Function;
import java.util.stream.Collectors;
@Slf4j
public abstract class AbstractConsumerService<N extends com.google.protobuf.GeneratedMessageV3> implements ApplicationListener<PartitionChangeEvent> {
public abstract class AbstractConsumerService<N extends com.google.protobuf.GeneratedMessageV3> extends TbApplicationEventListener<PartitionChangeEvent> {
protected volatile ExecutorService consumersExecutor;
protected volatile ExecutorService notificationsConsumerExecutor;

5
application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java

@ -56,6 +56,7 @@ import org.thingsboard.server.dao.util.mapping.JacksonUtil;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.queue.TbClusterService;
import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService;
@ -90,7 +91,7 @@ import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE;
@Service
@TbCoreComponent
@Slf4j
public class DefaultDeviceStateService implements DeviceStateService {
public class DefaultDeviceStateService extends TbApplicationEventListener<PartitionChangeEvent> implements DeviceStateService {
public static final String ACTIVITY_STATE = "active";
public static final String LAST_CONNECT_TIME = "lastConnectTime";
@ -294,7 +295,7 @@ public class DefaultDeviceStateService implements DeviceStateService {
}
@Override
public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
protected void onTbApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) {
deduplicationExecutor.submit(partitionChangeEvent.getPartitions());
}

5
application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java

@ -48,6 +48,7 @@ import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
import org.thingsboard.server.queue.util.TbCoreComponent;
@ -76,7 +77,7 @@ import java.util.function.Predicate;
@Slf4j
@TbCoreComponent
@Service
public class DefaultSubscriptionManagerService implements SubscriptionManagerService {
public class DefaultSubscriptionManagerService extends TbApplicationEventListener<PartitionChangeEvent> implements SubscriptionManagerService {
@Autowired
private AttributesService attrService;
@ -178,7 +179,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer
}
@Override
public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
protected void onTbApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) {
Set<TopicPartitionInfo> removedPartitions = new HashSet<>(currentPartitions);
removedPartitions.removeAll(partitionChangeEvent.getPartitions());

49
application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java

@ -28,6 +28,7 @@ import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.queue.TbClusterService;
import org.thingsboard.server.service.telemetry.sub.AlarmSubscriptionUpdate;
@ -62,6 +63,34 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer
private SubscriptionManagerService subscriptionManagerService;
private ExecutorService subscriptionUpdateExecutor;
private TbApplicationEventListener<PartitionChangeEvent> partitionChangeListener = new TbApplicationEventListener<>() {
@Override
protected void onTbApplicationEvent(PartitionChangeEvent event) {
if (ServiceType.TB_CORE.equals(event.getServiceType())) {
currentPartitions.clear();
currentPartitions.addAll(event.getPartitions());
}
}
};
private TbApplicationEventListener<ClusterTopologyChangeEvent> clusterTopologyChangeListener = new TbApplicationEventListener<>() {
@Override
protected void onTbApplicationEvent(ClusterTopologyChangeEvent event) {
if (event.getServiceQueueKeys().stream().anyMatch(key -> ServiceType.TB_CORE.equals(key.getServiceType()))) {
/*
* If the cluster topology has changed, we need to push all current subscriptions to SubscriptionManagerService again.
* Otherwise, the SubscriptionManagerService may "forget" those subscriptions in case of restart.
* Although this is resource consuming operation, it is cheaper than sending ping/pong commands periodically
* It is also cheaper then caching the subscriptions by entity id and then lookup of those caches every time we have new telemetry in SubscriptionManagerService.
* Even if we cache locally the list of active subscriptions by entity id, it is still time consuming operation to get them from cache
* Since number of subscriptions is usually much less then number of devices that are pushing data.
*/
subscriptionsBySessionId.values().forEach(map -> map.values()
.forEach(sub -> pushSubscriptionToManagerService(sub, true)));
}
}
};
@PostConstruct
public void initExecutor() {
@ -77,28 +106,14 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer
@Override
@EventListener(PartitionChangeEvent.class)
public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) {
currentPartitions.clear();
currentPartitions.addAll(partitionChangeEvent.getPartitions());
}
public void onApplicationEvent(PartitionChangeEvent event) {
partitionChangeListener.onApplicationEvent(event);
}
@Override
@EventListener(ClusterTopologyChangeEvent.class)
public void onApplicationEvent(ClusterTopologyChangeEvent event) {
if (event.getServiceQueueKeys().stream().anyMatch(key -> ServiceType.TB_CORE.equals(key.getServiceType()))) {
/*
* If the cluster topology has changed, we need to push all current subscriptions to SubscriptionManagerService again.
* Otherwise, the SubscriptionManagerService may "forget" those subscriptions in case of restart.
* Although this is resource consuming operation, it is cheaper than sending ping/pong commands periodically
* It is also cheaper then caching the subscriptions by entity id and then lookup of those caches every time we have new telemetry in SubscriptionManagerService.
* Even if we cache locally the list of active subscriptions by entity id, it is still time consuming operation to get them from cache
* Since number of subscriptions is usually much less then number of devices that are pushing data.
*/
subscriptionsBySessionId.values().forEach(map -> map.values()
.forEach(sub -> pushSubscriptionToManagerService(sub, true)));
}
clusterTopologyChangeListener.onApplicationEvent(event);
}
//TODO 3.1: replace null callbacks with callbacks from websocket service.

6
application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java

@ -41,6 +41,7 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
import org.thingsboard.server.service.queue.TbClusterService;
import org.thingsboard.server.service.subscription.SubscriptionManagerService;
import org.thingsboard.server.service.subscription.TbSubscriptionUtils;
@ -61,7 +62,7 @@ import java.util.function.Consumer;
* Created by ashvayka on 27.03.18.
*/
@Slf4j
public abstract class AbstractSubscriptionService implements ApplicationListener<PartitionChangeEvent> {
public abstract class AbstractSubscriptionService extends TbApplicationEventListener<PartitionChangeEvent>{
protected final Set<TopicPartitionInfo> currentPartitions = ConcurrentHashMap.newKeySet();
@ -97,8 +98,7 @@ public abstract class AbstractSubscriptionService implements ApplicationListener
}
@Override
@EventListener(PartitionChangeEvent.class)
public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
protected void onTbApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) {
currentPartitions.clear();
currentPartitions.addAll(partitionChangeEvent.getPartitions());

2
application/src/main/resources/logback.xml

@ -30,7 +30,7 @@
<!-- <logger name="org.thingsboard.server.service.queue" level="TRACE" />-->
<!-- <logger name="org.thingsboard.server.service.transport" level="TRACE" />-->
<!-- <logger name="org.thingsboard.server.queue.memory.InMemoryStorage" level="DEBUG" />-->
<!-- <logger name="org.thingsboard.server.service.ttl.AbstractCleanUpService" level="DEBUG" />-->
<!-- <logger name="org.thingsboard.server.service.subscription" level="TRACE"/>-->
<!-- <logger name="org.thingsboard.server.service.telemetry" level="TRACE"/>-->

20
application/src/main/resources/thingsboard.yml

@ -118,6 +118,15 @@ security:
githubMapper:
emailUrl: "${SECURITY_OAUTH2_GITHUB_MAPPER_EMAIL_URL_KEY:https://api.github.com/user/emails}"
# Usage statistics parameters
usage:
stats:
report:
enabled: "${USAGE_STATS_REPORT_ENABLED:true}"
interval: "${USAGE_STATS_REPORT_INTERVAL:10}"
check:
cycle: "${USAGE_STATS_CHECK_CYCLE:60000}"
# Dashboard parameters
dashboard:
# Maximum allowed datapoints fetched by widgets
@ -313,6 +322,9 @@ actors:
cache:
# caffeine or redis
type: "${CACHE_TYPE:caffeine}"
attributes:
# make sure that if cache.type is 'redis' and cache.attributes.enabled is 'true' that you change 'maxmemory-policy' Redis config property to 'allkeys-lru', 'allkeys-lfu' or 'allkeys-random'
enabled: "${CACHE_ATTRIBUTES_ENABLED:true}"
caffeine:
specs:
@ -346,6 +358,9 @@ caffeine:
deviceProfiles:
timeToLiveInMinutes: 1440
maxSize: 0
attributes:
timeToLiveInMinutes: 1440
maxSize: 100000
redis:
# standalone or cluster
@ -530,6 +545,7 @@ transport:
http:
enabled: "${HTTP_ENABLED:true}"
request_timeout: "${HTTP_REQUEST_TIMEOUT:60000}"
max_request_timeout: "${HTTP_MAX_REQUEST_TIMEOUT:300000}"
# Local MQTT transport parameters
mqtt:
# Enable/disable mqtt transport protocol.
@ -613,6 +629,10 @@ queue:
transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}"
notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}"
js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1}"
consumer-stats:
enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}"
print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}"
kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}"
aws_sqs:
use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}"
access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}"

1
common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java

@ -26,4 +26,5 @@ public class CacheConstants {
public static final String SECURITY_SETTINGS_CACHE = "securitySettings";
public static final String TENANT_PROFILE_CACHE = "tenantProfiles";
public static final String DEVICE_PROFILE_CACHE = "deviceProfiles";
public static final String ATTRIBUTES_CACHE = "attributes";
}

2
common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmCondition.java

@ -26,7 +26,7 @@ import java.util.concurrent.TimeUnit;
@JsonIgnoreProperties(ignoreUnknown = true)
public class AlarmCondition {
private List<KeyFilter> condition;
private List<AlarmConditionFilter> condition;
private AlarmConditionSpec spec;
}

30
common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilter.java

@ -0,0 +1,30 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.device.profile;
import lombok.Data;
import org.thingsboard.server.common.data.query.EntityKeyValueType;
import org.thingsboard.server.common.data.query.KeyFilterPredicate;
@Data
public class AlarmConditionFilter {
private AlarmConditionFilterKey key;
private EntityKeyValueType valueType;
private Object value;
private KeyFilterPredicate predicate;
}

26
common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilterKey.java

@ -0,0 +1,26 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.device.profile;
import lombok.Data;
@Data
public class AlarmConditionFilterKey {
private final AlarmConditionKeyType type;
private final String key;
}

23
common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionKeyType.java

@ -0,0 +1,23 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.device.profile;
public enum AlarmConditionKeyType {
ATTRIBUTE,
TIME_SERIES,
ENTITY_FIELD,
CONSTANT
}

12
common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java

@ -17,17 +17,23 @@ package org.thingsboard.server.common.data.query;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Data;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
@Data
@RequiredArgsConstructor
public class DynamicValue<T> {
@JsonIgnore
private T resolvedValue;
@Getter
private final DynamicValueSourceType sourceType;
@Getter
private final String sourceAttribute;
private final boolean inherit;
public DynamicValue(DynamicValueSourceType sourceType, String sourceAttribute) {
this.sourceAttribute = sourceAttribute;
this.sourceType = sourceType;
this.inherit = false;
}
}

2
common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java

@ -120,7 +120,7 @@ public final class TbMsg implements Serializable {
private TbMsg(String queueName, UUID id, long ts, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data,
RuleChainId ruleChainId, RuleNodeId ruleNodeId, int ruleNodeExecCounter, TbMsgCallback callback) {
this.id = id;
this.queueName = queueName;
this.queueName = queueName != null ? queueName : ServiceQueue.MAIN;
if (ts > 0) {
this.ts = ts;
} else {

4
common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java

@ -22,7 +22,9 @@ import org.thingsboard.server.common.msg.queue.ServiceQueueKey;
import java.util.Set;
public class ClusterTopologyChangeEvent extends ApplicationEvent {
public class ClusterTopologyChangeEvent extends TbApplicationEvent {
private static final long serialVersionUID = -2441739930040282254L;
@Getter
private final Set<ServiceQueueKey> serviceQueueKeys;

4
common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java

@ -126,7 +126,7 @@ public class HashPartitionService implements PartitionService {
}
@Override
public void recalculatePartitions(ServiceInfo currentService, List<ServiceInfo> otherServices) {
public synchronized void recalculatePartitions(ServiceInfo currentService, List<ServiceInfo> otherServices) {
logServiceInfo(currentService);
otherServices.forEach(this::logServiceInfo);
Map<ServiceQueueKey, List<ServiceInfo>> queueServicesMap = new HashMap<>();
@ -134,7 +134,7 @@ public class HashPartitionService implements PartitionService {
for (ServiceInfo other : otherServices) {
addNode(queueServicesMap, other);
}
queueServicesMap.values().forEach(list -> list.sort((a, b) -> a.getServiceId().compareTo(b.getServiceId())));
queueServicesMap.values().forEach(list -> list.sort(Comparator.comparing(ServiceInfo::getServiceId)));
ConcurrentMap<ServiceQueueKey, List<Integer>> oldPartitions = myPartitions;
TenantId myIsolatedOrSystemTenantId = getSystemOrIsolatedTenantId(currentService);

4
common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java

@ -24,7 +24,9 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import java.util.Set;
public class PartitionChangeEvent extends ApplicationEvent {
public class PartitionChangeEvent extends TbApplicationEvent {
private static final long serialVersionUID = -8731788167026510559L;
@Getter
private final ServiceQueueKey serviceQueueKey;

37
common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEvent.java

@ -0,0 +1,37 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.discovery;
import lombok.Getter;
import org.springframework.context.ApplicationEvent;
import java.util.concurrent.atomic.AtomicInteger;
public class TbApplicationEvent extends ApplicationEvent {
private static final long serialVersionUID = 3884264064887765146L;
private static final AtomicInteger sequence = new AtomicInteger();
@Getter
private final int sequenceNumber;
public TbApplicationEvent(Object source) {
super(source);
sequenceNumber = sequence.incrementAndGet();
}
}

52
common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java

@ -0,0 +1,52 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.discovery;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.ApplicationListener;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@Slf4j
public abstract class TbApplicationEventListener<T extends TbApplicationEvent> implements ApplicationListener<T> {
private int lastProcessedSequenceNumber = Integer.MIN_VALUE;
private final Lock seqNumberLock = new ReentrantLock();
@Override
public void onApplicationEvent(T event) {
boolean validUpdate = false;
seqNumberLock.lock();
try {
if (event.getSequenceNumber() > lastProcessedSequenceNumber) {
validUpdate = true;
lastProcessedSequenceNumber = event.getSequenceNumber();
}
} finally {
seqNumberLock.unlock();
}
if (validUpdate) {
onTbApplicationEvent(event);
} else {
log.info("Application event ignored due to invalid sequence number ({} > {}). Event: {}", lastProcessedSequenceNumber, event.getSequenceNumber(), event);
}
}
protected abstract void onTbApplicationEvent(T event);
}

37
common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatisticConfig.java

@ -0,0 +1,37 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.kafka;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
@Component
@ConditionalOnProperty(prefix = "queue", value = "type", havingValue = "kafka")
@Getter
@AllArgsConstructor
@NoArgsConstructor
public class TbKafkaConsumerStatisticConfig {
@Value("${queue.kafka.consumer-stats.enabled:true}")
private Boolean enabled;
@Value("${queue.kafka.consumer-stats.print-interval-ms:60000}")
private Long printIntervalMs;
@Value("${queue.kafka.consumer-stats.kafka-response-timeout-ms:1000}")
private Long kafkaResponseTimeoutMs;
}

184
common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java

@ -0,0 +1,184 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.kafka;
import lombok.Builder;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.admin.AdminClient;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.TopicPartition;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.queue.discovery.PartitionService;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@Slf4j
@Component
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "queue", value = "type", havingValue = "kafka")
public class TbKafkaConsumerStatsService {
private final Set<String> monitoredGroups = ConcurrentHashMap.newKeySet();
private final TbKafkaSettings kafkaSettings;
private final TbKafkaConsumerStatisticConfig statsConfig;
private final PartitionService partitionService;
private AdminClient adminClient;
private Consumer<String, byte[]> consumer;
private ScheduledExecutorService statsPrintScheduler;
@PostConstruct
public void init() {
if (!statsConfig.getEnabled()) {
return;
}
this.adminClient = AdminClient.create(kafkaSettings.toAdminProps());
this.statsPrintScheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("kafka-consumer-stats"));
Properties consumerProps = kafkaSettings.toConsumerProps();
consumerProps.put(ConsumerConfig.CLIENT_ID_CONFIG, "consumer-stats-loader-client");
consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, "consumer-stats-loader-client-group");
this.consumer = new KafkaConsumer<>(consumerProps);
startLogScheduling();
}
private void startLogScheduling() {
Duration timeoutDuration = Duration.ofMillis(statsConfig.getKafkaResponseTimeoutMs());
statsPrintScheduler.scheduleWithFixedDelay(() -> {
if (!isStatsPrintRequired()) {
return;
}
for (String groupId : monitoredGroups) {
try {
Map<TopicPartition, OffsetAndMetadata> groupOffsets = adminClient.listConsumerGroupOffsets(groupId).partitionsToOffsetAndMetadata()
.get(statsConfig.getKafkaResponseTimeoutMs(), TimeUnit.MILLISECONDS);
Map<TopicPartition, Long> endOffsets = consumer.endOffsets(groupOffsets.keySet(), timeoutDuration);
List<GroupTopicStats> lagTopicsStats = getTopicsStatsWithLag(groupOffsets, endOffsets);
if (!lagTopicsStats.isEmpty()) {
StringBuilder builder = new StringBuilder();
for (int i = 0; i < lagTopicsStats.size(); i++) {
builder.append(lagTopicsStats.get(i).toString());
if (i != lagTopicsStats.size() - 1) {
builder.append(", ");
}
}
log.info("[{}] Topic partitions with lag: [{}].", groupId, builder.toString());
}
} catch (Exception e) {
log.warn("[{}] Failed to get consumer group stats. Reason - {}.", groupId, e.getMessage());
log.trace("Detailed error: ", e);
}
}
}, statsConfig.getPrintIntervalMs(), statsConfig.getPrintIntervalMs(), TimeUnit.MILLISECONDS);
}
private boolean isStatsPrintRequired() {
boolean isMyRuleEnginePartition = partitionService.resolve(ServiceType.TB_RULE_ENGINE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID).isMyPartition();
boolean isMyCorePartition = partitionService.resolve(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID).isMyPartition();
return log.isInfoEnabled() && (isMyRuleEnginePartition || isMyCorePartition);
}
private List<GroupTopicStats> getTopicsStatsWithLag(Map<TopicPartition, OffsetAndMetadata> groupOffsets, Map<TopicPartition, Long> endOffsets) {
List<GroupTopicStats> consumerGroupStats = new ArrayList<>();
for (TopicPartition topicPartition : groupOffsets.keySet()) {
long endOffset = endOffsets.get(topicPartition);
long committedOffset = groupOffsets.get(topicPartition).offset();
long lag = endOffset - committedOffset;
if (lag != 0) {
GroupTopicStats groupTopicStats = GroupTopicStats.builder()
.topic(topicPartition.topic())
.partition(topicPartition.partition())
.committedOffset(committedOffset)
.endOffset(endOffset)
.lag(lag)
.build();
consumerGroupStats.add(groupTopicStats);
}
}
return consumerGroupStats;
}
public void registerClientGroup(String groupId) {
if (statsConfig.getEnabled() && !StringUtils.isEmpty(groupId)) {
monitoredGroups.add(groupId);
}
}
public void unregisterClientGroup(String groupId) {
if (statsConfig.getEnabled() && !StringUtils.isEmpty(groupId)) {
monitoredGroups.remove(groupId);
}
}
@PreDestroy
public void destroy() {
if (statsPrintScheduler != null) {
statsPrintScheduler.shutdownNow();
}
if (adminClient != null) {
adminClient.close();
}
if (consumer != null) {
consumer.close();
}
}
@Builder
@Data
private static class GroupTopicStats {
private String topic;
private int partition;
private long committedOffset;
private long endOffset;
private long lag;
@Override
public String toString() {
return "[" +
"topic=[" + topic + ']' +
", partition=[" + partition + "]" +
", committedOffset=[" + committedOffset + "]" +
", endOffset=[" + endOffset + "]" +
", lag=[" + lag + "]" +
"]";
}
}
}

16
common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java

@ -42,10 +42,13 @@ public class TbKafkaConsumerTemplate<T extends TbQueueMsg> extends AbstractTbQue
private final KafkaConsumer<String, byte[]> consumer;
private final TbKafkaDecoder<T> decoder;
private final TbKafkaConsumerStatsService statsService;
private final String groupId;
@Builder
private TbKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder<T> decoder,
String clientId, String groupId, String topic,
TbQueueAdmin admin) {
TbQueueAdmin admin, TbKafkaConsumerStatsService statsService) {
super(topic);
Properties props = settings.toConsumerProps();
props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId);
@ -53,6 +56,13 @@ public class TbKafkaConsumerTemplate<T extends TbQueueMsg> extends AbstractTbQue
props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
}
this.statsService = statsService;
this.groupId = groupId;
if (statsService != null) {
statsService.registerClientGroup(groupId);
}
this.admin = admin;
this.consumer = new KafkaConsumer<>(props);
this.decoder = decoder;
@ -96,6 +106,8 @@ public class TbKafkaConsumerTemplate<T extends TbQueueMsg> extends AbstractTbQue
consumer.unsubscribe();
consumer.close();
}
if (statsService != null) {
statsService.unregisterClientGroup(groupId);
}
}
}

11
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java

@ -39,6 +39,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.kafka.TbKafkaAdmin;
import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService;
import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate;
import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate;
import org.thingsboard.server.queue.kafka.TbKafkaSettings;
@ -65,6 +66,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueTransportNotificationSettings transportNotificationSettings;
private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
private final TbKafkaConsumerStatsService consumerStatsService;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
@ -79,6 +81,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
TbQueueTransportApiSettings transportApiSettings,
TbQueueTransportNotificationSettings transportNotificationSettings,
TbQueueRemoteJsInvokeSettings jsInvokeSettings,
TbKafkaConsumerStatsService consumerStatsService,
TbKafkaTopicConfigs kafkaTopicConfigs) {
this.partitionService = partitionService;
this.kafkaSettings = kafkaSettings;
@ -88,6 +91,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
this.transportApiSettings = transportApiSettings;
this.transportNotificationSettings = transportNotificationSettings;
this.jsInvokeSettings = jsInvokeSettings;
this.consumerStatsService = consumerStatsService;
this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs());
this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs());
@ -156,6 +160,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
consumerBuilder.groupId("re-" + queueName + "-consumer");
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()));
consumerBuilder.admin(ruleEngineAdmin);
consumerBuilder.statsService(consumerStatsService);
return consumerBuilder.build();
}
@ -168,6 +173,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
consumerBuilder.groupId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId());
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
consumerBuilder.admin(notificationAdmin);
consumerBuilder.statsService(consumerStatsService);
return consumerBuilder.build();
}
@ -180,6 +186,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
consumerBuilder.groupId("monolith-core-consumer");
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()));
consumerBuilder.admin(coreAdmin);
consumerBuilder.statsService(consumerStatsService);
return consumerBuilder.build();
}
@ -192,6 +199,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
consumerBuilder.groupId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId());
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
consumerBuilder.admin(notificationAdmin);
consumerBuilder.statsService(consumerStatsService);
return consumerBuilder.build();
}
@ -204,6 +212,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
consumerBuilder.groupId("monolith-transport-api-consumer");
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()));
consumerBuilder.admin(transportApiAdmin);
consumerBuilder.statsService(consumerStatsService);
return consumerBuilder.build();
}
@ -237,6 +246,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders());
}
);
responseBuilder.statsService(consumerStatsService);
responseBuilder.admin(jsExecutorAdmin);
DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
@ -259,6 +269,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
consumerBuilder.groupId("monolith-us-consumer");
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
consumerBuilder.admin(coreAdmin);
consumerBuilder.statsService(consumerStatsService);
return consumerBuilder.build();
}

9
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java

@ -39,6 +39,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.kafka.TbKafkaAdmin;
import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService;
import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate;
import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate;
import org.thingsboard.server.queue.kafka.TbKafkaSettings;
@ -62,6 +63,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
private final TbQueueRuleEngineSettings ruleEngineSettings;
private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
private final TbKafkaConsumerStatsService consumerStatsService;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
@ -75,6 +77,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
TbQueueRuleEngineSettings ruleEngineSettings,
TbQueueTransportApiSettings transportApiSettings,
TbQueueRemoteJsInvokeSettings jsInvokeSettings,
TbKafkaConsumerStatsService consumerStatsService,
TbKafkaTopicConfigs kafkaTopicConfigs) {
this.partitionService = partitionService;
this.kafkaSettings = kafkaSettings;
@ -83,6 +86,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
this.ruleEngineSettings = ruleEngineSettings;
this.transportApiSettings = transportApiSettings;
this.jsInvokeSettings = jsInvokeSettings;
this.consumerStatsService = consumerStatsService;
this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs());
this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs());
@ -150,6 +154,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
consumerBuilder.groupId("tb-core-node");
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()));
consumerBuilder.admin(coreAdmin);
consumerBuilder.statsService(consumerStatsService);
return consumerBuilder.build();
}
@ -162,6 +167,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
consumerBuilder.groupId("tb-core-notifications-node-" + serviceInfoProvider.getServiceId());
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
consumerBuilder.admin(notificationAdmin);
consumerBuilder.statsService(consumerStatsService);
return consumerBuilder.build();
}
@ -174,6 +180,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
consumerBuilder.groupId("tb-core-transport-api-consumer");
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()));
consumerBuilder.admin(transportApiAdmin);
consumerBuilder.statsService(consumerStatsService);
return consumerBuilder.build();
}
@ -208,6 +215,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
}
);
responseBuilder.admin(jsExecutorAdmin);
responseBuilder.statsService(consumerStatsService);
DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder();
@ -229,6 +237,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
consumerBuilder.groupId("tb-core-us-consumer");
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
consumerBuilder.admin(coreAdmin);
consumerBuilder.statsService(consumerStatsService);
return consumerBuilder.build();
}

7
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java

@ -37,6 +37,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.kafka.TbKafkaAdmin;
import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService;
import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate;
import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate;
import org.thingsboard.server.queue.kafka.TbKafkaSettings;
@ -59,6 +60,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
private final TbQueueCoreSettings coreSettings;
private final TbQueueRuleEngineSettings ruleEngineSettings;
private final TbQueueRemoteJsInvokeSettings jsInvokeSettings;
private final TbKafkaConsumerStatsService consumerStatsService;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
@ -70,6 +72,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
TbQueueCoreSettings coreSettings,
TbQueueRuleEngineSettings ruleEngineSettings,
TbQueueRemoteJsInvokeSettings jsInvokeSettings,
TbKafkaConsumerStatsService consumerStatsService,
TbKafkaTopicConfigs kafkaTopicConfigs) {
this.partitionService = partitionService;
this.kafkaSettings = kafkaSettings;
@ -77,6 +80,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
this.coreSettings = coreSettings;
this.ruleEngineSettings = ruleEngineSettings;
this.jsInvokeSettings = jsInvokeSettings;
this.consumerStatsService = consumerStatsService;
this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs());
this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs());
@ -145,6 +149,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
consumerBuilder.groupId("re-" + queueName + "-consumer");
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()));
consumerBuilder.admin(ruleEngineAdmin);
consumerBuilder.statsService(consumerStatsService);
return consumerBuilder.build();
}
@ -157,6 +162,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
consumerBuilder.groupId("tb-rule-engine-notifications-node-" + serviceInfoProvider.getServiceId());
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()));
consumerBuilder.admin(notificationAdmin);
consumerBuilder.statsService(consumerStatsService);
return consumerBuilder.build();
}
@ -181,6 +187,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
}
);
responseBuilder.admin(jsExecutorAdmin);
responseBuilder.statsService(consumerStatsService);
DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> builder = DefaultTbQueueRequestTemplate.builder();

6
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java

@ -32,6 +32,7 @@ import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.kafka.TbKafkaAdmin;
import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService;
import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate;
import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate;
import org.thingsboard.server.queue.kafka.TbKafkaSettings;
@ -54,6 +55,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory {
private final TbQueueRuleEngineSettings ruleEngineSettings;
private final TbQueueTransportApiSettings transportApiSettings;
private final TbQueueTransportNotificationSettings transportNotificationSettings;
private final TbKafkaConsumerStatsService consumerStatsService;
private final TbQueueAdmin coreAdmin;
private final TbQueueAdmin ruleEngineAdmin;
@ -66,6 +68,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory {
TbQueueRuleEngineSettings ruleEngineSettings,
TbQueueTransportApiSettings transportApiSettings,
TbQueueTransportNotificationSettings transportNotificationSettings,
TbKafkaConsumerStatsService consumerStatsService,
TbKafkaTopicConfigs kafkaTopicConfigs) {
this.kafkaSettings = kafkaSettings;
this.serviceInfoProvider = serviceInfoProvider;
@ -73,6 +76,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory {
this.ruleEngineSettings = ruleEngineSettings;
this.transportApiSettings = transportApiSettings;
this.transportNotificationSettings = transportNotificationSettings;
this.consumerStatsService = consumerStatsService;
this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs());
this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs());
@ -95,6 +99,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory {
responseBuilder.groupId("transport-node-" + serviceInfoProvider.getServiceId());
responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders()));
responseBuilder.admin(transportApiAdmin);
responseBuilder.statsService(consumerStatsService);
DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder
<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> templateBuilder = DefaultTbQueueRequestTemplate.builder();
@ -136,6 +141,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory {
responseBuilder.groupId("transport-node-" + serviceInfoProvider.getServiceId());
responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders()));
responseBuilder.admin(notificationAdmin);
responseBuilder.statsService(consumerStatsService);
return responseBuilder.build();
}

18
common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java

@ -17,9 +17,13 @@ package org.thingsboard.server.transport.http;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.coyote.ProtocolHandler;
import org.apache.coyote.http11.Http11NioProtocol;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.web.embedded.tomcat.TomcatConnectorCustomizer;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.transport.TransportContext;
@ -37,4 +41,18 @@ public class HttpTransportContext extends TransportContext {
@Value("${transport.http.request_timeout}")
private long defaultTimeout;
@Getter
@Value("${transport.http.max_request_timeout}")
private long maxRequestTimeout;
@Bean
public TomcatConnectorCustomizer tomcatAsyncTimeoutConnectorCustomizer() {
return connector -> {
ProtocolHandler handler = connector.getProtocolHandler();
if (handler instanceof Http11NioProtocol) {
log.trace("Setting async max request timeout {}", maxRequestTimeout);
connector.setAsyncTimeout(maxRequestTimeout);
}
};
}
}

41
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java

@ -51,6 +51,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ValidateBasicMqttCre
import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@ -245,12 +246,25 @@ public class JsonConverter {
}
private static void parseNumericValue(List<KvEntry> result, Entry<String, JsonElement> valueEntry, JsonPrimitive value) {
if (value.getAsString().contains(".")) {
result.add(new DoubleDataEntry(valueEntry.getKey(), value.getAsDouble()));
String valueAsString = value.getAsString();
String key = valueEntry.getKey();
if (valueAsString.contains("e") || valueAsString.contains("E")) {
var bd = new BigDecimal(valueAsString);
if (bd.stripTrailingZeros().scale() <= 0) {
try {
result.add(new LongDataEntry(key, bd.longValueExact()));
} catch (ArithmeticException e) {
result.add(new DoubleDataEntry(key, bd.doubleValue()));
}
} else {
result.add(new DoubleDataEntry(key, bd.doubleValue()));
}
} else if (valueAsString.contains(".")) {
result.add(new DoubleDataEntry(key, value.getAsDouble()));
} else {
try {
long longValue = Long.parseLong(value.getAsString());
result.add(new LongDataEntry(valueEntry.getKey(), longValue));
result.add(new LongDataEntry(key, longValue));
} catch (NumberFormatException e) {
throw new JsonSyntaxException("Big integer values are not supported!");
}
@ -285,7 +299,8 @@ public class JsonConverter {
return result;
}
public static JsonObject getJsonObjectForGateway(String deviceName, TransportProtos.GetAttributeResponseMsg responseMsg) {
public static JsonObject getJsonObjectForGateway(String deviceName, TransportProtos.GetAttributeResponseMsg
responseMsg) {
JsonObject result = new JsonObject();
result.addProperty("id", responseMsg.getRequestId());
result.addProperty(DEVICE_PROPERTY, deviceName);
@ -298,7 +313,8 @@ public class JsonConverter {
return result;
}
public static JsonObject getJsonObjectForGateway(String deviceName, AttributeUpdateNotificationMsg notificationMsg) {
public static JsonObject getJsonObjectForGateway(String deviceName, AttributeUpdateNotificationMsg
notificationMsg) {
JsonObject result = new JsonObject();
result.addProperty(DEVICE_PROPERTY, deviceName);
result.add("data", toJson(notificationMsg));
@ -446,7 +462,8 @@ public class JsonConverter {
return result;
}
public static JsonElement toGatewayJson(String deviceName, TransportProtos.ProvisionDeviceResponseMsg responseRequest) {
public static JsonElement toGatewayJson(String deviceName, TransportProtos.ProvisionDeviceResponseMsg
responseRequest) {
JsonObject result = new JsonObject();
result.addProperty(DEVICE_PROPERTY, deviceName);
result.add("data", JsonConverter.toJson(responseRequest));
@ -496,15 +513,18 @@ public class JsonConverter {
return result;
}
public static Map<Long, List<KvEntry>> convertToTelemetry(JsonElement jsonElement, long systemTs) throws JsonSyntaxException {
public static Map<Long, List<KvEntry>> convertToTelemetry(JsonElement jsonElement, long systemTs) throws
JsonSyntaxException {
return convertToTelemetry(jsonElement, systemTs, false);
}
public static Map<Long, List<KvEntry>> convertToSortedTelemetry(JsonElement jsonElement, long systemTs) throws JsonSyntaxException {
public static Map<Long, List<KvEntry>> convertToSortedTelemetry(JsonElement jsonElement, long systemTs) throws
JsonSyntaxException {
return convertToTelemetry(jsonElement, systemTs, true);
}
public static Map<Long, List<KvEntry>> convertToTelemetry(JsonElement jsonElement, long systemTs, boolean sorted) throws JsonSyntaxException {
public static Map<Long, List<KvEntry>> convertToTelemetry(JsonElement jsonElement, long systemTs, boolean sorted) throws
JsonSyntaxException {
Map<Long, List<KvEntry>> result = sorted ? new TreeMap<>() : new HashMap<>();
convertToTelemetry(jsonElement, systemTs, result, null);
return result;
@ -574,7 +594,8 @@ public class JsonConverter {
.build();
}
private static TransportProtos.ProvisionDeviceCredentialsMsg buildProvisionDeviceCredentialsMsg(String provisionKey, String provisionSecret) {
private static TransportProtos.ProvisionDeviceCredentialsMsg buildProvisionDeviceCredentialsMsg(String
provisionKey, String provisionSecret) {
return TransportProtos.ProvisionDeviceCredentialsMsg.newBuilder()
.setProvisionDeviceKey(provisionKey)
.setProvisionDeviceSecret(provisionSecret)

6
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java

@ -763,8 +763,8 @@ public class DefaultTransportService implements TransportService {
wrappedCallback);
}
protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) {
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator());
private void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) {
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tbMsg.getQueueName(), tenantId, tbMsg.getOriginator());
if (log.isTraceEnabled()) {
log.trace("[{}][{}] Pushing to topic {} message {}", tenantId, tbMsg.getOriginator(), tpi.getFullTopicName(), tbMsg);
}
@ -776,7 +776,7 @@ public class DefaultTransportService implements TransportService {
ruleEngineMsgProducer.send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), wrappedCallback);
}
protected void sendToRuleEngine(TenantId tenantId, DeviceId deviceId, TransportProtos.SessionInfoProto sessionInfo, JsonObject json,
private void sendToRuleEngine(TenantId tenantId, DeviceId deviceId, TransportProtos.SessionInfoProto sessionInfo, JsonObject json,
TbMsgMetaData metaData, SessionMsgType sessionMsgType, TbQueueCallback callback) {
DeviceProfileId deviceProfileId = new DeviceProfileId(new UUID(sessionInfo.getDeviceProfileIdMSB(), sessionInfo.getDeviceProfileIdLSB()));
DeviceProfile deviceProfile = deviceProfileCache.get(deviceProfileId);

53
common/transport/transport-api/src/test/java/JsonConverterTest.java

@ -0,0 +1,53 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import com.google.gson.JsonParser;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.junit.MockitoJUnitRunner;
import org.thingsboard.server.common.transport.adaptor.JsonConverter;
@RunWith(MockitoJUnitRunner.class)
public class JsonConverterTest {
private static final JsonParser JSON_PARSER = new JsonParser();
@Test
public void testParseBigDecimalAsLong() {
var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 1E+1}"), 0L);
Assert.assertEquals(10L, result.get(0L).get(0).getLongValue().get().longValue());
}
@Test
public void testParseBigDecimalAsDouble() {
var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 101E-1}"), 0L);
Assert.assertEquals(10.1, result.get(0L).get(0).getDoubleValue().get(), 0.0);
}
@Test
public void testParseAsDouble() {
var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 1.1}"), 0L);
Assert.assertEquals(1.1, result.get(0L).get(0).getDoubleValue().get(), 0.0);
}
@Test
public void testParseAsLong() {
var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 11}"), 0L);
Assert.assertEquals(11L, result.get(0L).get(0).getLongValue().get().longValue());
}
}

39
dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCacheKey.java

@ -0,0 +1,39 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.attributes;
import lombok.AllArgsConstructor;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import org.thingsboard.server.common.data.id.EntityId;
import java.io.Serializable;
@EqualsAndHashCode
@Getter
@AllArgsConstructor
public class AttributeCacheKey implements Serializable {
private static final long serialVersionUID = 2013369077925351881L;
private final String scope;
private final EntityId entityId;
private final String key;
@Override
public String toString() {
return entityId + "_" + scope + "_" + key;
}
}

39
dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java

@ -0,0 +1,39 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.attributes;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.service.Validator;
public class AttributeUtils {
public static void validate(EntityId id, String scope) {
Validator.validateId(id.getId(), "Incorrect id " + id);
Validator.validateString(scope, "Incorrect scope " + scope);
}
public static void validate(AttributeKvEntry kvEntry) {
if (kvEntry == null) {
throw new IncorrectParameterException("Key value entry can't be null");
} else if (kvEntry.getDataType() == null) {
throw new IncorrectParameterException("Incorrect kvEntry. Data type can't be null");
} else {
Validator.validateString(kvEntry.getKey(), "Incorrect kvEntry. Key can't be empty");
Validator.validatePositiveNumber(kvEntry.getLastUpdateTs(), "Incorrect last update ts. Ts should be positive");
}
}
}

63
dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesCacheWrapper.java

@ -0,0 +1,63 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.attributes;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import static org.thingsboard.server.common.data.CacheConstants.ATTRIBUTES_CACHE;
@Service
@ConditionalOnProperty(prefix = "cache.attributes", value = "enabled", havingValue = "true")
@Primary
@Slf4j
public class AttributesCacheWrapper {
private final Cache attributesCache;
public AttributesCacheWrapper(CacheManager cacheManager) {
this.attributesCache = cacheManager.getCache(ATTRIBUTES_CACHE);
}
public Cache.ValueWrapper get(AttributeCacheKey attributeCacheKey) {
try {
return attributesCache.get(attributeCacheKey);
} catch (Exception e) {
log.debug("Failed to retrieve element from cache for key {}. Reason - {}.", attributeCacheKey, e.getMessage());
return null;
}
}
public void put(AttributeCacheKey attributeCacheKey, AttributeKvEntry attributeKvEntry) {
try {
attributesCache.put(attributeCacheKey, attributeKvEntry);
} catch (Exception e) {
log.debug("Failed to put element from cache for key {}. Reason - {}.", attributeCacheKey, e.getMessage());
}
}
public void evict(AttributeCacheKey attributeCacheKey) {
try {
attributesCache.evict(attributeCacheKey);
} catch (Exception e) {
log.debug("Failed to evict element from cache for key {}. Reason - {}.", attributeCacheKey, e.getMessage());
}
}
}

47
dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java

@ -15,31 +15,39 @@
*/
package org.thingsboard.server.dao.attributes;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import org.springframework.beans.factory.annotation.Autowired;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.service.Validator;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import static org.thingsboard.server.dao.attributes.AttributeUtils.validate;
/**
* @author Andrew Shvayka
*/
@Service
@ConditionalOnProperty(prefix = "cache.attributes", value = "enabled", havingValue = "false", matchIfMissing = true)
@Primary
@Slf4j
public class BaseAttributesService implements AttributesService {
private final AttributesDao attributesDao;
@Autowired
private AttributesDao attributesDao;
public BaseAttributesService(AttributesDao attributesDao) {
this.attributesDao = attributesDao;
}
@Override
public ListenableFuture<Optional<AttributeKvEntry>> find(TenantId tenantId, EntityId entityId, String scope, String attributeKey) {
@ -75,33 +83,14 @@ public class BaseAttributesService implements AttributesService {
public ListenableFuture<List<Void>> save(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes) {
validate(entityId, scope);
attributes.forEach(attribute -> validate(attribute));
List<ListenableFuture<Void>> futures = Lists.newArrayListWithExpectedSize(attributes.size());
for (AttributeKvEntry attribute : attributes) {
futures.add(attributesDao.save(tenantId, entityId, scope, attribute));
}
return Futures.allAsList(futures);
List<ListenableFuture<Void>> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, scope, attribute)).collect(Collectors.toList());
return Futures.allAsList(saveFutures);
}
@Override
public ListenableFuture<List<Void>> removeAll(TenantId tenantId, EntityId entityId, String scope, List<String> keys) {
public ListenableFuture<List<Void>> removeAll(TenantId tenantId, EntityId entityId, String scope, List<String> attributeKeys) {
validate(entityId, scope);
return attributesDao.removeAll(tenantId, entityId, scope, keys);
}
private static void validate(EntityId id, String scope) {
Validator.validateId(id.getId(), "Incorrect id " + id);
Validator.validateString(scope, "Incorrect scope " + scope);
}
private static void validate(AttributeKvEntry kvEntry) {
if (kvEntry == null) {
throw new IncorrectParameterException("Key value entry can't be null");
} else if (kvEntry.getDataType() == null) {
throw new IncorrectParameterException("Incorrect kvEntry. Data type can't be null");
} else {
Validator.validateString(kvEntry.getKey(), "Incorrect kvEntry. Key can't be empty");
Validator.validatePositiveNumber(kvEntry.getLastUpdateTs(), "Incorrect last update ts. Ts should be positive");
}
return attributesDao.removeAll(tenantId, entityId, scope, attributeKeys);
}
}

193
dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java

@ -0,0 +1,193 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.attributes;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.stats.DefaultCounter;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.dao.service.Validator;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import static org.thingsboard.server.common.data.CacheConstants.ATTRIBUTES_CACHE;
import static org.thingsboard.server.dao.attributes.AttributeUtils.validate;
@Service
@ConditionalOnProperty(prefix = "cache.attributes", value = "enabled", havingValue = "true")
@Primary
@Slf4j
public class CachedAttributesService implements AttributesService {
private static final String STATS_NAME = "attributes.cache";
private final AttributesDao attributesDao;
private final AttributesCacheWrapper cacheWrapper;
private final DefaultCounter hitCounter;
private final DefaultCounter missCounter;
public CachedAttributesService(AttributesDao attributesDao,
AttributesCacheWrapper cacheWrapper,
StatsFactory statsFactory) {
this.attributesDao = attributesDao;
this.cacheWrapper = cacheWrapper;
this.hitCounter = statsFactory.createDefaultCounter(STATS_NAME, "result", "hit");
this.missCounter = statsFactory.createDefaultCounter(STATS_NAME, "result", "miss");
}
@Override
public ListenableFuture<Optional<AttributeKvEntry>> find(TenantId tenantId, EntityId entityId, String scope, String attributeKey) {
validate(entityId, scope);
Validator.validateString(attributeKey, "Incorrect attribute key " + attributeKey);
AttributeCacheKey attributeCacheKey = new AttributeCacheKey(scope, entityId, attributeKey);
Cache.ValueWrapper cachedAttributeValue = cacheWrapper.get(attributeCacheKey);
if (cachedAttributeValue != null) {
hitCounter.increment();
AttributeKvEntry cachedAttributeKvEntry = (AttributeKvEntry) cachedAttributeValue.get();
return Futures.immediateFuture(Optional.ofNullable(cachedAttributeKvEntry));
} else {
missCounter.increment();
ListenableFuture<Optional<AttributeKvEntry>> result = attributesDao.find(tenantId, entityId, scope, attributeKey);
return Futures.transform(result, foundAttrKvEntry -> {
// TODO: think if it's a good idea to store 'empty' attributes
cacheWrapper.put(attributeCacheKey, foundAttrKvEntry.orElse(null));
return foundAttrKvEntry;
}, MoreExecutors.directExecutor());
}
}
@Override
public ListenableFuture<List<AttributeKvEntry>> find(TenantId tenantId, EntityId entityId, String scope, Collection<String> attributeKeys) {
validate(entityId, scope);
attributeKeys.forEach(attributeKey -> Validator.validateString(attributeKey, "Incorrect attribute key " + attributeKey));
Map<String, Cache.ValueWrapper> wrappedCachedAttributes = findCachedAttributes(entityId, scope, attributeKeys);
List<AttributeKvEntry> cachedAttributes = wrappedCachedAttributes.values().stream()
.map(wrappedCachedAttribute -> (AttributeKvEntry) wrappedCachedAttribute.get())
.filter(Objects::nonNull)
.collect(Collectors.toList());
if (wrappedCachedAttributes.size() == attributeKeys.size()) {
return Futures.immediateFuture(cachedAttributes);
}
Set<String> notFoundAttributeKeys = new HashSet<>(attributeKeys);
notFoundAttributeKeys.removeAll(wrappedCachedAttributes.keySet());
ListenableFuture<List<AttributeKvEntry>> result = attributesDao.find(tenantId, entityId, scope, notFoundAttributeKeys);
return Futures.transform(result, foundInDbAttributes -> mergeDbAndCacheAttributes(entityId, scope, cachedAttributes, notFoundAttributeKeys, foundInDbAttributes), MoreExecutors.directExecutor());
}
private Map<String, Cache.ValueWrapper> findCachedAttributes(EntityId entityId, String scope, Collection<String> attributeKeys) {
Map<String, Cache.ValueWrapper> cachedAttributes = new HashMap<>();
for (String attributeKey : attributeKeys) {
Cache.ValueWrapper cachedAttributeValue = cacheWrapper.get(new AttributeCacheKey(scope, entityId, attributeKey));
if (cachedAttributeValue != null) {
hitCounter.increment();
cachedAttributes.put(attributeKey, cachedAttributeValue);
} else {
missCounter.increment();
}
}
return cachedAttributes;
}
private List<AttributeKvEntry> mergeDbAndCacheAttributes(EntityId entityId, String scope, List<AttributeKvEntry> cachedAttributes, Set<String> notFoundAttributeKeys, List<AttributeKvEntry> foundInDbAttributes) {
for (AttributeKvEntry foundInDbAttribute : foundInDbAttributes) {
AttributeCacheKey attributeCacheKey = new AttributeCacheKey(scope, entityId, foundInDbAttribute.getKey());
cacheWrapper.put(attributeCacheKey, foundInDbAttribute);
notFoundAttributeKeys.remove(foundInDbAttribute.getKey());
}
for (String key : notFoundAttributeKeys){
cacheWrapper.put(new AttributeCacheKey(scope, entityId, key), null);
}
List<AttributeKvEntry> mergedAttributes = new ArrayList<>(cachedAttributes);
mergedAttributes.addAll(foundInDbAttributes);
return mergedAttributes;
}
@Override
public ListenableFuture<List<AttributeKvEntry>> findAll(TenantId tenantId, EntityId entityId, String scope) {
validate(entityId, scope);
return attributesDao.findAll(tenantId, entityId, scope);
}
@Override
public List<String> findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId) {
return attributesDao.findAllKeysByDeviceProfileId(tenantId, deviceProfileId);
}
@Override
public List<String> findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List<EntityId> entityIds) {
return attributesDao.findAllKeysByEntityIds(tenantId, entityType, entityIds);
}
@Override
public ListenableFuture<List<Void>> save(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes) {
validate(entityId, scope);
attributes.forEach(AttributeUtils::validate);
List<ListenableFuture<Void>> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, scope, attribute)).collect(Collectors.toList());
ListenableFuture<List<Void>> future = Futures.allAsList(saveFutures);
// TODO: can do if (attributesCache.get() != null) attributesCache.put() instead, but will be more twice more requests to cache
List<String> attributeKeys = attributes.stream().map(KvEntry::getKey).collect(Collectors.toList());
future.addListener(() -> evictAttributesFromCache(tenantId, entityId, scope, attributeKeys), MoreExecutors.directExecutor());
return future;
}
@Override
public ListenableFuture<List<Void>> removeAll(TenantId tenantId, EntityId entityId, String scope, List<String> attributeKeys) {
validate(entityId, scope);
ListenableFuture<List<Void>> future = attributesDao.removeAll(tenantId, entityId, scope, attributeKeys);
future.addListener(() -> evictAttributesFromCache(tenantId, entityId, scope, attributeKeys), MoreExecutors.directExecutor());
return future;
}
private void evictAttributesFromCache(TenantId tenantId, EntityId entityId, String scope, List<String> attributeKeys) {
try {
for (String attributeKey : attributeKeys) {
cacheWrapper.evict(new AttributeCacheKey(scope, entityId, attributeKey));
}
} catch (Exception e) {
log.error("[{}][{}] Failed to remove values from cache.", tenantId, entityId, e);
}
}
}

6
rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java

@ -2188,7 +2188,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable {
Map<String, String> params = new HashMap<>();
addPageLinkToParam(params, pageLink);
return restTemplate.exchange(
baseURL + "/api/tenantProfiles" + getUrlParams(pageLink),
baseURL + "/api/tenantProfiles?" + getUrlParams(pageLink),
HttpMethod.GET,
HttpEntity.EMPTY,
new ParameterizedTypeReference<PageData<TenantProfile>>() {
@ -2199,7 +2199,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable {
Map<String, String> params = new HashMap<>();
addPageLinkToParam(params, pageLink);
return restTemplate.exchange(
baseURL + "/api/tenantProfileInfos" + getUrlParams(pageLink),
baseURL + "/api/tenantProfileInfos?" + getUrlParams(pageLink),
HttpMethod.GET,
HttpEntity.EMPTY,
new ParameterizedTypeReference<PageData<EntityInfo>>() {
@ -2256,7 +2256,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable {
Map<String, String> params = new HashMap<>();
addPageLinkToParam(params, pageLink);
return restTemplate.exchange(
baseURL + "/api/users" + getUrlParams(pageLink),
baseURL + "/api/users?" + getUrlParams(pageLink),
HttpMethod.GET,
HttpEntity.EMPTY,
new ParameterizedTypeReference<PageData<User>>() {

2
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java

@ -56,7 +56,7 @@ import java.security.spec.PKCS8EncodedKeySpec;
public class CertPemCredentials implements ClientCredentials {
private static final String TLS_VERSION = "TLSv1.2";
private String caCert;
protected String caCert;
private String cert;
private String privateKey;
private String password;

3
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java

@ -104,9 +104,10 @@ public class TbMsgGeneratorNode implements TbNode {
}
},
t -> {
if (initialized) {
if (initialized && (config.getMsgCount() == TbMsgGeneratorNodeConfiguration.UNLIMITED_MSG_COUNT || currentMsgCount < config.getMsgCount())) {
ctx.tellFailure(msg, t);
scheduleTickMsg(ctx);
currentMsgCount++;
}
});
}

19
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/CalculateDeltaNode.java

@ -45,11 +45,13 @@ import java.util.concurrent.ConcurrentHashMap;
@Slf4j
@RuleNode(type = ComponentType.ENRICHMENT,
name = "calculate delta",
name = "calculate delta", relationTypes = {"Success", "Failure", "Other"},
configClazz = CalculateDeltaNodeConfiguration.class,
nodeDescription = "Calculates and adds 'delta' value into message based on the incoming and previous value",
nodeDetails = "Calculates delta and period based on the previous time-series reading and current data. " +
"Delta calculation is done in scope of the message originator, e.g. device, asset or customer.",
"Delta calculation is done in scope of the message originator, e.g. device, asset or customer. " +
"If there is input key, the output relation will be 'Success' unless delta is negative and corresponding configuration parameter is set. " +
"If there is no input value key in the incoming message, the output relation will be 'Other'.",
uiResources = {"static/rulenode/rulenode-core-config.js"},
configDirective = "tbEnrichmentNodeCalculateDeltaConfig")
public class CalculateDeltaNode implements TbNode {
@ -93,12 +95,17 @@ public class CalculateDeltaNode implements TbNode {
return;
}
if (config.getRound() != null) {
delta = delta.setScale(config.getRound(), RoundingMode.HALF_UP);
}
ObjectNode result = (ObjectNode) json;
result.put(config.getOutputValueKey(), delta);
if (delta.stripTrailingZeros().scale() > 0) {
result.put(config.getOutputValueKey(), delta.doubleValue());
} else {
result.put(config.getOutputValueKey(), delta.longValueExact());
}
if (config.isAddPeriodBetweenMsgs()) {
long period = previousData != null ? currentTs - previousData.ts : 0;
@ -107,13 +114,11 @@ public class CalculateDeltaNode implements TbNode {
ctx.tellSuccess(TbMsg.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), JacksonUtil.toString(result)));
},
t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor());
} else if (config.isTellFailureIfInputValueKeyIsAbsent()) {
ctx.tellNext(msg, TbRelationTypes.FAILURE);
} else {
ctx.tellSuccess(msg);
ctx.tellNext(msg, "Other");
}
} else {
ctx.tellSuccess(msg);
ctx.tellNext(msg, "Other");
}
}

4
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/CalculateDeltaNodeConfiguration.java

@ -15,10 +15,12 @@
*/
package org.thingsboard.rule.engine.metadata;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;
import org.thingsboard.rule.engine.api.NodeConfiguration;
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
public class CalculateDeltaNodeConfiguration implements NodeConfiguration<CalculateDeltaNodeConfiguration> {
private String inputValueKey;
private String outputValueKey;
@ -26,7 +28,6 @@ public class CalculateDeltaNodeConfiguration implements NodeConfiguration<Calcul
private boolean addPeriodBetweenMsgs;
private String periodValueKey;
private Integer round;
private boolean tellFailureIfInputValueKeyIsAbsent;
private boolean tellFailureIfDeltaIsNegative;
@Override
@ -37,7 +38,6 @@ public class CalculateDeltaNodeConfiguration implements NodeConfiguration<Calcul
configuration.setUseCache(true);
configuration.setAddPeriodBetweenMsgs(false);
configuration.setPeriodValueKey("periodInMs");
configuration.setTellFailureIfInputValueKeyIsAbsent(true);
configuration.setTellFailureIfDeltaIsNegative(true);
return configuration;
}

3
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/azure/AzureIotHubSasCredentials.java

@ -40,8 +40,7 @@ import java.security.cert.X509Certificate;
@JsonIgnoreProperties(ignoreUnknown = true)
public class AzureIotHubSasCredentials extends CertPemCredentials {
private String sasKey;
private String caCert;
@Override
public SslContext initSslContext() {
try {

84
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java

@ -16,9 +16,13 @@
package org.thingsboard.rule.engine.profile;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.rule.engine.profile.state.PersistedAlarmRuleState;
import org.thingsboard.server.common.data.alarm.AlarmSeverity;
import org.thingsboard.server.common.data.device.profile.AlarmCondition;
import org.thingsboard.server.common.data.device.profile.AlarmConditionFilter;
import org.thingsboard.server.common.data.device.profile.AlarmConditionFilterKey;
import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType;
import org.thingsboard.server.common.data.device.profile.AlarmConditionSpec;
import org.thingsboard.server.common.data.device.profile.AlarmRule;
import org.thingsboard.server.common.data.device.profile.CustomTimeSchedule;
@ -45,6 +49,7 @@ import java.util.Set;
import java.util.function.Function;
@Data
@Slf4j
class AlarmRuleState {
private final AlarmSeverity severity;
@ -52,12 +57,12 @@ class AlarmRuleState {
private final AlarmConditionSpec spec;
private final long requiredDurationInMs;
private final long requiredRepeats;
private final Set<EntityKey> entityKeys;
private final Set<AlarmConditionFilterKey> entityKeys;
private PersistedAlarmRuleState state;
private boolean updateFlag;
private final DynamicPredicateValueCtx dynamicPredicateValueCtx;
AlarmRuleState(AlarmSeverity severity, AlarmRule alarmRule, Set<EntityKey> entityKeys, PersistedAlarmRuleState state, DynamicPredicateValueCtx dynamicPredicateValueCtx) {
AlarmRuleState(AlarmSeverity severity, AlarmRule alarmRule, Set<AlarmConditionFilterKey> entityKeys, PersistedAlarmRuleState state, DynamicPredicateValueCtx dynamicPredicateValueCtx) {
this.severity = severity;
this.alarmRule = alarmRule;
this.entityKeys = entityKeys;
@ -84,8 +89,8 @@ class AlarmRuleState {
this.dynamicPredicateValueCtx = dynamicPredicateValueCtx;
}
public boolean validateTsUpdate(Set<EntityKey> changedKeys) {
for (EntityKey key : changedKeys) {
public boolean validateTsUpdate(Set<AlarmConditionFilterKey> changedKeys) {
for (AlarmConditionFilterKey key : changedKeys) {
if (entityKeys.contains(key)) {
return true;
}
@ -93,18 +98,14 @@ class AlarmRuleState {
return false;
}
public boolean validateAttrUpdate(Set<EntityKey> changedKeys) {
public boolean validateAttrUpdate(Set<AlarmConditionFilterKey> changedKeys) {
//If the attribute was updated, but no new telemetry arrived - we ignore this until new telemetry is there.
for (EntityKey key : entityKeys) {
if (key.getType().equals(EntityKeyType.TIME_SERIES)) {
for (AlarmConditionFilterKey key : entityKeys) {
if (key.getType().equals(AlarmConditionKeyType.TIME_SERIES)) {
return false;
}
}
for (EntityKey key : changedKeys) {
EntityKeyType keyType = key.getType();
if (EntityKeyType.CLIENT_ATTRIBUTE.equals(keyType) || EntityKeyType.SERVER_ATTRIBUTE.equals(keyType) || EntityKeyType.SHARED_ATTRIBUTE.equals(keyType)) {
key = new EntityKey(EntityKeyType.ATTRIBUTE, key.getKey());
}
for (AlarmConditionFilterKey key : changedKeys) {
if (entityKeys.contains(key)) {
return true;
}
@ -259,16 +260,46 @@ class AlarmRuleState {
private boolean eval(AlarmCondition condition, DataSnapshot data) {
boolean eval = true;
for (KeyFilter keyFilter : condition.getCondition()) {
EntityKeyValue value = data.getValue(keyFilter.getKey());
for (var filter : condition.getCondition()) {
EntityKeyValue value;
if (filter.getKey().getType().equals(AlarmConditionKeyType.CONSTANT)) {
try {
value = getConstantValue(filter);
} catch (RuntimeException e) {
log.warn("Failed to parse constant value from filter: {}", filter, e);
value = null;
}
} else {
value = data.getValue(filter.getKey());
}
if (value == null) {
return false;
}
eval = eval && eval(data, value, keyFilter.getPredicate());
eval = eval && eval(data, value, filter.getPredicate());
}
return eval;
}
private EntityKeyValue getConstantValue(AlarmConditionFilter filter) {
EntityKeyValue value = new EntityKeyValue();
String valueStr = filter.getValue().toString();
switch (filter.getValueType()) {
case STRING:
value.setStrValue(valueStr);
break;
case DATE_TIME:
value.setLngValue(Long.valueOf(valueStr));
break;
case NUMERIC:
value.setDblValue(Double.valueOf(valueStr));
break;
case BOOLEAN:
value.setBoolValue(Boolean.valueOf(valueStr));
break;
}
return value;
}
private boolean eval(DataSnapshot data, EntityKeyValue value, KeyFilterPredicate predicate) {
switch (predicate.getType()) {
case STRING:
@ -388,23 +419,18 @@ class AlarmRuleState {
EntityKeyValue ekv = null;
if (value.getDynamicValue() != null) {
switch (value.getDynamicValue().getSourceType()) {
case CURRENT_TENANT:
ekv = dynamicPredicateValueCtx.getTenantValue(value.getDynamicValue().getSourceAttribute());
break;
case CURRENT_DEVICE:
ekv = data.getValue(new AlarmConditionFilterKey(AlarmConditionKeyType.ATTRIBUTE, value.getDynamicValue().getSourceAttribute()));
if (ekv != null || !value.getDynamicValue().isInherit()) {
break;
}
case CURRENT_CUSTOMER:
ekv = dynamicPredicateValueCtx.getCustomerValue(value.getDynamicValue().getSourceAttribute());
break;
case CURRENT_DEVICE:
ekv = data.getValue(new EntityKey(EntityKeyType.ATTRIBUTE, value.getDynamicValue().getSourceAttribute()));
if (ekv == null) {
ekv = data.getValue(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, value.getDynamicValue().getSourceAttribute()));
if (ekv == null) {
ekv = data.getValue(new EntityKey(EntityKeyType.SHARED_ATTRIBUTE, value.getDynamicValue().getSourceAttribute()));
if (ekv == null) {
ekv = data.getValue(new EntityKey(EntityKeyType.CLIENT_ATTRIBUTE, value.getDynamicValue().getSourceAttribute()));
}
}
if (ekv != null || !value.getDynamicValue().isInherit()) {
break;
}
case CURRENT_TENANT:
ekv = dynamicPredicateValueCtx.getTenantValue(value.getDynamicValue().getSourceAttribute());
}
}
return ekv;

7
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java

@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.alarm.Alarm;
import org.thingsboard.server.common.data.alarm.AlarmSeverity;
import org.thingsboard.server.common.data.alarm.AlarmStatus;
import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType;
import org.thingsboard.server.common.data.device.profile.DeviceProfileAlarm;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.EntityId;
@ -138,9 +139,9 @@ class AlarmState {
public boolean validateUpdate(SnapshotUpdate update, AlarmRuleState state) {
if (update != null) {
//Check that the update type and that keys match.
if (update.getType().equals(EntityKeyType.TIME_SERIES)) {
if (update.getType().equals(AlarmConditionKeyType.TIME_SERIES)) {
return state.validateTsUpdate(update.getKeys());
} else if (update.getType().equals(EntityKeyType.ATTRIBUTE)) {
} else if (update.getType().equals(AlarmConditionKeyType.ATTRIBUTE)) {
return state.validateAttrUpdate(update.getKeys());
}
}
@ -252,7 +253,7 @@ class AlarmState {
String alarmDetailsStr = ruleState.getAlarmRule().getAlarmDetails();
if (StringUtils.isNotEmpty(alarmDetailsStr)) {
for (KeyFilter keyFilter : ruleState.getAlarmRule().getCondition().getCondition()) {
for (var keyFilter : ruleState.getAlarmRule().getCondition().getCondition()) {
EntityKeyValue entityKeyValue = dataSnapshot.getValue(keyFilter.getKey());
alarmDetailsStr = alarmDetailsStr.replaceAll(String.format("\\$\\{%s}", keyFilter.getKey().getKey()), getValueAsString(entityKeyValue));
}

85
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DataSnapshot.java

@ -17,6 +17,8 @@ package org.thingsboard.rule.engine.profile;
import lombok.Getter;
import lombok.Setter;
import org.thingsboard.server.common.data.device.profile.AlarmConditionFilterKey;
import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType;
import org.thingsboard.server.common.data.query.EntityKey;
import org.thingsboard.server.common.data.query.EntityKeyType;
@ -30,55 +32,42 @@ class DataSnapshot {
@Getter
@Setter
private long ts;
private final Set<EntityKey> keys;
private final Map<EntityKey, EntityKeyValue> values = new ConcurrentHashMap<>();
private final Set<AlarmConditionFilterKey> keys;
private final Map<AlarmConditionFilterKey, EntityKeyValue> values = new ConcurrentHashMap<>();
DataSnapshot(Set<EntityKey> entityKeysToFetch) {
DataSnapshot(Set<AlarmConditionFilterKey> entityKeysToFetch) {
this.keys = entityKeysToFetch;
}
void removeValue(EntityKey key) {
switch (key.getType()) {
case ATTRIBUTE:
values.remove(key);
values.remove(getAttrKey(key, EntityKeyType.CLIENT_ATTRIBUTE));
values.remove(getAttrKey(key, EntityKeyType.SHARED_ATTRIBUTE));
values.remove(getAttrKey(key, EntityKeyType.SERVER_ATTRIBUTE));
break;
case CLIENT_ATTRIBUTE:
case SHARED_ATTRIBUTE:
case SERVER_ATTRIBUTE:
values.remove(key);
values.remove(getAttrKey(key, EntityKeyType.ATTRIBUTE));
break;
default:
values.remove(key);
}
static AlarmConditionFilterKey toConditionKey(EntityKey key) {
return new AlarmConditionFilterKey(toConditionKeyType(key.getType()), key.getKey());
}
boolean putValue(EntityKey key, long newTs, EntityKeyValue value) {
boolean updateOfTs = ts != newTs;
boolean result = false;
switch (key.getType()) {
static AlarmConditionKeyType toConditionKeyType(EntityKeyType keyType) {
switch (keyType) {
case ATTRIBUTE:
result |= putIfKeyExists(key, value, updateOfTs);
result |= putIfKeyExists(getAttrKey(key, EntityKeyType.CLIENT_ATTRIBUTE), value, updateOfTs);
result |= putIfKeyExists(getAttrKey(key, EntityKeyType.SHARED_ATTRIBUTE), value, updateOfTs);
result |= putIfKeyExists(getAttrKey(key, EntityKeyType.SERVER_ATTRIBUTE), value, updateOfTs);
break;
case CLIENT_ATTRIBUTE:
case SHARED_ATTRIBUTE:
case SERVER_ATTRIBUTE:
result |= putIfKeyExists(key, value, updateOfTs);
result |= putIfKeyExists(getAttrKey(key, EntityKeyType.ATTRIBUTE), value, updateOfTs);
break;
case SHARED_ATTRIBUTE:
case CLIENT_ATTRIBUTE:
return AlarmConditionKeyType.ATTRIBUTE;
case TIME_SERIES:
return AlarmConditionKeyType.TIME_SERIES;
case ENTITY_FIELD:
return AlarmConditionKeyType.ENTITY_FIELD;
default:
result |= putIfKeyExists(key, value, updateOfTs);
throw new RuntimeException("Not supported entity key: " + keyType.name());
}
return result;
}
private boolean putIfKeyExists(EntityKey key, EntityKeyValue value, boolean updateOfTs) {
void removeValue(EntityKey key) {
values.remove(toConditionKey(key));
}
boolean putValue(AlarmConditionFilterKey key, long newTs, EntityKeyValue value) {
return putIfKeyExists(key, value, ts != newTs);
}
private boolean putIfKeyExists(AlarmConditionFilterKey key, EntityKeyValue value, boolean updateOfTs) {
if (keys.contains(key)) {
EntityKeyValue oldValue = values.put(key, value);
if (updateOfTs) {
@ -91,25 +80,7 @@ class DataSnapshot {
}
}
EntityKeyValue getValue(EntityKey key) {
if (EntityKeyType.ATTRIBUTE.equals(key.getType())) {
EntityKeyValue value = values.get(key);
if (value == null) {
value = values.get(getAttrKey(key, EntityKeyType.CLIENT_ATTRIBUTE));
if (value == null) {
value = values.get(getAttrKey(key, EntityKeyType.SHARED_ATTRIBUTE));
if (value == null) {
value = values.get(getAttrKey(key, EntityKeyType.SERVER_ATTRIBUTE));
}
}
}
return value;
} else {
return values.get(key);
}
}
private EntityKey getAttrKey(EntityKey key, EntityKeyType clientAttribute) {
return new EntityKey(clientAttribute, key.getKey());
EntityKeyValue getValue(AlarmConditionFilterKey key) {
return values.get(key);
}
}

67
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java

@ -26,6 +26,8 @@ import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.alarm.Alarm;
import org.thingsboard.server.common.data.device.profile.AlarmConditionFilterKey;
import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType;
import org.thingsboard.server.common.data.device.profile.DeviceProfileAlarm;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.DeviceProfileId;
@ -97,10 +99,10 @@ class DeviceState {
}
public void updateProfile(TbContext ctx, DeviceProfile deviceProfile) throws ExecutionException, InterruptedException {
Set<EntityKey> oldKeys = this.deviceProfile.getEntityKeys();
Set<AlarmConditionFilterKey> oldKeys = this.deviceProfile.getEntityKeys();
this.deviceProfile.updateDeviceProfile(deviceProfile);
if (latestValues != null) {
Set<EntityKey> keysToFetch = new HashSet<>(this.deviceProfile.getEntityKeys());
Set<AlarmConditionFilterKey> keysToFetch = new HashSet<>(this.deviceProfile.getEntityKeys());
keysToFetch.removeAll(oldKeys);
if (!keysToFetch.isEmpty()) {
addEntityKeysToSnapshot(ctx, deviceId, keysToFetch, latestValues);
@ -260,29 +262,29 @@ class DeviceState {
}
private SnapshotUpdate merge(DataSnapshot latestValues, Long newTs, List<KvEntry> data) {
Set<EntityKey> keys = new HashSet<>();
Set<AlarmConditionFilterKey> keys = new HashSet<>();
for (KvEntry entry : data) {
EntityKey entityKey = new EntityKey(EntityKeyType.TIME_SERIES, entry.getKey());
AlarmConditionFilterKey entityKey = new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, entry.getKey());
if (latestValues.putValue(entityKey, newTs, toEntityValue(entry))) {
keys.add(entityKey);
}
}
latestValues.setTs(newTs);
return new SnapshotUpdate(EntityKeyType.TIME_SERIES, keys);
return new SnapshotUpdate(AlarmConditionKeyType.TIME_SERIES, keys);
}
private SnapshotUpdate merge(DataSnapshot latestValues, Set<AttributeKvEntry> attributes, String scope) {
long newTs = 0;
Set<EntityKey> keys = new HashSet<>();
Set<AlarmConditionFilterKey> keys = new HashSet<>();
for (AttributeKvEntry entry : attributes) {
newTs = Math.max(newTs, entry.getLastUpdateTs());
EntityKey entityKey = new EntityKey(getKeyTypeFromScope(scope), entry.getKey());
AlarmConditionFilterKey entityKey = new AlarmConditionFilterKey(AlarmConditionKeyType.ATTRIBUTE, entry.getKey());
if (latestValues.putValue(entityKey, newTs, toEntityValue(entry))) {
keys.add(entityKey);
}
}
latestValues.setTs(newTs);
return new SnapshotUpdate(EntityKeyType.ATTRIBUTE, keys);
return new SnapshotUpdate(AlarmConditionKeyType.ATTRIBUTE, keys);
}
private static EntityKeyType getKeyTypeFromScope(String scope) {
@ -298,37 +300,22 @@ class DeviceState {
}
private DataSnapshot fetchLatestValues(TbContext ctx, EntityId originator) throws ExecutionException, InterruptedException {
Set<EntityKey> entityKeysToFetch = deviceProfile.getEntityKeys();
Set<AlarmConditionFilterKey> entityKeysToFetch = deviceProfile.getEntityKeys();
DataSnapshot result = new DataSnapshot(entityKeysToFetch);
addEntityKeysToSnapshot(ctx, originator, entityKeysToFetch, result);
return result;
}
private void addEntityKeysToSnapshot(TbContext ctx, EntityId originator, Set<EntityKey> entityKeysToFetch, DataSnapshot result) throws InterruptedException, ExecutionException {
Set<String> serverAttributeKeys = new HashSet<>();
Set<String> clientAttributeKeys = new HashSet<>();
Set<String> sharedAttributeKeys = new HashSet<>();
Set<String> commonAttributeKeys = new HashSet<>();
private void addEntityKeysToSnapshot(TbContext ctx, EntityId originator, Set<AlarmConditionFilterKey> entityKeysToFetch, DataSnapshot result) throws InterruptedException, ExecutionException {
Set<String> attributeKeys = new HashSet<>();
Set<String> latestTsKeys = new HashSet<>();
Device device = null;
for (EntityKey entityKey : entityKeysToFetch) {
for (AlarmConditionFilterKey entityKey : entityKeysToFetch) {
String key = entityKey.getKey();
switch (entityKey.getType()) {
case SERVER_ATTRIBUTE:
serverAttributeKeys.add(key);
break;
case CLIENT_ATTRIBUTE:
clientAttributeKeys.add(key);
break;
case SHARED_ATTRIBUTE:
sharedAttributeKeys.add(key);
break;
case ATTRIBUTE:
serverAttributeKeys.add(key);
clientAttributeKeys.add(key);
sharedAttributeKeys.add(key);
commonAttributeKeys.add(key);
attributeKeys.add(key);
break;
case TIME_SERIES:
latestTsKeys.add(key);
@ -361,32 +348,22 @@ class DeviceState {
List<TsKvEntry> data = ctx.getTimeseriesService().findLatest(ctx.getTenantId(), originator, latestTsKeys).get();
for (TsKvEntry entry : data) {
if (entry.getValue() != null) {
result.putValue(new EntityKey(EntityKeyType.TIME_SERIES, entry.getKey()), entry.getTs(), toEntityValue(entry));
result.putValue(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, entry.getKey()), entry.getTs(), toEntityValue(entry));
}
}
}
if (!clientAttributeKeys.isEmpty()) {
addToSnapshot(result, commonAttributeKeys,
ctx.getAttributesService().find(ctx.getTenantId(), originator, DataConstants.CLIENT_SCOPE, clientAttributeKeys).get());
}
if (!sharedAttributeKeys.isEmpty()) {
addToSnapshot(result, commonAttributeKeys,
ctx.getAttributesService().find(ctx.getTenantId(), originator, DataConstants.SHARED_SCOPE, sharedAttributeKeys).get());
}
if (!serverAttributeKeys.isEmpty()) {
addToSnapshot(result, commonAttributeKeys,
ctx.getAttributesService().find(ctx.getTenantId(), originator, DataConstants.SERVER_SCOPE, serverAttributeKeys).get());
if (!attributeKeys.isEmpty()) {
addToSnapshot(result, ctx.getAttributesService().find(ctx.getTenantId(), originator, DataConstants.CLIENT_SCOPE, attributeKeys).get());
addToSnapshot(result, ctx.getAttributesService().find(ctx.getTenantId(), originator, DataConstants.SHARED_SCOPE, attributeKeys).get());
addToSnapshot(result, ctx.getAttributesService().find(ctx.getTenantId(), originator, DataConstants.SERVER_SCOPE, attributeKeys).get());
}
}
private void addToSnapshot(DataSnapshot snapshot, Set<String> commonAttributeKeys, List<AttributeKvEntry> data) {
private void addToSnapshot(DataSnapshot snapshot, List<AttributeKvEntry> data) {
for (AttributeKvEntry entry : data) {
if (entry.getValue() != null) {
EntityKeyValue value = toEntityValue(entry);
snapshot.putValue(new EntityKey(EntityKeyType.CLIENT_ATTRIBUTE, entry.getKey()), entry.getLastUpdateTs(), value);
if (commonAttributeKeys.contains(entry.getKey())) {
snapshot.putValue(new EntityKey(EntityKeyType.ATTRIBUTE, entry.getKey()), entry.getLastUpdateTs(), value);
}
snapshot.putValue(new AlarmConditionFilterKey(AlarmConditionKeyType.ATTRIBUTE, entry.getKey()), entry.getLastUpdateTs(), value);
}
}
}

33
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java

@ -19,6 +19,9 @@ import lombok.AccessLevel;
import lombok.Getter;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.alarm.AlarmSeverity;
import org.thingsboard.server.common.data.device.profile.AlarmConditionFilter;
import org.thingsboard.server.common.data.device.profile.AlarmConditionFilterKey;
import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType;
import org.thingsboard.server.common.data.device.profile.AlarmRule;
import org.thingsboard.server.common.data.device.profile.DeviceProfileAlarm;
import org.thingsboard.server.common.data.id.DeviceProfileId;
@ -50,10 +53,10 @@ class ProfileState {
@Getter(AccessLevel.PACKAGE)
private final List<DeviceProfileAlarm> alarmSettings = new CopyOnWriteArrayList<>();
@Getter(AccessLevel.PACKAGE)
private final Set<EntityKey> entityKeys = ConcurrentHashMap.newKeySet();
private final Set<AlarmConditionFilterKey> entityKeys = ConcurrentHashMap.newKeySet();
private final Map<String, Map<AlarmSeverity, Set<EntityKey>>> alarmCreateKeys = new HashMap<>();
private final Map<String, Set<EntityKey>> alarmClearKeys = new HashMap<>();
private final Map<String, Map<AlarmSeverity, Set<AlarmConditionFilterKey>>> alarmCreateKeys = new HashMap<>();
private final Map<String, Set<AlarmConditionFilterKey>> alarmClearKeys = new HashMap<>();
ProfileState(DeviceProfile deviceProfile) {
updateDeviceProfile(deviceProfile);
@ -68,18 +71,18 @@ class ProfileState {
if (deviceProfile.getProfileData().getAlarms() != null) {
alarmSettings.addAll(deviceProfile.getProfileData().getAlarms());
for (DeviceProfileAlarm alarm : deviceProfile.getProfileData().getAlarms()) {
Map<AlarmSeverity, Set<EntityKey>> createAlarmKeys = alarmCreateKeys.computeIfAbsent(alarm.getId(), id -> new HashMap<>());
Map<AlarmSeverity, Set<AlarmConditionFilterKey>> createAlarmKeys = alarmCreateKeys.computeIfAbsent(alarm.getId(), id -> new HashMap<>());
alarm.getCreateRules().forEach(((severity, alarmRule) -> {
Set<EntityKey> ruleKeys = createAlarmKeys.computeIfAbsent(severity, id -> new HashSet<>());
for (KeyFilter keyFilter : alarmRule.getCondition().getCondition()) {
var ruleKeys = createAlarmKeys.computeIfAbsent(severity, id -> new HashSet<>());
for (var keyFilter : alarmRule.getCondition().getCondition()) {
entityKeys.add(keyFilter.getKey());
ruleKeys.add(keyFilter.getKey());
addDynamicValuesRecursively(keyFilter.getPredicate(), entityKeys, ruleKeys);
}
}));
if (alarm.getClearRule() != null) {
Set<EntityKey> clearAlarmKeys = alarmClearKeys.computeIfAbsent(alarm.getId(), id -> new HashSet<>());
for (KeyFilter keyFilter : alarm.getClearRule().getCondition().getCondition()) {
var clearAlarmKeys = alarmClearKeys.computeIfAbsent(alarm.getId(), id -> new HashSet<>());
for (var keyFilter : alarm.getClearRule().getCondition().getCondition()) {
entityKeys.add(keyFilter.getKey());
clearAlarmKeys.add(keyFilter.getKey());
addDynamicValuesRecursively(keyFilter.getPredicate(), entityKeys, clearAlarmKeys);
@ -89,7 +92,7 @@ class ProfileState {
}
}
private void addDynamicValuesRecursively(KeyFilterPredicate predicate, Set<EntityKey> entityKeys, Set<EntityKey> ruleKeys) {
private void addDynamicValuesRecursively(KeyFilterPredicate predicate, Set<AlarmConditionFilterKey> entityKeys, Set<AlarmConditionFilterKey> ruleKeys) {
switch (predicate.getType()) {
case STRING:
case NUMERIC:
@ -98,7 +101,7 @@ class ProfileState {
if (value != null && (value.getSourceType() == DynamicValueSourceType.CURRENT_TENANT ||
value.getSourceType() == DynamicValueSourceType.CURRENT_CUSTOMER ||
value.getSourceType() == DynamicValueSourceType.CURRENT_DEVICE)) {
EntityKey entityKey = new EntityKey(EntityKeyType.ATTRIBUTE, value.getSourceAttribute());
AlarmConditionFilterKey entityKey = new AlarmConditionFilterKey(AlarmConditionKeyType.ATTRIBUTE, value.getSourceAttribute());
entityKeys.add(entityKey);
ruleKeys.add(entityKey);
}
@ -115,12 +118,12 @@ class ProfileState {
return deviceProfile.getId();
}
Set<EntityKey> getCreateAlarmKeys(String id, AlarmSeverity severity) {
Map<AlarmSeverity, Set<EntityKey>> sKeys = alarmCreateKeys.get(id);
Set<AlarmConditionFilterKey> getCreateAlarmKeys(String id, AlarmSeverity severity) {
Map<AlarmSeverity, Set<AlarmConditionFilterKey>> sKeys = alarmCreateKeys.get(id);
if (sKeys == null) {
return Collections.emptySet();
} else {
Set<EntityKey> keys = sKeys.get(severity);
Set<AlarmConditionFilterKey> keys = sKeys.get(severity);
if (keys == null) {
return Collections.emptySet();
} else {
@ -129,8 +132,8 @@ class ProfileState {
}
}
Set<EntityKey> getClearAlarmKeys(String id) {
Set<EntityKey> keys = alarmClearKeys.get(id);
Set<AlarmConditionFilterKey> getClearAlarmKeys(String id) {
Set<AlarmConditionFilterKey> keys = alarmClearKeys.get(id);
if (keys == null) {
return Collections.emptySet();
} else {

8
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/SnapshotUpdate.java

@ -16,6 +16,8 @@
package org.thingsboard.rule.engine.profile;
import lombok.Getter;
import org.thingsboard.server.common.data.device.profile.AlarmConditionFilterKey;
import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType;
import org.thingsboard.server.common.data.query.EntityKey;
import org.thingsboard.server.common.data.query.EntityKeyType;
@ -24,11 +26,11 @@ import java.util.Set;
class SnapshotUpdate {
@Getter
private final EntityKeyType type;
private final AlarmConditionKeyType type;
@Getter
private final Set<EntityKey> keys;
private final Set<AlarmConditionFilterKey> keys;
SnapshotUpdate(EntityKeyType type, Set<EntityKey> keys) {
SnapshotUpdate(AlarmConditionKeyType type, Set<AlarmConditionFilterKey> keys) {
this.type = type;
this.keys = keys;
}

344
rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java

@ -36,6 +36,9 @@ import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.alarm.AlarmSeverity;
import org.thingsboard.server.common.data.device.profile.AlarmCondition;
import org.thingsboard.server.common.data.device.profile.AlarmConditionFilter;
import org.thingsboard.server.common.data.device.profile.AlarmConditionFilterKey;
import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType;
import org.thingsboard.server.common.data.device.profile.AlarmRule;
import org.thingsboard.server.common.data.device.profile.DeviceProfileAlarm;
import org.thingsboard.server.common.data.device.profile.DeviceProfileData;
@ -44,6 +47,7 @@ import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.query.BooleanFilterPredicate;
import org.thingsboard.server.common.data.query.DynamicValue;
import org.thingsboard.server.common.data.query.DynamicValueSourceType;
import org.thingsboard.server.common.data.query.EntityKey;
@ -62,6 +66,7 @@ import org.thingsboard.server.dao.model.sql.AttributeKvCompositeKey;
import org.thingsboard.server.dao.model.sql.AttributeKvEntity;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
@ -69,6 +74,7 @@ import java.util.TreeMap;
import java.util.UUID;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
@RunWith(MockitoJUnitRunner.class)
@ -141,8 +147,8 @@ public class TbDeviceProfileNodeTest {
DeviceProfile deviceProfile = new DeviceProfile();
DeviceProfileData deviceProfileData = new DeviceProfileData();
KeyFilter highTempFilter = new KeyFilter();
highTempFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "temperature"));
AlarmConditionFilter highTempFilter = new AlarmConditionFilter();
highTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature"));
highTempFilter.setValueType(EntityKeyValueType.NUMERIC);
NumericFilterPredicate highTemperaturePredicate = new NumericFilterPredicate();
highTemperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER);
@ -157,8 +163,8 @@ public class TbDeviceProfileNodeTest {
dpa.setAlarmType("highTemperatureAlarm");
dpa.setCreateRules(new TreeMap<>(Collections.singletonMap(AlarmSeverity.CRITICAL, alarmRule)));
KeyFilter lowTempFilter = new KeyFilter();
lowTempFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "temperature"));
AlarmConditionFilter lowTempFilter = new AlarmConditionFilter();
lowTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature"));
lowTempFilter.setValueType(EntityKeyValueType.NUMERIC);
NumericFilterPredicate lowTemperaturePredicate = new NumericFilterPredicate();
lowTemperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS);
@ -203,6 +209,175 @@ public class TbDeviceProfileNodeTest {
}
@Test
public void testConstantKeyFilterSimple() throws Exception {
init();
DeviceProfile deviceProfile = new DeviceProfile();
deviceProfile.setId(deviceProfileId);
DeviceProfileData deviceProfileData = new DeviceProfileData();
Device device = new Device();
device.setId(deviceId);
device.setCustomerId(customerId);
AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey(
EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "alarmEnabled"
);
AttributeKvEntity attributeKvEntity = new AttributeKvEntity();
attributeKvEntity.setId(compositeKey);
attributeKvEntity.setBooleanValue(Boolean.TRUE);
attributeKvEntity.setLastUpdateTs(System.currentTimeMillis());
AttributeKvEntry entry = attributeKvEntity.toData();
ListenableFuture<List<AttributeKvEntry>> attrListListenableFuture = Futures.immediateFuture(Collections.singletonList(entry));
AlarmConditionFilter alarmEnabledFilter = new AlarmConditionFilter();
alarmEnabledFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.CONSTANT, "alarmEnabled"));
alarmEnabledFilter.setValue(Boolean.TRUE);
alarmEnabledFilter.setValueType(EntityKeyValueType.BOOLEAN);
BooleanFilterPredicate alarmEnabledPredicate = new BooleanFilterPredicate();
alarmEnabledPredicate.setOperation(BooleanFilterPredicate.BooleanOperation.EQUAL);
alarmEnabledPredicate.setValue(new FilterPredicateValue<>(
Boolean.FALSE,
null,
new DynamicValue<>(DynamicValueSourceType.CURRENT_DEVICE, "alarmEnabled")
));
alarmEnabledFilter.setPredicate(alarmEnabledPredicate);
AlarmConditionFilter temperatureFilter = new AlarmConditionFilter();
temperatureFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature"));
temperatureFilter.setValueType(EntityKeyValueType.NUMERIC);
NumericFilterPredicate temperaturePredicate = new NumericFilterPredicate();
temperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER);
temperaturePredicate.setValue(new FilterPredicateValue<>(20.0, null, null));
temperatureFilter.setPredicate(temperaturePredicate);
AlarmCondition alarmCondition = new AlarmCondition();
alarmCondition.setCondition(Arrays.asList(alarmEnabledFilter, temperatureFilter));
AlarmRule alarmRule = new AlarmRule();
alarmRule.setCondition(alarmCondition);
DeviceProfileAlarm dpa = new DeviceProfileAlarm();
dpa.setId("alarmEnabledAlarmID");
dpa.setAlarmType("alarmEnabledAlarm");
dpa.setCreateRules(new TreeMap<>(Collections.singletonMap(AlarmSeverity.CRITICAL, alarmRule)));
deviceProfileData.setAlarms(Collections.singletonList(dpa));
deviceProfile.setProfileData(deviceProfileData);
Mockito.when(cache.get(tenantId, deviceId)).thenReturn(deviceProfile);
Mockito.when(timeseriesService.findLatest(tenantId, deviceId, Collections.singleton("temperature")))
.thenReturn(Futures.immediateFuture(Collections.emptyList()));
Mockito.when(alarmService.findLatestByOriginatorAndType(tenantId, deviceId, "alarmEnabledAlarm"))
.thenReturn(Futures.immediateFuture(null));
Mockito.when(alarmService.createOrUpdateAlarm(Mockito.any())).thenAnswer(AdditionalAnswers.returnsFirstArg());
Mockito.when(ctx.getAttributesService()).thenReturn(attributesService);
Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet()))
.thenReturn(attrListListenableFuture);
TbMsg theMsg = TbMsg.newMsg("ALARM", deviceId, new TbMsgMetaData(), "");
Mockito.when(ctx.newMsg(Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.anyString()))
.thenReturn(theMsg);
ObjectNode data = mapper.createObjectNode();
data.put("temperature", 21);
TbMsg msg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, new TbMsgMetaData(),
TbMsgDataType.JSON, mapper.writeValueAsString(data), null, null);
node.onMsg(ctx, msg);
verify(ctx).tellSuccess(msg);
verify(ctx).tellNext(theMsg, "Alarm Created");
verify(ctx, Mockito.never()).tellFailure(Mockito.any(), Mockito.any());
}
@Test
public void testConstantKeyFilterInherited() throws Exception {
init();
DeviceProfile deviceProfile = new DeviceProfile();
deviceProfile.setId(deviceProfileId);
DeviceProfileData deviceProfileData = new DeviceProfileData();
Device device = new Device();
device.setId(deviceId);
device.setCustomerId(customerId);
AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey(
EntityType.TENANT, tenantId.getId(), "SERVER_SCOPE", "alarmEnabled"
);
AttributeKvEntity attributeKvEntity = new AttributeKvEntity();
attributeKvEntity.setId(compositeKey);
attributeKvEntity.setBooleanValue(Boolean.TRUE);
attributeKvEntity.setLastUpdateTs(System.currentTimeMillis());
AttributeKvEntry entry = attributeKvEntity.toData();
ListenableFuture<Optional<AttributeKvEntry>> attrListListenableFuture = Futures.immediateFuture(Optional.of(entry));
AlarmConditionFilter alarmEnabledFilter = new AlarmConditionFilter();
alarmEnabledFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.CONSTANT, "alarmEnabled"));
alarmEnabledFilter.setValue(Boolean.TRUE);
alarmEnabledFilter.setValueType(EntityKeyValueType.BOOLEAN);
BooleanFilterPredicate alarmEnabledPredicate = new BooleanFilterPredicate();
alarmEnabledPredicate.setOperation(BooleanFilterPredicate.BooleanOperation.EQUAL);
alarmEnabledPredicate.setValue(new FilterPredicateValue<>(
Boolean.FALSE,
null,
new DynamicValue<>(DynamicValueSourceType.CURRENT_DEVICE, "alarmEnabled", true)
));
alarmEnabledFilter.setPredicate(alarmEnabledPredicate);
AlarmConditionFilter temperatureFilter = new AlarmConditionFilter();
temperatureFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature"));
temperatureFilter.setValueType(EntityKeyValueType.NUMERIC);
NumericFilterPredicate temperaturePredicate = new NumericFilterPredicate();
temperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER);
temperaturePredicate.setValue(new FilterPredicateValue<>(20.0, null, null));
temperatureFilter.setPredicate(temperaturePredicate);
AlarmCondition alarmCondition = new AlarmCondition();
alarmCondition.setCondition(Arrays.asList(alarmEnabledFilter, temperatureFilter));
AlarmRule alarmRule = new AlarmRule();
alarmRule.setCondition(alarmCondition);
DeviceProfileAlarm dpa = new DeviceProfileAlarm();
dpa.setId("alarmEnabledAlarmID");
dpa.setAlarmType("alarmEnabledAlarm");
dpa.setCreateRules(new TreeMap<>(Collections.singletonMap(AlarmSeverity.CRITICAL, alarmRule)));
deviceProfileData.setAlarms(Collections.singletonList(dpa));
deviceProfile.setProfileData(deviceProfileData);
Mockito.when(deviceService.findDeviceById(tenantId, deviceId)).thenReturn(device);
Mockito.when(cache.get(tenantId, deviceId)).thenReturn(deviceProfile);
Mockito.when(timeseriesService.findLatest(tenantId, deviceId, Collections.singleton("temperature")))
.thenReturn(Futures.immediateFuture(Collections.emptyList()));
Mockito.when(alarmService.findLatestByOriginatorAndType(tenantId, deviceId, "alarmEnabledAlarm"))
.thenReturn(Futures.immediateFuture(null));
Mockito.when(alarmService.createOrUpdateAlarm(Mockito.any())).thenAnswer(AdditionalAnswers.returnsFirstArg());
Mockito.when(ctx.getAttributesService()).thenReturn(attributesService);
Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet()))
.thenReturn(Futures.immediateFuture(Collections.emptyList()));
Mockito.when(attributesService.find(eq(tenantId), eq(customerId), Mockito.anyString(), Mockito.anyString()))
.thenReturn(Futures.immediateFuture(Optional.empty()));
Mockito.when(attributesService.find(eq(tenantId), eq(tenantId), Mockito.anyString(), Mockito.anyString()))
.thenReturn(attrListListenableFuture);
TbMsg theMsg = TbMsg.newMsg("ALARM", deviceId, new TbMsgMetaData(), "");
Mockito.when(ctx.newMsg(Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.anyString()))
.thenReturn(theMsg);
ObjectNode data = mapper.createObjectNode();
data.put("temperature", 21);
TbMsg msg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, new TbMsgMetaData(),
TbMsgDataType.JSON, mapper.writeValueAsString(data), null, null);
node.onMsg(ctx, msg);
verify(ctx).tellSuccess(msg);
verify(ctx).tellNext(theMsg, "Alarm Created");
verify(ctx, Mockito.never()).tellFailure(Mockito.any(), Mockito.any());
}
@Test
public void testCurrentDeviceAttributeForDynamicValue() throws Exception {
init();
@ -228,8 +403,8 @@ public class TbDeviceProfileNodeTest {
ListenableFuture<List<AttributeKvEntry>> listListenableFutureWithLess =
Futures.immediateFuture(Collections.singletonList(entry));
KeyFilter highTempFilter = new KeyFilter();
highTempFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "temperature"));
AlarmConditionFilter highTempFilter = new AlarmConditionFilter();
highTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature"));
highTempFilter.setValueType(EntityKeyValueType.NUMERIC);
NumericFilterPredicate highTemperaturePredicate = new NumericFilterPredicate();
highTemperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER);
@ -303,8 +478,8 @@ public class TbDeviceProfileNodeTest {
ListenableFuture<Optional<AttributeKvEntry>> optionalListenableFutureWithLess =
Futures.immediateFuture(Optional.of(entry));
KeyFilter lowTempFilter = new KeyFilter();
lowTempFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "temperature"));
AlarmConditionFilter lowTempFilter = new AlarmConditionFilter();
lowTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature"));
lowTempFilter.setValueType(EntityKeyValueType.NUMERIC);
NumericFilterPredicate lowTempPredicate = new NumericFilterPredicate();
lowTempPredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS);
@ -382,8 +557,8 @@ public class TbDeviceProfileNodeTest {
ListenableFuture<Optional<AttributeKvEntry>> optionalListenableFutureWithLess =
Futures.immediateFuture(Optional.of(entry));
KeyFilter lowTempFilter = new KeyFilter();
lowTempFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "temperature"));
AlarmConditionFilter lowTempFilter = new AlarmConditionFilter();
lowTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature"));
lowTempFilter.setValueType(EntityKeyValueType.NUMERIC);
NumericFilterPredicate lowTempPredicate = new NumericFilterPredicate();
lowTempPredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS);
@ -416,7 +591,7 @@ public class TbDeviceProfileNodeTest {
Mockito.when(ctx.getAttributesService()).thenReturn(attributesService);
Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet()))
.thenReturn(listListenableFutureWithLess);
Mockito.when(attributesService.find(eq(tenantId), eq(tenantId), eq(DataConstants.SERVER_SCOPE), Mockito.anyString()))
Mockito.when(attributesService.find(eq(tenantId), eq(tenantId), eq(DataConstants.SERVER_SCOPE), Mockito.anyString()))
.thenReturn(optionalListenableFutureWithLess);
TbMsg theMsg = TbMsg.newMsg("ALARM", deviceId, new TbMsgMetaData(), "");
@ -434,11 +609,152 @@ public class TbDeviceProfileNodeTest {
verify(ctx, Mockito.never()).tellFailure(Mockito.any(), Mockito.any());
}
private void init() throws TbNodeException {
@Test
public void testTenantInheritModeForDynamicValues() throws Exception {
init();
UUID uuid = new UUID(6041557255264276971L, -9019477126543226049L);
System.out.println(uuid);
DeviceProfile deviceProfile = new DeviceProfile();
DeviceProfileData deviceProfileData = new DeviceProfileData();
AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey(
EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "tenantAttribute"
);
AttributeKvEntity attributeKvEntity = new AttributeKvEntity();
attributeKvEntity.setId(compositeKey);
attributeKvEntity.setLongValue(100L);
attributeKvEntity.setLastUpdateTs(0L);
AttributeKvEntry entry = attributeKvEntity.toData();
ListenableFuture<List<AttributeKvEntry>> listListenableFutureWithLess =
Futures.immediateFuture(Collections.singletonList(entry));
AlarmConditionFilter lowTempFilter = new AlarmConditionFilter();
lowTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature"));
lowTempFilter.setValueType(EntityKeyValueType.NUMERIC);
NumericFilterPredicate lowTempPredicate = new NumericFilterPredicate();
lowTempPredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER);
lowTempPredicate.setValue(
new FilterPredicateValue<>(
0.0,
null,
new DynamicValue<>(DynamicValueSourceType.CURRENT_DEVICE, "tenantAttribute", true))
);
lowTempFilter.setPredicate(lowTempPredicate);
AlarmCondition alarmCondition = new AlarmCondition();
alarmCondition.setCondition(Collections.singletonList(lowTempFilter));
AlarmRule alarmRule = new AlarmRule();
alarmRule.setCondition(alarmCondition);
DeviceProfileAlarm dpa = new DeviceProfileAlarm();
dpa.setId("lesstempID");
dpa.setAlarmType("lessTemperatureAlarm");
dpa.setCreateRules(new TreeMap<>(Collections.singletonMap(AlarmSeverity.CRITICAL, alarmRule)));
deviceProfileData.setAlarms(Collections.singletonList(dpa));
deviceProfile.setProfileData(deviceProfileData);
Mockito.when(cache.get(tenantId, deviceId)).thenReturn(deviceProfile);
Mockito.when(timeseriesService.findLatest(tenantId, deviceId, Collections.singleton("temperature")))
.thenReturn(Futures.immediateFuture(Collections.emptyList()));
Mockito.when(alarmService.findLatestByOriginatorAndType(tenantId, deviceId, "lessTemperatureAlarm"))
.thenReturn(Futures.immediateFuture(null));
Mockito.when(alarmService.createOrUpdateAlarm(Mockito.any()))
.thenAnswer(AdditionalAnswers.returnsFirstArg());
Mockito.when(ctx.getAttributesService()).thenReturn(attributesService);
Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet()))
.thenReturn(listListenableFutureWithLess);
TbMsg theMsg = TbMsg.newMsg("ALARM", deviceId, new TbMsgMetaData(), "");
Mockito.when(ctx.newMsg(Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.anyString()))
.thenReturn(theMsg);
ObjectNode data = mapper.createObjectNode();
data.put("temperature", 150L);
TbMsg msg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, new TbMsgMetaData(),
TbMsgDataType.JSON, mapper.writeValueAsString(data), null, null);
node.onMsg(ctx, msg);
verify(ctx).tellSuccess(msg);
verify(ctx).tellNext(theMsg, "Alarm Created");
verify(ctx, Mockito.never()).tellFailure(Mockito.any(), Mockito.any());
}
@Test
public void testCustomerInheritModeForDynamicValues() throws Exception {
init();
DeviceProfile deviceProfile = new DeviceProfile();
DeviceProfileData deviceProfileData = new DeviceProfileData();
AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey(
EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "customerAttribute"
);
AttributeKvEntity attributeKvEntity = new AttributeKvEntity();
attributeKvEntity.setId(compositeKey);
attributeKvEntity.setLongValue(100L);
attributeKvEntity.setLastUpdateTs(0L);
AttributeKvEntry entry = attributeKvEntity.toData();
ListenableFuture<List<AttributeKvEntry>> listListenableFutureWithLess =
Futures.immediateFuture(Collections.singletonList(entry));
ListenableFuture<Optional<AttributeKvEntry>> optionalListenableFutureWithLess =
Futures.immediateFuture(Optional.of(entry));
AlarmConditionFilter lowTempFilter = new AlarmConditionFilter();
lowTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature"));
lowTempFilter.setValueType(EntityKeyValueType.NUMERIC);
NumericFilterPredicate lowTempPredicate = new NumericFilterPredicate();
lowTempPredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER);
lowTempPredicate.setValue(
new FilterPredicateValue<>(
0.0,
null,
new DynamicValue<>(DynamicValueSourceType.CURRENT_CUSTOMER, "customerAttribute", true))
);
lowTempFilter.setPredicate(lowTempPredicate);
AlarmCondition alarmCondition = new AlarmCondition();
alarmCondition.setCondition(Collections.singletonList(lowTempFilter));
AlarmRule alarmRule = new AlarmRule();
alarmRule.setCondition(alarmCondition);
DeviceProfileAlarm dpa = new DeviceProfileAlarm();
dpa.setId("lesstempID");
dpa.setAlarmType("lessTemperatureAlarm");
dpa.setCreateRules(new TreeMap<>(Collections.singletonMap(AlarmSeverity.CRITICAL, alarmRule)));
deviceProfileData.setAlarms(Collections.singletonList(dpa));
deviceProfile.setProfileData(deviceProfileData);
Mockito.when(cache.get(tenantId, deviceId)).thenReturn(deviceProfile);
Mockito.when(timeseriesService.findLatest(tenantId, deviceId, Collections.singleton("temperature")))
.thenReturn(Futures.immediateFuture(Collections.emptyList()));
Mockito.when(alarmService.findLatestByOriginatorAndType(tenantId, deviceId, "lessTemperatureAlarm"))
.thenReturn(Futures.immediateFuture(null));
Mockito.when(alarmService.createOrUpdateAlarm(Mockito.any()))
.thenAnswer(AdditionalAnswers.returnsFirstArg());
Mockito.when(ctx.getAttributesService()).thenReturn(attributesService);
Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet()))
.thenReturn(listListenableFutureWithLess);
Mockito.when(attributesService.find(eq(tenantId), eq(tenantId), eq(DataConstants.SERVER_SCOPE), Mockito.anyString()))
.thenReturn(optionalListenableFutureWithLess);
TbMsg theMsg = TbMsg.newMsg("ALARM", deviceId, new TbMsgMetaData(), "");
Mockito.when(ctx.newMsg(Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.anyString()))
.thenReturn(theMsg);
ObjectNode data = mapper.createObjectNode();
data.put("temperature", 150L);
TbMsg msg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, new TbMsgMetaData(),
TbMsgDataType.JSON, mapper.writeValueAsString(data), null, null);
node.onMsg(ctx, msg);
verify(ctx).tellSuccess(msg);
verify(ctx).tellNext(theMsg, "Alarm Created");
verify(ctx, Mockito.never()).tellFailure(Mockito.any(), Mockito.any());
}
private void init() throws TbNodeException {
Mockito.when(ctx.getTenantId()).thenReturn(tenantId);
Mockito.when(ctx.getDeviceProfileCache()).thenReturn(cache);
Mockito.when(ctx.getTimeseriesService()).thenReturn(timeseriesService);

1
tools/src/main/shell/keygen.properties

@ -15,6 +15,7 @@
#
DOMAIN_SUFFIX="$(hostname)"
SUBJECT_ALTERNATIVE_NAMES="ip:127.0.0.1"
ORGANIZATIONAL_UNIT=Thingsboard
ORGANIZATION=Thingsboard
CITY=SF

9
tools/src/main/shell/server.keygen.sh

@ -86,6 +86,12 @@ fi
echo "Generating SSL Key Pair..."
EXT=""
if [[ ! -z "$SUBJECT_ALTERNATIVE_NAMES" ]]; then
EXT="-ext san=$SUBJECT_ALTERNATIVE_NAMES "
fi
keytool -genkeypair -v \
-alias $SERVER_KEY_ALIAS \
-dname "CN=$DOMAIN_SUFFIX, OU=$ORGANIZATIONAL_UNIT, O=$ORGANIZATION, L=$CITY, ST=$STATE_OR_PROVINCE, C=$TWO_LETTER_COUNTRY_CODE" \
@ -94,7 +100,8 @@ keytool -genkeypair -v \
-storepass $SERVER_KEYSTORE_PASSWORD \
-keyalg $SERVER_KEY_ALG \
-keysize $SERVER_KEY_SIZE \
-validity 9999
-validity 9999 \
$EXT
status=$?
if [[ $status != 0 ]]; then

1
transport/http/src/main/resources/tb-http-transport.yml

@ -39,6 +39,7 @@ zk:
transport:
http:
request_timeout: "${HTTP_REQUEST_TIMEOUT:60000}"
max_request_timeout: "${HTTP_MAX_REQUEST_TIMEOUT:300000}"
sessions:
inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}"
report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:30000}"

1
ui-ngx/src/app/core/http/entity.service.ts

@ -414,6 +414,7 @@ export class EntityService {
{
key: nameField,
valueType: EntityKeyValueType.STRING,
value: null,
predicate: {
type: FilterPredicateType.STRING,
operation: StringOperation.STARTS_WITH,

2
ui-ngx/src/app/modules/home/components/alarm/alarm-table-config.ts

@ -43,6 +43,7 @@ import {
AlarmDetailsDialogComponent,
AlarmDetailsDialogData
} from '@home/components/alarm/alarm-details-dialog.component';
import { DAY, historyInterval } from '@shared/models/time/time.models';
export class AlarmTableConfig extends EntityTableConfig<AlarmInfo, TimePageLink> {
@ -59,6 +60,7 @@ export class AlarmTableConfig extends EntityTableConfig<AlarmInfo, TimePageLink>
this.loadDataOnInit = false;
this.tableTitle = '';
this.useTimePageLink = true;
this.defaultTimewindowInterval = historyInterval(DAY * 30);
this.detailsPanelEnabled = false;
this.selectionEnabled = false;
this.searchEnabled = true;

8
ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts

@ -55,11 +55,11 @@ import { EntityTypeTranslation } from '@shared/models/entity-type.models';
import { DialogService } from '@core/services/dialog.service';
import { AddEntityDialogComponent } from './add-entity-dialog.component';
import { AddEntityDialogData, EntityAction } from '@home/models/entity/entity-component.models';
import { DAY, historyInterval, HistoryWindowType, Timewindow } from '@shared/models/time/time.models';
import { HistoryWindowType, Timewindow } from '@shared/models/time/time.models';
import { DomSanitizer, SafeHtml } from '@angular/platform-browser';
import { TbAnchorComponent } from '@shared/components/tb-anchor.component';
import { isDefined, isUndefined } from '@core/utils';
import { HasUUID } from '../../../../shared/models/id/has-uuid';
import { HasUUID } from '@shared/models/id/has-uuid';
@Component({
selector: 'tb-entities-table',
@ -202,7 +202,7 @@ export class EntitiesTableComponent extends PageComponent implements AfterViewIn
this.pageSizeOptions = [this.defaultPageSize, this.defaultPageSize * 2, this.defaultPageSize * 3];
if (this.entitiesTableConfig.useTimePageLink) {
this.timewindow = historyInterval(DAY);
this.timewindow = this.entitiesTableConfig.defaultTimewindowInterval;
const currentTime = Date.now();
this.pageLink = new TimePageLink(10, 0, null, sortOrder,
currentTime - this.timewindow.history.timewindowMs, currentTime);
@ -446,7 +446,7 @@ export class EntitiesTableComponent extends PageComponent implements AfterViewIn
resetSortAndFilter(update: boolean = true, preserveTimewindow: boolean = false) {
this.pageLink.textSearch = null;
if (this.entitiesTableConfig.useTimePageLink && !preserveTimewindow) {
this.timewindow = historyInterval(DAY);
this.timewindow = this.entitiesTableConfig.defaultTimewindowInterval;
}
if (this.displayPagination) {
this.paginator.pageIndex = 0;

6
ui-ngx/src/app/modules/home/components/entity/entity-details-panel.component.ts

@ -280,7 +280,11 @@ export class EntityDetailsPanelComponent extends PageComponent implements OnInit
saveEntity() {
if (this.detailsForm.valid) {
const editingEntity = mergeDeep(this.editingEntity, this.entityComponent.entityFormValue());
const editingEntity = {...this.editingEntity, ...this.entityComponent.entityFormValue()};
if (this.editingEntity.hasOwnProperty('additionalInfo')) {
editingEntity.additionalInfo =
mergeDeep((this.editingEntity as any).additionalInfo, this.entityComponent.entityFormValue()?.additionalInfo);
}
this.entitiesTableConfig.saveEntity(editingEntity).subscribe(
(entity) => {
this.entity = entity;

5
ui-ngx/src/app/modules/home/components/filter/boolean-filter-predicate.component.html

@ -16,7 +16,7 @@
-->
<div fxFlex fxLayout="row" fxLayoutAlign="start start" fxLayoutGap="8px" [formGroup]="booleanFilterPredicateFormGroup">
<mat-form-field floatLabel="always" hideRequiredMarker fxFlex="40" class="mat-block">
<mat-form-field floatLabel="always" hideRequiredMarker fxFlex="30" class="mat-block">
<mat-label></mat-label>
<mat-select required formControlName="operation" placeholder="{{'filter.operation.operation' | translate}}">
<mat-option *ngFor="let operation of booleanOperations" [value]="operation">
@ -25,7 +25,8 @@
</mat-select>
</mat-form-field>
<tb-filter-predicate-value [allowUserDynamicSource]="allowUserDynamicSource"
fxFlex="60"
[onlyUserDynamicSource]="onlyUserDynamicSource"
fxFlex="70"
[valueType]="valueTypeEnum.BOOLEAN"
formControlName="value">
</tb-filter-predicate-value>

2
ui-ngx/src/app/modules/home/components/filter/boolean-filter-predicate.component.ts

@ -41,6 +41,8 @@ export class BooleanFilterPredicateComponent implements ControlValueAccessor, On
@Input() allowUserDynamicSource = true;
@Input() onlyUserDynamicSource = false;
valueTypeEnum = EntityKeyValueType;
booleanFilterPredicateFormGroup: FormGroup;

1
ui-ngx/src/app/modules/home/components/filter/complex-filter-predicate-dialog.component.html

@ -39,6 +39,7 @@
[valueType]="data.valueType"
[displayUserParameters]="data.displayUserParameters"
[allowUserDynamicSource]="data.allowUserDynamicSource"
[onlyUserDynamicSource]="data.onlyUserDynamicSource"
[operation]="complexFilterFormGroup.get('operation').value"
[key]="data.key"
formControlName="predicates">

1
ui-ngx/src/app/modules/home/components/filter/complex-filter-predicate-dialog.component.ts

@ -37,6 +37,7 @@ export interface ComplexFilterPredicateDialogData {
valueType: EntityKeyValueType;
displayUserParameters: boolean;
allowUserDynamicSource: boolean;
onlyUserDynamicSource: boolean;
}
@Component({

5
ui-ngx/src/app/modules/home/components/filter/complex-filter-predicate.component.ts

@ -52,6 +52,8 @@ export class ComplexFilterPredicateComponent implements ControlValueAccessor, On
@Input() allowUserDynamicSource = true;
@Input() onlyUserDynamicSource = false;
private propagateChange = null;
private complexFilterPredicate: ComplexFilterPredicateInfo;
@ -89,7 +91,8 @@ export class ComplexFilterPredicateComponent implements ControlValueAccessor, On
isAdd: false,
key: this.key,
displayUserParameters: this.displayUserParameters,
allowUserDynamicSource: this.allowUserDynamicSource
allowUserDynamicSource: this.allowUserDynamicSource,
onlyUserDynamicSource: this.onlyUserDynamicSource
}
}).afterClosed().subscribe(
(result) => {

5
ui-ngx/src/app/modules/home/components/filter/filter-predicate-list.component.html

@ -26,12 +26,12 @@
<span fxFlex="8"></span>
<div fxLayout="row" fxLayoutAlign="start center" fxLayoutGap="8px" fxFlex="92">
<div fxFlex fxLayout="row" fxLayoutGap="8px">
<div fxFlex="40" fxLayout="row" fxLayoutAlign="start center" fxLayoutGap="8px">
<div fxFlex="30" fxLayout="row" fxLayoutAlign="start center" fxLayoutGap="8px">
<label fxFlex translate class="tb-title no-padding">filter.operation.operation</label>
<label *ngIf="valueType === valueTypeEnum.STRING"
translate class="tb-title no-padding" style="min-width: 70px;">filter.ignore-case</label>
</div>
<label fxFlex="60" translate class="tb-title no-padding">filter.value</label>
<label fxFlex="70" translate class="tb-title no-padding">filter.value</label>
</div>
<label *ngIf="displayUserParameters"
translate class="tb-title no-padding" style="width: 60px;">filter.user-parameters</label>
@ -53,6 +53,7 @@
[valueType]="valueType"
[displayUserParameters]="displayUserParameters"
[allowUserDynamicSource]="allowUserDynamicSource"
[onlyUserDynamicSource]="onlyUserDynamicSource"
[key]="key"
[formControl]="predicateControl">
</tb-filter-predicate>

5
ui-ngx/src/app/modules/home/components/filter/filter-predicate-list.component.ts

@ -66,6 +66,8 @@ export class FilterPredicateListComponent implements ControlValueAccessor, OnIni
@Input() allowUserDynamicSource = true;
@Input() onlyUserDynamicSource = false;
filterListFormGroup: FormGroup;
valueTypeEnum = EntityKeyValueType;
@ -159,7 +161,8 @@ export class FilterPredicateListComponent implements ControlValueAccessor, OnIni
key: this.key,
isAdd: true,
displayUserParameters: this.displayUserParameters,
allowUserDynamicSource: this.allowUserDynamicSource
allowUserDynamicSource: this.allowUserDynamicSource,
onlyUserDynamicSource: this.onlyUserDynamicSource
}
}).afterClosed().pipe(
map((result) => {

16
ui-ngx/src/app/modules/home/components/filter/filter-predicate-value.component.html

@ -16,7 +16,7 @@
-->
<div fxFlex fxLayout="row" fxLayoutAlign="start center" fxLayoutGap="8px" [formGroup]="filterPredicateValueFormGroup">
<div fxFlex fxLayout="column" [fxShow]="!dynamicMode">
<div fxFlex fxLayout="column" [fxShow]="!dynamicMode && !onlyUserDynamicSource">
<div fxLayout="column" [ngSwitch]="valueType">
<ng-template [ngSwitchCase]="valueTypeEnum.STRING">
<mat-form-field floatLabel="always" hideRequiredMarker class="mat-block">
@ -45,9 +45,9 @@
</div>
<div class="tb-hint" translate>filter.default-value</div>
</div>
<div fxFlex fxLayout="column" [fxShow]="dynamicMode">
<div fxFlex fxLayout="column" [fxShow]="dynamicMode || onlyUserDynamicSource">
<div formGroupName="dynamicValue" fxLayout="row" fxLayoutAlign="start center" fxLayoutGap="8px">
<div fxFlex fxLayout="column">
<div fxFlex="35" fxLayout="column">
<mat-form-field floatLabel="always" hideRequiredMarker class="mat-block">
<mat-label></mat-label>
<mat-select formControlName="sourceType" placeholder="{{'filter.dynamic-source-type' | translate}}">
@ -68,9 +68,17 @@
</mat-form-field>
<div class="tb-hint" translate>filter.source-attribute</div>
</div>
<div *ngIf="!allow && inheritMode"
fxLayout="column"
style="padding-top: 6px">
<mat-checkbox formControlName="inherit">
{{ 'filter.inherit-owner' | translate}}
</mat-checkbox>
<div class="tb-hint" translate>filter.source-attribute-not-set</div>
</div>
</div>
</div>
<button mat-icon-button
<button mat-icon-button *ngIf="!onlyUserDynamicSource"
class="mat-elevation-z1 tb-mat-32"
color="primary"
type="button"

27
ui-ngx/src/app/modules/home/components/filter/filter-predicate-value.component.ts

@ -44,6 +44,10 @@ import {
})
export class FilterPredicateValueComponent implements ControlValueAccessor, OnInit {
private readonly inheritModeForSources: DynamicValueSourceType[] = [
DynamicValueSourceType.CURRENT_CUSTOMER,
DynamicValueSourceType.CURRENT_DEVICE];
@Input() disabled: boolean;
@Input()
@ -58,6 +62,8 @@ export class FilterPredicateValueComponent implements ControlValueAccessor, OnIn
}
}
@Input() onlyUserDynamicSource = false;
@Input()
valueType: EntityKeyValueType;
@ -72,6 +78,8 @@ export class FilterPredicateValueComponent implements ControlValueAccessor, OnIn
dynamicMode = false;
inheritMode = false;
allow = true;
private propagateChange = null;
@ -105,7 +113,8 @@ export class FilterPredicateValueComponent implements ControlValueAccessor, OnIn
dynamicValue: this.fb.group(
{
sourceType: [null],
sourceAttribute: [null]
sourceAttribute: [null],
inherit: [false]
}
)
});
@ -114,6 +123,7 @@ export class FilterPredicateValueComponent implements ControlValueAccessor, OnIn
if (!sourceType) {
this.filterPredicateValueFormGroup.get('dynamicValue').get('sourceAttribute').patchValue(null, {emitEvent: false});
}
this.updateShowInheritMode(sourceType);
}
);
this.filterPredicateValueFormGroup.valueChanges.subscribe(() => {
@ -139,10 +149,13 @@ export class FilterPredicateValueComponent implements ControlValueAccessor, OnIn
writeValue(predicateValue: FilterPredicateValue<string | number | boolean>): void {
this.filterPredicateValueFormGroup.get('defaultValue').patchValue(predicateValue.defaultValue, {emitEvent: false});
this.filterPredicateValueFormGroup.get('dynamicValue').get('sourceType').patchValue(predicateValue.dynamicValue ?
this.filterPredicateValueFormGroup.get('dynamicValue.sourceType').patchValue(predicateValue.dynamicValue ?
predicateValue.dynamicValue.sourceType : null, {emitEvent: false});
this.filterPredicateValueFormGroup.get('dynamicValue').get('sourceAttribute').patchValue(predicateValue.dynamicValue ?
this.filterPredicateValueFormGroup.get('dynamicValue.sourceAttribute').patchValue(predicateValue.dynamicValue ?
predicateValue.dynamicValue.sourceAttribute : null, {emitEvent: false});
this.filterPredicateValueFormGroup.get('dynamicValue.inherit').patchValue(predicateValue.dynamicValue ?
predicateValue.dynamicValue.inherit : false, {emitEvent: false});
this.updateShowInheritMode(predicateValue?.dynamicValue?.sourceType);
}
private updateModel() {
@ -158,4 +171,12 @@ export class FilterPredicateValueComponent implements ControlValueAccessor, OnIn
this.propagateChange(predicateValue);
}
private updateShowInheritMode(sourceType: DynamicValueSourceType) {
if (this.inheritModeForSources.includes(sourceType)) {
this.inheritMode = true;
} else {
this.filterPredicateValueFormGroup.get('dynamicValue.inherit').patchValue(false, {emitEvent: false});
this.inheritMode = false;
}
}
}

4
ui-ngx/src/app/modules/home/components/filter/filter-predicate.component.html

@ -20,12 +20,14 @@
<ng-template [ngSwitchCase]="filterPredicateType.STRING">
<tb-string-filter-predicate
[allowUserDynamicSource]="allowUserDynamicSource"
[onlyUserDynamicSource]="onlyUserDynamicSource"
formControlName="predicate">
</tb-string-filter-predicate>
</ng-template>
<ng-template [ngSwitchCase]="filterPredicateType.NUMERIC">
<tb-numeric-filter-predicate
[allowUserDynamicSource]="allowUserDynamicSource"
[onlyUserDynamicSource]="onlyUserDynamicSource"
[valueType]="valueType"
formControlName="predicate">
</tb-numeric-filter-predicate>
@ -33,12 +35,14 @@
<ng-template [ngSwitchCase]="filterPredicateType.BOOLEAN">
<tb-boolean-filter-predicate
[allowUserDynamicSource]="allowUserDynamicSource"
[onlyUserDynamicSource]="onlyUserDynamicSource"
formControlName="predicate">
</tb-boolean-filter-predicate>
</ng-template>
<ng-template [ngSwitchCase]="filterPredicateType.COMPLEX">
<tb-complex-filter-predicate
[allowUserDynamicSource]="allowUserDynamicSource"
[onlyUserDynamicSource]="onlyUserDynamicSource"
[key]="key"
[valueType]="valueType"
[displayUserParameters]="displayUserParameters"

2
ui-ngx/src/app/modules/home/components/filter/filter-predicate.component.ts

@ -45,6 +45,8 @@ export class FilterPredicateComponent implements ControlValueAccessor, OnInit {
@Input() allowUserDynamicSource = true;
@Input() onlyUserDynamicSource = false;
filterPredicateFormGroup: FormGroup;
type: FilterPredicateType;

32
ui-ngx/src/app/modules/home/components/filter/key-filter-dialog.component.html

@ -28,7 +28,7 @@
<div mat-dialog-content>
<fieldset [disabled]="isLoading$ | async" fxLayout="column">
<section fxLayout="row" fxLayoutGap="8px" class="entity-key">
<section fxFlex="70" fxLayout="row" formGroupName="key" fxLayoutGap="8px">
<section [fxFlex]="isConstantKeyType ? 45 : 70" fxLayout="row" formGroupName="key" fxLayoutGap="8px">
<mat-form-field fxFlex="40" class="mat-block">
<mat-label translate>filter.key-type.key-type</mat-label>
<mat-select required formControlName="type">
@ -60,7 +60,7 @@
</mat-error>
</mat-form-field>
</section>
<mat-form-field fxFlex="30" class="mat-block">
<mat-form-field [fxFlex]="isConstantKeyType ? 20 : 30" class="mat-block">
<mat-label translate>filter.value-type.value-type</mat-label>
<mat-select formControlName="valueType">
<mat-select-trigger>
@ -76,9 +76,37 @@
{{ 'filter.value-type-required' | translate }}
</mat-error>
</mat-form-field>
<div fxFlex *ngIf="isConstantKeyType">
<div fxLayout="column" [ngSwitch]="keyFilterFormGroup.get('valueType').value">
<ng-template [ngSwitchCase]="entityKeyValueTypeEnum.STRING">
<mat-form-field class="mat-block">
<mat-label translate>filter.value</mat-label>
<input matInput required formControlName="value" placeholder="{{'filter.value' | translate}}">
</mat-form-field>
</ng-template>
<ng-template [ngSwitchCase]="entityKeyValueTypeEnum.NUMERIC">
<mat-form-field class="mat-block">
<mat-label translate>filter.value</mat-label>
<input required type="number" matInput formControlName="value">
</mat-form-field>
</ng-template>
<ng-template [ngSwitchCase]="entityKeyValueTypeEnum.DATE_TIME">
<tb-datetime formControlName="value"
dateText="filter.date"
timeText="filter.time"
required></tb-datetime>
</ng-template>
<ng-template [ngSwitchCase]="entityKeyValueTypeEnum.BOOLEAN">
<mat-checkbox formControlName="value" style="margin-top: 20px">
{{ (keyFilterFormGroup.get('value').value ? 'value.true' : 'value.false') | translate }}
</mat-checkbox>
</ng-template>
</div>
</div>
</section>
<tb-filter-predicate-list *ngIf="keyFilterFormGroup.get('valueType').value"
[allowUserDynamicSource]="data.allowUserDynamicSource"
[onlyUserDynamicSource]="isConstantKeyType"
[displayUserParameters]="data.displayUserParameters"
[valueType]="keyFilterFormGroup.get('valueType').value"
[key]="keyFilterFormGroup.get('key.key').value"

33
ui-ngx/src/app/modules/home/components/filter/key-filter-dialog.component.ts

@ -14,7 +14,7 @@
/// limitations under the License.
///
import { Component, ElementRef, Inject, OnDestroy, OnInit, SkipSelf, ViewChild } from '@angular/core';
import { Component, ElementRef, Inject, OnDestroy, SkipSelf, ViewChild } from '@angular/core';
import { ErrorStateMatcher } from '@angular/material/core';
import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog';
import { Store } from '@ngrx/store';
@ -57,7 +57,7 @@ export interface KeyFilterDialogData {
})
export class KeyFilterDialogComponent extends
DialogComponent<KeyFilterDialogComponent, KeyFilterInfo>
implements OnInit, OnDestroy, ErrorStateMatcher {
implements OnDestroy, ErrorStateMatcher {
@ViewChild('keyNameInput', {static: true}) private keyNameInput: ElementRef;
@ -69,7 +69,7 @@ export class KeyFilterDialogComponent extends
entityKeyTypes =
this.data.telemetryKeysOnly ?
[EntityKeyType.ATTRIBUTE, EntityKeyType.TIME_SERIES] :
[EntityKeyType.ATTRIBUTE, EntityKeyType.TIME_SERIES, EntityKeyType.CONSTANT] :
[EntityKeyType.ENTITY_FIELD, EntityKeyType.ATTRIBUTE, EntityKeyType.TIME_SERIES];
entityKeyTypeTranslations = entityKeyTypeTranslationMap;
@ -107,6 +107,7 @@ export class KeyFilterDialogComponent extends
key: [this.data.keyFilter.key.key, [Validators.required]]
}
),
value: [this.data.keyFilter.value],
valueType: [this.data.keyFilter.valueType, [Validators.required]],
predicates: [this.data.keyFilter.predicates, [Validators.required]]
}
@ -143,6 +144,13 @@ export class KeyFilterDialogComponent extends
} else {
this.showAutocomplete = false;
}
if (type === EntityKeyType.CONSTANT) {
this.keyFilterFormGroup.get('value').setValidators(Validators.required);
this.keyFilterFormGroup.get('value').updateValueAndValidity();
} else {
this.keyFilterFormGroup.get('value').clearValidators();
this.keyFilterFormGroup.get('value').updateValueAndValidity();
}
});
this.keyFilterFormGroup.get('key.key').valueChanges.pipe(
@ -156,19 +164,18 @@ export class KeyFilterDialogComponent extends
this.keyFilterFormGroup.get('valueType').patchValue(newValueType, {emitEvent: false});
}
});
this.filteredKeysName = this.keyFilterFormGroup.get('key.key').valueChanges
.pipe(
map(value => value ? value : ''),
mergeMap(name => this.fetchEntityName(name)),
takeUntil(this.destroy$)
);
} else {
this.keyFilterFormGroup.disable({emitEvent: false});
}
}
ngOnInit() {
this.filteredKeysName = this.keyFilterFormGroup.get('key.key').valueChanges
.pipe(
map(value => value ? value : ''),
mergeMap(name => this.fetchEntityName(name))
);
}
ngOnDestroy() {
super.ngOnDestroy();
this.destroy$.next();
@ -208,6 +215,10 @@ export class KeyFilterDialogComponent extends
}
}
get isConstantKeyType(): boolean {
return this.keyFilterFormGroup.get('key.type').value === EntityKeyType.CONSTANT;
}
private fetchEntityName(searchText?: string): Observable<Array<string>> {
this.searchText = searchText;
return this.getEntityKeys().pipe(

1
ui-ngx/src/app/modules/home/components/filter/key-filter-list.component.ts

@ -161,6 +161,7 @@ export class KeyFilterListComponent implements ControlValueAccessor, OnInit {
type: EntityKeyType.ATTRIBUTE
},
valueType: null,
value: null,
predicates: []
};
}

5
ui-ngx/src/app/modules/home/components/filter/numeric-filter-predicate.component.html

@ -16,7 +16,7 @@
-->
<div fxFlex fxLayout="row" fxLayoutAlign="start start" fxLayoutGap="8px" [formGroup]="numericFilterPredicateFormGroup">
<mat-form-field floatLabel="always" hideRequiredMarker fxFlex="40" class="mat-block">
<mat-form-field floatLabel="always" hideRequiredMarker fxFlex="30" class="mat-block">
<mat-label></mat-label>
<mat-select required formControlName="operation" placeholder="{{'filter.operation.operation' | translate}}">
<mat-option *ngFor="let operation of numericOperations" [value]="operation">
@ -25,7 +25,8 @@
</mat-select>
</mat-form-field>
<tb-filter-predicate-value [allowUserDynamicSource]="allowUserDynamicSource"
fxFlex="60"
[onlyUserDynamicSource]="onlyUserDynamicSource"
fxFlex="70"
[valueType]="valueType"
formControlName="value">
</tb-filter-predicate-value>

2
ui-ngx/src/app/modules/home/components/filter/numeric-filter-predicate.component.ts

@ -42,6 +42,8 @@ export class NumericFilterPredicateComponent implements ControlValueAccessor, On
@Input() allowUserDynamicSource = true;
@Input() onlyUserDynamicSource = false;
@Input() valueType: EntityKeyValueType;
numericFilterPredicateFormGroup: FormGroup;

5
ui-ngx/src/app/modules/home/components/filter/string-filter-predicate.component.html

@ -16,7 +16,7 @@
-->
<div fxFlex fxLayout="row" fxLayoutAlign="start start" fxLayoutGap="8px" [formGroup]="stringFilterPredicateFormGroup">
<div fxFlex="40" fxLayout="row" fxLayoutAlign="start center" fxLayoutGap="8px">
<div fxFlex="30" fxLayout="row" fxLayoutAlign="start center" fxLayoutGap="8px">
<mat-form-field floatLabel="always" hideRequiredMarker fxFlex class="mat-block">
<mat-label></mat-label>
<mat-select required formControlName="operation" placeholder="{{'filter.operation.operation' | translate}}">
@ -29,7 +29,8 @@
</mat-checkbox>
</div>
<tb-filter-predicate-value [allowUserDynamicSource]="allowUserDynamicSource"
fxFlex="60"
[onlyUserDynamicSource]="onlyUserDynamicSource"
fxFlex="70"
[valueType]="valueTypeEnum.STRING"
formControlName="value">
</tb-filter-predicate-value>

2
ui-ngx/src/app/modules/home/components/filter/string-filter-predicate.component.ts

@ -42,6 +42,8 @@ export class StringFilterPredicateComponent implements ControlValueAccessor, OnI
@Input() allowUserDynamicSource = true;
@Input() onlyUserDynamicSource = false;
valueTypeEnum = EntityKeyValueType;
stringFilterPredicateFormGroup: FormGroup;

4
ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts

@ -30,6 +30,7 @@ import { EntitiesTableComponent } from '@home/components/entity/entities-table.c
import { EntityTableHeaderComponent } from '@home/components/entity/entity-table-header.component';
import { ActivatedRoute } from '@angular/router';
import { EntityTabsComponent } from '../../components/entity/entity-tabs.component';
import { DAY, historyInterval } from '@shared/models/time/time.models';
export type EntityBooleanFunction<T extends BaseData<HasId>> = (entity: T) => boolean;
export type EntityStringFunction<T extends BaseData<HasId>> = (entity: T) => string;
@ -135,6 +136,7 @@ export class EntityTableConfig<T extends BaseData<HasId>, P extends PageLink = P
onLoadAction: (route: ActivatedRoute) => void = null;
table: EntitiesTableComponent = null;
useTimePageLink = false;
defaultTimewindowInterval = historyInterval(DAY);
entityType: EntityType = null;
tableTitle = '';
selectionEnabled = true;
@ -162,7 +164,7 @@ export class EntityTableConfig<T extends BaseData<HasId>, P extends PageLink = P
dataSource: (dataLoadedFunction: (col?: number, row?: number) => void)
=> EntitiesDataSource<L> = (dataLoadedFunction: (col?: number, row?: number) => void) => {
return new EntitiesDataSource(this.entitiesFetchFunction, this.entitySelectionEnabled, dataLoadedFunction);
};
}
detailsReadonly: EntityBooleanFunction<T> = () => false;
entitySelectionEnabled: EntityBooleanFunction<L> = () => true;
deleteEnabled: EntityBooleanFunction<T | L> = () => true;

2
ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html

@ -42,7 +42,7 @@
</mat-tab>
<mat-tab *ngIf="entity" #alarmRules="matTab"
label="{{'device-profile.alarm-rules-with-count' | translate:
{count: entity.profileData?.alarms?.length ? entity.profileData.alarms.length : 0}
{count: this.detailsForm.get('profileData.alarms').value?.length ? this.detailsForm.get('profileData.alarms').value.length : 0}
}}">
<div class="mat-padding" [formGroup]="detailsForm" *ngIf="alarmRules.isActive">
<div formGroupName="profileData">

4
ui-ngx/src/app/shared/components/file-input.component.ts

@ -34,6 +34,7 @@ import { Subscription } from 'rxjs';
import { coerceBooleanProperty } from '@angular/cdk/coercion';
import { FlowDirective } from '@flowjs/ngx-flow';
import { TranslateService } from '@ngx-translate/core';
import { UtilsService } from '@core/services/utils.service';
@Component({
selector: 'tb-file-input',
@ -59,7 +60,7 @@ export class FileInputComponent extends PageComponent implements AfterViewInit,
noFileText = 'import.no-file';
@Input()
inputId = 'select';
inputId = this.utils.guid();
@Input()
allowedExtensions: string;
@ -114,6 +115,7 @@ export class FileInputComponent extends PageComponent implements AfterViewInit,
private propagateChange = null;
constructor(protected store: Store<AppState>,
private utils: UtilsService,
public translate: TranslateService) {
super(store);
}

12
ui-ngx/src/app/shared/models/query/query.models.ts

@ -35,14 +35,16 @@ export enum EntityKeyType {
SERVER_ATTRIBUTE = 'SERVER_ATTRIBUTE',
TIME_SERIES = 'TIME_SERIES',
ENTITY_FIELD = 'ENTITY_FIELD',
ALARM_FIELD = 'ALARM_FIELD'
ALARM_FIELD = 'ALARM_FIELD',
CONSTANT = 'CONSTANT'
}
export const entityKeyTypeTranslationMap = new Map<EntityKeyType, string>(
[
[EntityKeyType.ATTRIBUTE, 'filter.key-type.attribute'],
[EntityKeyType.TIME_SERIES, 'filter.key-type.timeseries'],
[EntityKeyType.ENTITY_FIELD, 'filter.key-type.entity-field']
[EntityKeyType.ENTITY_FIELD, 'filter.key-type.entity-field'],
[EntityKeyType.CONSTANT, 'filter.key-type.constant']
]
);
@ -285,6 +287,7 @@ export const dynamicValueSourceTypeTranslationMap = new Map<DynamicValueSourceTy
export interface DynamicValue<T> {
sourceType: DynamicValueSourceType;
sourceAttribute: string;
inherit?: boolean;
}
export interface FilterPredicateValue<T> {
@ -343,12 +346,14 @@ export interface KeyFilterPredicateInfo {
export interface KeyFilter {
key: EntityKey;
valueType: EntityKeyValueType;
value: string | number | boolean;
predicate: KeyFilterPredicate;
}
export interface KeyFilterInfo {
key: EntityKey;
valueType: EntityKeyValueType;
value: string | number | boolean;
predicates: Array<KeyFilterPredicateInfo>;
}
@ -465,6 +470,7 @@ export function keyFilterInfosToKeyFilters(keyFilterInfos: Array<KeyFilterInfo>)
const keyFilter: KeyFilter = {
key,
valueType: keyFilterInfo.valueType,
value: keyFilterInfo.value,
predicate: keyFilterPredicateInfoToKeyFilterPredicate(predicate)
};
keyFilters.push(keyFilter);
@ -485,6 +491,7 @@ export function keyFiltersToKeyFilterInfos(keyFilters: Array<KeyFilter>): Array<
keyFilterInfo = {
key,
valueType: keyFilter.valueType,
value: keyFilter.value,
predicates: []
};
keyFilterInfoMap[infoKey] = keyFilterInfo;
@ -507,6 +514,7 @@ export function filterInfoToKeyFilters(filter: FilterInfo): Array<KeyFilter> {
const keyFilter: KeyFilter = {
key,
valueType: keyFilterInfo.valueType,
value: keyFilterInfo.value,
predicate: keyFilterPredicateInfoToKeyFilterPredicate(predicate)
};
keyFilters.push(keyFilter);

7
ui-ngx/src/assets/locale/locale.constant-en_US.json

@ -1564,7 +1564,8 @@
"key-type": "Key type",
"attribute": "Attribute",
"timeseries": "Timeseries",
"entity-field": "Entity field"
"entity-field": "Entity field",
"constant": "Constant"
},
"value-type": {
"value-type": "Value type",
@ -1591,7 +1592,9 @@
"no-dynamic-value": "No dynamic value",
"source-attribute": "Source attribute",
"switch-to-dynamic-value": "Switch to dynamic value",
"switch-to-default-value": "Switch to default value"
"switch-to-default-value": "Switch to default value",
"inherit-owner": "Inherit from owner",
"source-attribute-not-set": "If source attribute isn't set"
},
"fullscreen": {
"expand": "Expand to fullscreen",

Loading…
Cancel
Save