diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql b/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql index a650244d5a..3c2d43e197 100644 --- a/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql +++ b/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql @@ -84,11 +84,12 @@ BEGIN END IF; END IF; END IF; - END IF; - IF partition_to_delete IS NOT NULL THEN - RAISE NOTICE 'Partition to delete by max ttl: %', partition_to_delete; - EXECUTE format('DROP TABLE %I', partition_to_delete); - deleted := deleted + 1; + IF partition_to_delete IS NOT NULL THEN + RAISE NOTICE 'Partition to delete by max ttl: %', partition_to_delete; + EXECUTE format('DROP TABLE IF EXISTS %I', partition_to_delete); + partition_to_delete := NULL; + deleted := deleted + 1; + END IF; END IF; END LOOP; END IF; diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 6d27516811..e1717e3b9b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; @@ -278,7 +279,21 @@ class DefaultTbContext implements TbContext { } public TbMsg deviceCreatedMsg(Device device, RuleNodeId ruleNodeId) { - return entityActionMsg(device, device.getId(), ruleNodeId, DataConstants.ENTITY_CREATED); + RuleChainId ruleChainId = null; + String queueName = ServiceQueue.MAIN; + if (device.getDeviceProfileId() != null) { + DeviceProfile deviceProfile = mainCtx.getDeviceProfileCache().find(device.getDeviceProfileId()); + if (deviceProfile == null) { + log.warn("[{}] Device profile is null!", device.getDeviceProfileId()); + ruleChainId = null; + queueName = ServiceQueue.MAIN; + } else { + ruleChainId = deviceProfile.getDefaultRuleChainId(); + String defaultQueueName = deviceProfile.getDefaultQueueName(); + queueName = defaultQueueName != null ? defaultQueueName : ServiceQueue.MAIN; + } + } + return entityActionMsg(device, device.getId(), ruleNodeId, DataConstants.ENTITY_CREATED, queueName, ruleChainId); } public TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId) { @@ -286,12 +301,31 @@ class DefaultTbContext implements TbContext { } public TbMsg alarmActionMsg(Alarm alarm, RuleNodeId ruleNodeId, String action) { - return entityActionMsg(alarm, alarm.getId(), ruleNodeId, action); + RuleChainId ruleChainId = null; + String queueName = ServiceQueue.MAIN; + if (EntityType.DEVICE.equals(alarm.getOriginator().getEntityType())) { + DeviceId deviceId = new DeviceId(alarm.getOriginator().getId()); + DeviceProfile deviceProfile = mainCtx.getDeviceProfileCache().get(getTenantId(), deviceId); + if (deviceProfile == null) { + log.warn("[{}] Device profile is null!", deviceId); + ruleChainId = null; + queueName = ServiceQueue.MAIN; + } else { + ruleChainId = deviceProfile.getDefaultRuleChainId(); + String defaultQueueName = deviceProfile.getDefaultQueueName(); + queueName = defaultQueueName != null ? defaultQueueName : ServiceQueue.MAIN; + } + } + return entityActionMsg(alarm, alarm.getId(), ruleNodeId, action, queueName, ruleChainId); } public TbMsg entityActionMsg(E entity, I id, RuleNodeId ruleNodeId, String action) { + return entityActionMsg(entity, id, ruleNodeId, action, ServiceQueue.MAIN, null); + } + + public TbMsg entityActionMsg(E entity, I id, RuleNodeId ruleNodeId, String action, String queueName, RuleChainId ruleChainId) { try { - return TbMsg.newMsg(action, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity))); + return TbMsg.newMsg(queueName, action, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity)), ruleChainId, null); } catch (JsonProcessingException | IllegalArgumentException e) { throw new RuntimeException("Failed to process " + id.getEntityType().name().toLowerCase() + " " + action + " msg: " + e); } diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index 6b7b5ff3d1..05363dfd59 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -34,6 +34,7 @@ import org.thingsboard.server.actors.app.AppInitMsg; import org.thingsboard.server.actors.stats.StatsActor; import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -43,7 +44,7 @@ import java.util.concurrent.ScheduledExecutorService; @Service @Slf4j -public class DefaultActorService implements ActorService { +public class DefaultActorService extends TbApplicationEventListener implements ActorService { public static final String APP_DISPATCHER_NAME = "app-dispatcher"; public static final String TENANT_DISPATCHER_NAME = "tenant-dispatcher"; @@ -120,10 +121,10 @@ public class DefaultActorService implements ActorService { appActor.tellWithHighPriority(new AppInitMsg()); } - @EventListener(PartitionChangeEvent.class) - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + @Override + protected void onTbApplicationEvent(PartitionChangeEvent event) { log.info("Received partition change event."); - this.appActor.tellWithHighPriority(new PartitionChangeMsg(partitionChangeEvent.getServiceQueueKey(), partitionChangeEvent.getPartitions())); + this.appActor.tellWithHighPriority(new PartitionChangeMsg(event.getServiceQueueKey(), event.getPartitions())); } @PreDestroy diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 6e2bffa366..6a7ad5158c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -27,7 +27,22 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.ExceptionHandler; -import org.thingsboard.server.common.data.*; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.DashboardInfo; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceInfo; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.EntityViewInfo; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantInfo; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.asset.Asset; @@ -84,6 +99,7 @@ import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService; import org.thingsboard.server.dao.oauth2.OAuth2Service; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.user.UserService; @@ -95,7 +111,6 @@ import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.profile.TbDeviceProfileCache; -import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; @@ -123,6 +138,9 @@ public abstract class BaseController { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; public static final String YOU_DON_T_HAVE_PERMISSION_TO_PERFORM_THIS_OPERATION = "You don't have permission to perform this operation!"; + protected static final String DEFAULT_DASHBOARD = "defaultDashboardId"; + protected static final String HOME_DASHBOARD = "homeDashboardId"; + private static final ObjectMapper json = new ObjectMapper(); @Autowired @@ -858,4 +876,14 @@ public abstract class BaseController { } } } + + protected void processDashboardIdFromAdditionalInfo(ObjectNode additionalInfo, String requiredFields) throws ThingsboardException { + String dashboardId = additionalInfo.has(requiredFields) ? additionalInfo.get(requiredFields).asText() : null; + if(dashboardId != null && !dashboardId.equals("null")) { + if(dashboardService.findDashboardById(getTenantId(), new DashboardId(UUID.fromString(dashboardId))) == null) { + additionalInfo.remove(requiredFields); + } + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java index 1a2357450c..dd0312169b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java @@ -55,7 +55,11 @@ public class CustomerController extends BaseController { checkParameter(CUSTOMER_ID, strCustomerId); try { CustomerId customerId = new CustomerId(toUUID(strCustomerId)); - return checkCustomerId(customerId, Operation.READ); + Customer customer = checkCustomerId(customerId, Operation.READ); + if(!customer.getAdditionalInfo().isNull()) { + processDashboardIdFromAdditionalInfo((ObjectNode) customer.getAdditionalInfo(), HOME_DASHBOARD); + } + return customer; } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index 2bd8a04578..70b5efe40d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.controller; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; @@ -59,7 +60,11 @@ public class TenantController extends BaseController { checkParameter("tenantId", strTenantId); try { TenantId tenantId = new TenantId(toUUID(strTenantId)); - return checkTenantId(tenantId, Operation.READ); + Tenant tenant = checkTenantId(tenantId, Operation.READ); + if(!tenant.getAdditionalInfo().isNull()) { + processDashboardIdFromAdditionalInfo((ObjectNode) tenant.getAdditionalInfo(), HOME_DASHBOARD); + } + return tenant; } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/UserController.java b/application/src/main/java/org/thingsboard/server/controller/UserController.java index 47be552c2f..5c6e7e9fe8 100644 --- a/application/src/main/java/org/thingsboard/server/controller/UserController.java +++ b/application/src/main/java/org/thingsboard/server/controller/UserController.java @@ -53,7 +53,6 @@ import org.thingsboard.server.service.security.model.token.JwtTokenFactory; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.system.SystemSecurityService; -import org.thingsboard.server.utils.MiscUtils; import javax.servlet.http.HttpServletRequest; @@ -90,7 +89,12 @@ public class UserController extends BaseController { checkParameter(USER_ID, strUserId); try { UserId userId = new UserId(toUUID(strUserId)); - return checkUserId(userId, Operation.READ); + User user = checkUserId(userId, Operation.READ); + if(!user.getAdditionalInfo().isNull()) { + processDashboardIdFromAdditionalInfo((ObjectNode) user.getAdditionalInfo(), DEFAULT_DASHBOARD); + processDashboardIdFromAdditionalInfo((ObjectNode) user.getAdditionalInfo(), HOME_DASHBOARD); + } + return user; } catch (Exception e) { throw handleException(e); } @@ -329,4 +333,5 @@ public class UserController extends BaseController { throw handleException(e); } } + } diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 658c4a1fba..d97866fcb5 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -187,6 +187,10 @@ public class ThingsboardInstallService { databaseEntitiesUpgradeService.upgradeDatabase("3.2.0"); case "3.2.1": log.info("Upgrading ThingsBoard from version 3.2.1 to 3.3.0 ..."); + if (databaseTsUpgradeService != null) { + databaseTsUpgradeService.upgradeDatabase("3.2.1"); + } + log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); break; diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index d4a5d42320..d0a3984660 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -54,6 +54,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.scheduler.SchedulerComponent; import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.telemetry.InternalTelemetryService; @@ -78,7 +79,7 @@ import java.util.stream.Collectors; @Slf4j @Service -public class DefaultTbApiUsageStateService implements TbApiUsageStateService { +public class DefaultTbApiUsageStateService extends TbApplicationEventListener implements TbApiUsageStateService { public static final String HOURLY = "Hourly"; public static final FutureCallback VOID_CALLBACK = new FutureCallback() { @@ -188,7 +189,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { } @Override - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + protected void onTbApplicationEvent(PartitionChangeEvent partitionChangeEvent) { if (partitionChangeEvent.getServiceType().equals(ServiceType.TB_CORE)) { myTenantStates.entrySet().removeIf(entry -> !partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition()); otherTenantStates.entrySet().removeIf(entry -> partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition()); diff --git a/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java index 76693192cf..0a64a59a08 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java @@ -50,6 +50,7 @@ public class CassandraTsDatabaseUpgradeService extends AbstractCassandraDatabase break; case "2.5.0": case "3.1.1": + case "3.2.1": break; default: throw new RuntimeException("Unable to upgrade Cassandra database, unsupported fromVersion: " + fromVersion); diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index ac502926f8..9028fe279e 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -36,6 +36,9 @@ import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.device.profile.AlarmCondition; +import org.thingsboard.server.common.data.device.profile.AlarmConditionFilter; +import org.thingsboard.server.common.data.device.profile.AlarmConditionFilterKey; +import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType; import org.thingsboard.server.common.data.device.profile.AlarmRule; import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration; @@ -290,16 +293,16 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { AlarmCondition temperatureCondition = new AlarmCondition(); temperatureCondition.setSpec(new SimpleAlarmConditionSpec()); - KeyFilter temperatureAlarmFlagAttributeFilter = new KeyFilter(); - temperatureAlarmFlagAttributeFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperatureAlarmFlag")); + AlarmConditionFilter temperatureAlarmFlagAttributeFilter = new AlarmConditionFilter(); + temperatureAlarmFlagAttributeFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.ATTRIBUTE, "temperatureAlarmFlag")); temperatureAlarmFlagAttributeFilter.setValueType(EntityKeyValueType.BOOLEAN); BooleanFilterPredicate temperatureAlarmFlagAttributePredicate = new BooleanFilterPredicate(); temperatureAlarmFlagAttributePredicate.setOperation(BooleanFilterPredicate.BooleanOperation.EQUAL); temperatureAlarmFlagAttributePredicate.setValue(new FilterPredicateValue<>(Boolean.TRUE)); temperatureAlarmFlagAttributeFilter.setPredicate(temperatureAlarmFlagAttributePredicate); - KeyFilter temperatureTimeseriesFilter = new KeyFilter(); - temperatureTimeseriesFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); + AlarmConditionFilter temperatureTimeseriesFilter = new AlarmConditionFilter(); + temperatureTimeseriesFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); temperatureTimeseriesFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate temperatureTimeseriesFilterPredicate = new NumericFilterPredicate(); temperatureTimeseriesFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); @@ -317,8 +320,8 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { AlarmCondition clearTemperatureCondition = new AlarmCondition(); clearTemperatureCondition.setSpec(new SimpleAlarmConditionSpec()); - KeyFilter clearTemperatureTimeseriesFilter = new KeyFilter(); - clearTemperatureTimeseriesFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); + AlarmConditionFilter clearTemperatureTimeseriesFilter = new AlarmConditionFilter(); + clearTemperatureTimeseriesFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); clearTemperatureTimeseriesFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate clearTemperatureTimeseriesFilterPredicate = new NumericFilterPredicate(); clearTemperatureTimeseriesFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS_OR_EQUAL); @@ -340,16 +343,16 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { AlarmCondition humidityCondition = new AlarmCondition(); humidityCondition.setSpec(new SimpleAlarmConditionSpec()); - KeyFilter humidityAlarmFlagAttributeFilter = new KeyFilter(); - humidityAlarmFlagAttributeFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "humidityAlarmFlag")); + AlarmConditionFilter humidityAlarmFlagAttributeFilter = new AlarmConditionFilter(); + humidityAlarmFlagAttributeFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.ATTRIBUTE, "humidityAlarmFlag")); humidityAlarmFlagAttributeFilter.setValueType(EntityKeyValueType.BOOLEAN); BooleanFilterPredicate humidityAlarmFlagAttributePredicate = new BooleanFilterPredicate(); humidityAlarmFlagAttributePredicate.setOperation(BooleanFilterPredicate.BooleanOperation.EQUAL); humidityAlarmFlagAttributePredicate.setValue(new FilterPredicateValue<>(Boolean.TRUE)); humidityAlarmFlagAttributeFilter.setPredicate(humidityAlarmFlagAttributePredicate); - KeyFilter humidityTimeseriesFilter = new KeyFilter(); - humidityTimeseriesFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "humidity")); + AlarmConditionFilter humidityTimeseriesFilter = new AlarmConditionFilter(); + humidityTimeseriesFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "humidity")); humidityTimeseriesFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate humidityTimeseriesFilterPredicate = new NumericFilterPredicate(); humidityTimeseriesFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS); @@ -368,8 +371,8 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { AlarmCondition clearHumidityCondition = new AlarmCondition(); clearHumidityCondition.setSpec(new SimpleAlarmConditionSpec()); - KeyFilter clearHumidityTimeseriesFilter = new KeyFilter(); - clearHumidityTimeseriesFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "humidity")); + AlarmConditionFilter clearHumidityTimeseriesFilter = new AlarmConditionFilter(); + clearHumidityTimeseriesFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "humidity")); clearHumidityTimeseriesFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate clearHumidityTimeseriesFilterPredicate = new NumericFilterPredicate(); clearHumidityTimeseriesFilterPredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER_OR_EQUAL); diff --git a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java index 1aed8c01c4..fddec0367d 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java @@ -196,11 +196,17 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe } break; case "3.1.1": + case "3.2.1": try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { log.info("Load TTL functions ..."); loadSql(conn, LOAD_TTL_FUNCTIONS_SQL); log.info("Load Drop Partitions functions ..."); loadSql(conn, LOAD_DROP_PARTITIONS_FUNCTIONS_SQL); + + executeQuery(conn, "DROP PROCEDURE IF EXISTS cleanup_timeseries_by_ttl(character varying, bigint, bigint);"); + executeQuery(conn, "DROP FUNCTION IF EXISTS delete_asset_records_from_ts_kv(character varying, character varying, bigint);"); + executeQuery(conn, "DROP FUNCTION IF EXISTS delete_device_records_from_ts_kv(character varying, character varying, bigint);"); + executeQuery(conn, "DROP FUNCTION IF EXISTS delete_customer_records_from_ts_kv(character varying, character varying, bigint);"); } break; default: diff --git a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java index 0920c2c07d..112ecc3018 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java @@ -178,6 +178,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr } break; case "3.1.1": + case "3.2.1": break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 51fd388af9..133447022b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -145,7 +145,7 @@ public class DefaultTbClusterService implements TbClusterService { tbMsg = transformMsg(tbMsg, deviceProfileCache.get(tenantId, new DeviceProfileId(entityId.getId()))); } } - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tbMsg.getQueueName(), tenantId, entityId); log.trace("PUSHING msg: {} to:{}", tbMsg, tpi); ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder() .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index df0e7f86b5..af9239d298 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -151,12 +151,12 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService tpi.newByTopic(usageStatsConsumer.getTopic())) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 106ca40c94..390798a3e2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -140,11 +140,11 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } @Override - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (partitionChangeEvent.getServiceType().equals(getServiceType())) { - ServiceQueue serviceQueue = partitionChangeEvent.getServiceQueueKey().getServiceQueue(); - log.info("[{}] Subscribing to partitions: {}", serviceQueue.getQueue(), partitionChangeEvent.getPartitions()); - consumers.get(serviceQueue.getQueue()).subscribe(partitionChangeEvent.getPartitions()); + protected void onTbApplicationEvent(PartitionChangeEvent event) { + if (event.getServiceType().equals(getServiceType())) { + ServiceQueue serviceQueue = event.getServiceQueueKey().getServiceQueue(); + log.info("[{}] Subscribing to partitions: {}", serviceQueue.getQueue(), event.getPartitions()); + consumers.get(serviceQueue.getQueue()).subscribe(event.getPartitions()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java index d342b4b565..bca5a90fa8 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java @@ -55,31 +55,22 @@ public class TbCoreConsumerStats { public TbCoreConsumerStats(StatsFactory statsFactory) { String statsKey = StatsType.CORE.getName(); - this.totalCounter = statsFactory.createStatsCounter(statsKey, TOTAL_MSGS); - this.sessionEventCounter = statsFactory.createStatsCounter(statsKey, SESSION_EVENTS); - this.getAttributesCounter = statsFactory.createStatsCounter(statsKey, GET_ATTRIBUTE); - this.subscribeToAttributesCounter = statsFactory.createStatsCounter(statsKey, ATTRIBUTE_SUBSCRIBES); - this.subscribeToRPCCounter = statsFactory.createStatsCounter(statsKey, RPC_SUBSCRIBES); - this.toDeviceRPCCallResponseCounter = statsFactory.createStatsCounter(statsKey, TO_DEVICE_RPC_CALL_RESPONSES); - this.subscriptionInfoCounter = statsFactory.createStatsCounter(statsKey, SUBSCRIPTION_INFO); - this.claimDeviceCounter = statsFactory.createStatsCounter(statsKey, DEVICE_CLAIMS); - this.deviceStateCounter = statsFactory.createStatsCounter(statsKey, DEVICE_STATES); - this.subscriptionMsgCounter = statsFactory.createStatsCounter(statsKey, SUBSCRIPTION_MSGS); - this.toCoreNotificationsCounter = statsFactory.createStatsCounter(statsKey, TO_CORE_NOTIFICATIONS); - - - counters.add(totalCounter); - counters.add(sessionEventCounter); - counters.add(getAttributesCounter); - counters.add(subscribeToAttributesCounter); - counters.add(subscribeToRPCCounter); - counters.add(toDeviceRPCCallResponseCounter); - counters.add(subscriptionInfoCounter); - counters.add(claimDeviceCounter); + this.totalCounter = register(statsFactory.createStatsCounter(statsKey, TOTAL_MSGS)); + this.sessionEventCounter = register(statsFactory.createStatsCounter(statsKey, SESSION_EVENTS)); + this.getAttributesCounter = register(statsFactory.createStatsCounter(statsKey, GET_ATTRIBUTE)); + this.subscribeToAttributesCounter = register(statsFactory.createStatsCounter(statsKey, ATTRIBUTE_SUBSCRIBES)); + this.subscribeToRPCCounter = register(statsFactory.createStatsCounter(statsKey, RPC_SUBSCRIBES)); + this.toDeviceRPCCallResponseCounter = register(statsFactory.createStatsCounter(statsKey, TO_DEVICE_RPC_CALL_RESPONSES)); + this.subscriptionInfoCounter = register(statsFactory.createStatsCounter(statsKey, SUBSCRIPTION_INFO)); + this.claimDeviceCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_CLAIMS)); + this.deviceStateCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_STATES)); + this.subscriptionMsgCounter = register(statsFactory.createStatsCounter(statsKey, SUBSCRIPTION_MSGS)); + this.toCoreNotificationsCounter = register(statsFactory.createStatsCounter(statsKey, TO_CORE_NOTIFICATIONS)); + } - counters.add(deviceStateCounter); - counters.add(subscriptionMsgCounter); - counters.add(toCoreNotificationsCounter); + private StatsCounter register(StatsCounter counter){ + counters.add(counter); + return counter; } public void log(TransportProtos.TransportToDeviceActorMsg msg) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 31d5cf47c3..02378eb557 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -36,6 +36,7 @@ import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; @@ -56,7 +57,7 @@ import java.util.function.Function; import java.util.stream.Collectors; @Slf4j -public abstract class AbstractConsumerService implements ApplicationListener { +public abstract class AbstractConsumerService extends TbApplicationEventListener { protected volatile ExecutorService consumersExecutor; protected volatile ExecutorService notificationsConsumerExecutor; diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 3087c68f88..be31a0df45 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -56,6 +56,7 @@ import org.thingsboard.server.dao.util.mapping.JacksonUtil; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; @@ -90,7 +91,7 @@ import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; @Service @TbCoreComponent @Slf4j -public class DefaultDeviceStateService implements DeviceStateService { +public class DefaultDeviceStateService extends TbApplicationEventListener implements DeviceStateService { public static final String ACTIVITY_STATE = "active"; public static final String LAST_CONNECT_TIME = "lastConnectTime"; @@ -294,7 +295,7 @@ public class DefaultDeviceStateService implements DeviceStateService { } @Override - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + protected void onTbApplicationEvent(PartitionChangeEvent partitionChangeEvent) { if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { deduplicationExecutor.submit(partitionChangeEvent.getPartitions()); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java index bafbb45ba8..844db92de8 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -48,6 +48,7 @@ import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -76,7 +77,7 @@ import java.util.function.Predicate; @Slf4j @TbCoreComponent @Service -public class DefaultSubscriptionManagerService implements SubscriptionManagerService { +public class DefaultSubscriptionManagerService extends TbApplicationEventListener implements SubscriptionManagerService { @Autowired private AttributesService attrService; @@ -178,7 +179,7 @@ public class DefaultSubscriptionManagerService implements SubscriptionManagerSer } @Override - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + protected void onTbApplicationEvent(PartitionChangeEvent partitionChangeEvent) { if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { Set removedPartitions = new HashSet<>(currentPartitions); removedPartitions.removeAll(partitionChangeEvent.getPartitions()); diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java index ee00b28562..0220a94964 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java @@ -28,6 +28,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.telemetry.sub.AlarmSubscriptionUpdate; @@ -62,6 +63,34 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer private SubscriptionManagerService subscriptionManagerService; private ExecutorService subscriptionUpdateExecutor; + + private TbApplicationEventListener partitionChangeListener = new TbApplicationEventListener<>() { + @Override + protected void onTbApplicationEvent(PartitionChangeEvent event) { + if (ServiceType.TB_CORE.equals(event.getServiceType())) { + currentPartitions.clear(); + currentPartitions.addAll(event.getPartitions()); + } + } + }; + + private TbApplicationEventListener clusterTopologyChangeListener = new TbApplicationEventListener<>() { + @Override + protected void onTbApplicationEvent(ClusterTopologyChangeEvent event) { + if (event.getServiceQueueKeys().stream().anyMatch(key -> ServiceType.TB_CORE.equals(key.getServiceType()))) { + /* + * If the cluster topology has changed, we need to push all current subscriptions to SubscriptionManagerService again. + * Otherwise, the SubscriptionManagerService may "forget" those subscriptions in case of restart. + * Although this is resource consuming operation, it is cheaper than sending ping/pong commands periodically + * It is also cheaper then caching the subscriptions by entity id and then lookup of those caches every time we have new telemetry in SubscriptionManagerService. + * Even if we cache locally the list of active subscriptions by entity id, it is still time consuming operation to get them from cache + * Since number of subscriptions is usually much less then number of devices that are pushing data. + */ + subscriptionsBySessionId.values().forEach(map -> map.values() + .forEach(sub -> pushSubscriptionToManagerService(sub, true))); + } + } + }; @PostConstruct public void initExecutor() { @@ -77,28 +106,14 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer @Override @EventListener(PartitionChangeEvent.class) - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { - if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { - currentPartitions.clear(); - currentPartitions.addAll(partitionChangeEvent.getPartitions()); - } + public void onApplicationEvent(PartitionChangeEvent event) { + partitionChangeListener.onApplicationEvent(event); } @Override @EventListener(ClusterTopologyChangeEvent.class) public void onApplicationEvent(ClusterTopologyChangeEvent event) { - if (event.getServiceQueueKeys().stream().anyMatch(key -> ServiceType.TB_CORE.equals(key.getServiceType()))) { - /* - * If the cluster topology has changed, we need to push all current subscriptions to SubscriptionManagerService again. - * Otherwise, the SubscriptionManagerService may "forget" those subscriptions in case of restart. - * Although this is resource consuming operation, it is cheaper than sending ping/pong commands periodically - * It is also cheaper then caching the subscriptions by entity id and then lookup of those caches every time we have new telemetry in SubscriptionManagerService. - * Even if we cache locally the list of active subscriptions by entity id, it is still time consuming operation to get them from cache - * Since number of subscriptions is usually much less then number of devices that are pushing data. - */ - subscriptionsBySessionId.values().forEach(map -> map.values() - .forEach(sub -> pushSubscriptionToManagerService(sub, true))); - } + clusterTopologyChangeListener.onApplicationEvent(event); } //TODO 3.1: replace null callbacks with callbacks from websocket service. diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java index 8827a71f70..168e1271fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java @@ -41,6 +41,7 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -61,7 +62,7 @@ import java.util.function.Consumer; * Created by ashvayka on 27.03.18. */ @Slf4j -public abstract class AbstractSubscriptionService implements ApplicationListener { +public abstract class AbstractSubscriptionService extends TbApplicationEventListener{ protected final Set currentPartitions = ConcurrentHashMap.newKeySet(); @@ -97,8 +98,7 @@ public abstract class AbstractSubscriptionService implements ApplicationListener } @Override - @EventListener(PartitionChangeEvent.class) - public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + protected void onTbApplicationEvent(PartitionChangeEvent partitionChangeEvent) { if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { currentPartitions.clear(); currentPartitions.addAll(partitionChangeEvent.getPartitions()); diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 6d10a74854..9170a96ea6 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -30,7 +30,7 @@ - + diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 6002e96bd0..e598b9db56 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -118,6 +118,15 @@ security: githubMapper: emailUrl: "${SECURITY_OAUTH2_GITHUB_MAPPER_EMAIL_URL_KEY:https://api.github.com/user/emails}" +# Usage statistics parameters +usage: + stats: + report: + enabled: "${USAGE_STATS_REPORT_ENABLED:true}" + interval: "${USAGE_STATS_REPORT_INTERVAL:10}" + check: + cycle: "${USAGE_STATS_CHECK_CYCLE:60000}" + # Dashboard parameters dashboard: # Maximum allowed datapoints fetched by widgets @@ -313,6 +322,9 @@ actors: cache: # caffeine or redis type: "${CACHE_TYPE:caffeine}" + attributes: + # make sure that if cache.type is 'redis' and cache.attributes.enabled is 'true' that you change 'maxmemory-policy' Redis config property to 'allkeys-lru', 'allkeys-lfu' or 'allkeys-random' + enabled: "${CACHE_ATTRIBUTES_ENABLED:true}" caffeine: specs: @@ -346,6 +358,9 @@ caffeine: deviceProfiles: timeToLiveInMinutes: 1440 maxSize: 0 + attributes: + timeToLiveInMinutes: 1440 + maxSize: 100000 redis: # standalone or cluster @@ -530,6 +545,7 @@ transport: http: enabled: "${HTTP_ENABLED:true}" request_timeout: "${HTTP_REQUEST_TIMEOUT:60000}" + max_request_timeout: "${HTTP_MAX_REQUEST_TIMEOUT:300000}" # Local MQTT transport parameters mqtt: # Enable/disable mqtt transport protocol. @@ -613,6 +629,10 @@ queue: transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1}" + consumer-stats: + enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" + kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}" aws_sqs: use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java index 8088652588..c3490aa85f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java @@ -26,4 +26,5 @@ public class CacheConstants { public static final String SECURITY_SETTINGS_CACHE = "securitySettings"; public static final String TENANT_PROFILE_CACHE = "tenantProfiles"; public static final String DEVICE_PROFILE_CACHE = "deviceProfiles"; + public static final String ATTRIBUTES_CACHE = "attributes"; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmCondition.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmCondition.java index a13e9d05cc..bbe20bdcaa 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmCondition.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmCondition.java @@ -26,7 +26,7 @@ import java.util.concurrent.TimeUnit; @JsonIgnoreProperties(ignoreUnknown = true) public class AlarmCondition { - private List condition; + private List condition; private AlarmConditionSpec spec; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilter.java new file mode 100644 index 0000000000..86aafc19e5 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilter.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.device.profile; + +import lombok.Data; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.KeyFilterPredicate; + +@Data +public class AlarmConditionFilter { + + private AlarmConditionFilterKey key; + private EntityKeyValueType valueType; + private Object value; + private KeyFilterPredicate predicate; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilterKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilterKey.java new file mode 100644 index 0000000000..33ee0b0628 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilterKey.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.device.profile; + +import lombok.Data; + +@Data +public class AlarmConditionFilterKey { + + private final AlarmConditionKeyType type; + private final String key; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionKeyType.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionKeyType.java new file mode 100644 index 0000000000..d607df8e35 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionKeyType.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.device.profile; + +public enum AlarmConditionKeyType { + ATTRIBUTE, + TIME_SERIES, + ENTITY_FIELD, + CONSTANT +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java index 3d46f8d2ab..2fa0a4d36a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java @@ -17,17 +17,23 @@ package org.thingsboard.server.common.data.query; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; -import lombok.Getter; +import lombok.RequiredArgsConstructor; @Data +@RequiredArgsConstructor public class DynamicValue { @JsonIgnore private T resolvedValue; - @Getter private final DynamicValueSourceType sourceType; - @Getter private final String sourceAttribute; + private final boolean inherit; + + public DynamicValue(DynamicValueSourceType sourceType, String sourceAttribute) { + this.sourceAttribute = sourceAttribute; + this.sourceType = sourceType; + this.inherit = false; + } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 348e8021e4..66a3670ddd 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -120,7 +120,7 @@ public final class TbMsg implements Serializable { private TbMsg(String queueName, UUID id, long ts, String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId, int ruleNodeExecCounter, TbMsgCallback callback) { this.id = id; - this.queueName = queueName; + this.queueName = queueName != null ? queueName : ServiceQueue.MAIN; if (ts > 0) { this.ts = ts; } else { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java index 0602dee3ae..1e5b90b5fe 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java @@ -22,7 +22,9 @@ import org.thingsboard.server.common.msg.queue.ServiceQueueKey; import java.util.Set; -public class ClusterTopologyChangeEvent extends ApplicationEvent { +public class ClusterTopologyChangeEvent extends TbApplicationEvent { + + private static final long serialVersionUID = -2441739930040282254L; @Getter private final Set serviceQueueKeys; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index d8164f4be8..2da438417a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -126,7 +126,7 @@ public class HashPartitionService implements PartitionService { } @Override - public void recalculatePartitions(ServiceInfo currentService, List otherServices) { + public synchronized void recalculatePartitions(ServiceInfo currentService, List otherServices) { logServiceInfo(currentService); otherServices.forEach(this::logServiceInfo); Map> queueServicesMap = new HashMap<>(); @@ -134,7 +134,7 @@ public class HashPartitionService implements PartitionService { for (ServiceInfo other : otherServices) { addNode(queueServicesMap, other); } - queueServicesMap.values().forEach(list -> list.sort((a, b) -> a.getServiceId().compareTo(b.getServiceId()))); + queueServicesMap.values().forEach(list -> list.sort(Comparator.comparing(ServiceInfo::getServiceId))); ConcurrentMap> oldPartitions = myPartitions; TenantId myIsolatedOrSystemTenantId = getSystemOrIsolatedTenantId(currentService); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java index e4edabbe19..2edcd2ceca 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java @@ -24,7 +24,9 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.Set; -public class PartitionChangeEvent extends ApplicationEvent { +public class PartitionChangeEvent extends TbApplicationEvent { + + private static final long serialVersionUID = -8731788167026510559L; @Getter private final ServiceQueueKey serviceQueueKey; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEvent.java new file mode 100644 index 0000000000..face2d36d6 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEvent.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import lombok.Getter; +import org.springframework.context.ApplicationEvent; + +import java.util.concurrent.atomic.AtomicInteger; + +public class TbApplicationEvent extends ApplicationEvent { + + private static final long serialVersionUID = 3884264064887765146L; + + private static final AtomicInteger sequence = new AtomicInteger(); + + @Getter + private final int sequenceNumber; + + public TbApplicationEvent(Object source) { + super(source); + sequenceNumber = sequence.incrementAndGet(); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java new file mode 100644 index 0000000000..9158d8f0c8 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationListener; + +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +@Slf4j +public abstract class TbApplicationEventListener implements ApplicationListener { + + private int lastProcessedSequenceNumber = Integer.MIN_VALUE; + private final Lock seqNumberLock = new ReentrantLock(); + + @Override + public void onApplicationEvent(T event) { + boolean validUpdate = false; + seqNumberLock.lock(); + try { + if (event.getSequenceNumber() > lastProcessedSequenceNumber) { + validUpdate = true; + lastProcessedSequenceNumber = event.getSequenceNumber(); + } + } finally { + seqNumberLock.unlock(); + } + if (validUpdate) { + onTbApplicationEvent(event); + } else { + log.info("Application event ignored due to invalid sequence number ({} > {}). Event: {}", lastProcessedSequenceNumber, event.getSequenceNumber(), event); + } + } + + protected abstract void onTbApplicationEvent(T event); + + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatisticConfig.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatisticConfig.java new file mode 100644 index 0000000000..bfe7d29bb6 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatisticConfig.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.kafka; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +@Component +@ConditionalOnProperty(prefix = "queue", value = "type", havingValue = "kafka") +@Getter +@AllArgsConstructor +@NoArgsConstructor +public class TbKafkaConsumerStatisticConfig { + @Value("${queue.kafka.consumer-stats.enabled:true}") + private Boolean enabled; + @Value("${queue.kafka.consumer-stats.print-interval-ms:60000}") + private Long printIntervalMs; + @Value("${queue.kafka.consumer-stats.kafka-response-timeout-ms:1000}") + private Long kafkaResponseTimeoutMs; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java new file mode 100644 index 0000000000..0ccf9ba42c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java @@ -0,0 +1,184 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.kafka; + +import lombok.Builder; +import lombok.Data; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.TopicPartition; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.queue.discovery.PartitionService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +@Slf4j +@Component +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "queue", value = "type", havingValue = "kafka") +public class TbKafkaConsumerStatsService { + private final Set monitoredGroups = ConcurrentHashMap.newKeySet(); + + private final TbKafkaSettings kafkaSettings; + private final TbKafkaConsumerStatisticConfig statsConfig; + private final PartitionService partitionService; + + private AdminClient adminClient; + private Consumer consumer; + private ScheduledExecutorService statsPrintScheduler; + + @PostConstruct + public void init() { + if (!statsConfig.getEnabled()) { + return; + } + this.adminClient = AdminClient.create(kafkaSettings.toAdminProps()); + this.statsPrintScheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("kafka-consumer-stats")); + + Properties consumerProps = kafkaSettings.toConsumerProps(); + consumerProps.put(ConsumerConfig.CLIENT_ID_CONFIG, "consumer-stats-loader-client"); + consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, "consumer-stats-loader-client-group"); + this.consumer = new KafkaConsumer<>(consumerProps); + + startLogScheduling(); + } + + private void startLogScheduling() { + Duration timeoutDuration = Duration.ofMillis(statsConfig.getKafkaResponseTimeoutMs()); + statsPrintScheduler.scheduleWithFixedDelay(() -> { + if (!isStatsPrintRequired()) { + return; + } + for (String groupId : monitoredGroups) { + try { + Map groupOffsets = adminClient.listConsumerGroupOffsets(groupId).partitionsToOffsetAndMetadata() + .get(statsConfig.getKafkaResponseTimeoutMs(), TimeUnit.MILLISECONDS); + Map endOffsets = consumer.endOffsets(groupOffsets.keySet(), timeoutDuration); + + List lagTopicsStats = getTopicsStatsWithLag(groupOffsets, endOffsets); + if (!lagTopicsStats.isEmpty()) { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < lagTopicsStats.size(); i++) { + builder.append(lagTopicsStats.get(i).toString()); + if (i != lagTopicsStats.size() - 1) { + builder.append(", "); + } + } + log.info("[{}] Topic partitions with lag: [{}].", groupId, builder.toString()); + } + } catch (Exception e) { + log.warn("[{}] Failed to get consumer group stats. Reason - {}.", groupId, e.getMessage()); + log.trace("Detailed error: ", e); + } + } + + }, statsConfig.getPrintIntervalMs(), statsConfig.getPrintIntervalMs(), TimeUnit.MILLISECONDS); + } + + private boolean isStatsPrintRequired() { + boolean isMyRuleEnginePartition = partitionService.resolve(ServiceType.TB_RULE_ENGINE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID).isMyPartition(); + boolean isMyCorePartition = partitionService.resolve(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID).isMyPartition(); + return log.isInfoEnabled() && (isMyRuleEnginePartition || isMyCorePartition); + } + + private List getTopicsStatsWithLag(Map groupOffsets, Map endOffsets) { + List consumerGroupStats = new ArrayList<>(); + for (TopicPartition topicPartition : groupOffsets.keySet()) { + long endOffset = endOffsets.get(topicPartition); + long committedOffset = groupOffsets.get(topicPartition).offset(); + long lag = endOffset - committedOffset; + if (lag != 0) { + GroupTopicStats groupTopicStats = GroupTopicStats.builder() + .topic(topicPartition.topic()) + .partition(topicPartition.partition()) + .committedOffset(committedOffset) + .endOffset(endOffset) + .lag(lag) + .build(); + consumerGroupStats.add(groupTopicStats); + } + } + return consumerGroupStats; + } + + public void registerClientGroup(String groupId) { + if (statsConfig.getEnabled() && !StringUtils.isEmpty(groupId)) { + monitoredGroups.add(groupId); + } + } + + public void unregisterClientGroup(String groupId) { + if (statsConfig.getEnabled() && !StringUtils.isEmpty(groupId)) { + monitoredGroups.remove(groupId); + } + } + + @PreDestroy + public void destroy() { + if (statsPrintScheduler != null) { + statsPrintScheduler.shutdownNow(); + } + if (adminClient != null) { + adminClient.close(); + } + if (consumer != null) { + consumer.close(); + } + } + + + @Builder + @Data + private static class GroupTopicStats { + private String topic; + private int partition; + private long committedOffset; + private long endOffset; + private long lag; + + @Override + public String toString() { + return "[" + + "topic=[" + topic + ']' + + ", partition=[" + partition + "]" + + ", committedOffset=[" + committedOffset + "]" + + ", endOffset=[" + endOffset + "]" + + ", lag=[" + lag + "]" + + "]"; + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 8743f4d6cc..b1b3d5ce05 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -42,10 +42,13 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue private final KafkaConsumer consumer; private final TbKafkaDecoder decoder; + private final TbKafkaConsumerStatsService statsService; + private final String groupId; + @Builder private TbKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, String clientId, String groupId, String topic, - TbQueueAdmin admin) { + TbQueueAdmin admin, TbKafkaConsumerStatsService statsService) { super(topic); Properties props = settings.toConsumerProps(); props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); @@ -53,6 +56,13 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); } + this.statsService = statsService; + this.groupId = groupId; + + if (statsService != null) { + statsService.registerClientGroup(groupId); + } + this.admin = admin; this.consumer = new KafkaConsumer<>(props); this.decoder = decoder; @@ -96,6 +106,8 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue consumer.unsubscribe(); consumer.close(); } + if (statsService != null) { + statsService.unregisterClientGroup(groupId); + } } - } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index eebffc37b0..e07f3e43d3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -39,6 +39,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; @@ -65,6 +66,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -79,6 +81,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbKafkaConsumerStatsService consumerStatsService, TbKafkaTopicConfigs kafkaTopicConfigs) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; @@ -88,6 +91,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.jsInvokeSettings = jsInvokeSettings; + this.consumerStatsService = consumerStatsService; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -156,6 +160,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.groupId("re-" + queueName + "-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(ruleEngineAdmin); + consumerBuilder.statsService(consumerStatsService); return consumerBuilder.build(); } @@ -168,6 +173,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.groupId("monolith-rule-engine-notifications-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(notificationAdmin); + consumerBuilder.statsService(consumerStatsService); return consumerBuilder.build(); } @@ -180,6 +186,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.groupId("monolith-core-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(coreAdmin); + consumerBuilder.statsService(consumerStatsService); return consumerBuilder.build(); } @@ -192,6 +199,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.groupId("monolith-core-notifications-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(notificationAdmin); + consumerBuilder.statsService(consumerStatsService); return consumerBuilder.build(); } @@ -204,6 +212,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.groupId("monolith-transport-api-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(transportApiAdmin); + consumerBuilder.statsService(consumerStatsService); return consumerBuilder.build(); } @@ -237,6 +246,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); } ); + responseBuilder.statsService(consumerStatsService); responseBuilder.admin(jsExecutorAdmin); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder @@ -259,6 +269,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.groupId("monolith-us-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(coreAdmin); + consumerBuilder.statsService(consumerStatsService); return consumerBuilder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index 15771a3b78..24e80adec5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -39,6 +39,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; @@ -62,6 +63,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -75,6 +77,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbKafkaConsumerStatsService consumerStatsService, TbKafkaTopicConfigs kafkaTopicConfigs) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; @@ -83,6 +86,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.jsInvokeSettings = jsInvokeSettings; + this.consumerStatsService = consumerStatsService; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -150,6 +154,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { consumerBuilder.groupId("tb-core-node"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(coreAdmin); + consumerBuilder.statsService(consumerStatsService); return consumerBuilder.build(); } @@ -162,6 +167,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { consumerBuilder.groupId("tb-core-notifications-node-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(notificationAdmin); + consumerBuilder.statsService(consumerStatsService); return consumerBuilder.build(); } @@ -174,6 +180,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { consumerBuilder.groupId("tb-core-transport-api-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(transportApiAdmin); + consumerBuilder.statsService(consumerStatsService); return consumerBuilder.build(); } @@ -208,6 +215,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { } ); responseBuilder.admin(jsExecutorAdmin); + responseBuilder.statsService(consumerStatsService); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); @@ -229,6 +237,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { consumerBuilder.groupId("tb-core-us-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(coreAdmin); + consumerBuilder.statsService(consumerStatsService); return consumerBuilder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index b45ec62b04..a8247dc11e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -37,6 +37,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; @@ -59,6 +60,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueCoreSettings coreSettings; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -70,6 +72,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbKafkaConsumerStatsService consumerStatsService, TbKafkaTopicConfigs kafkaTopicConfigs) { this.partitionService = partitionService; this.kafkaSettings = kafkaSettings; @@ -77,6 +80,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; this.jsInvokeSettings = jsInvokeSettings; + this.consumerStatsService = consumerStatsService; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -145,6 +149,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { consumerBuilder.groupId("re-" + queueName + "-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(ruleEngineAdmin); + consumerBuilder.statsService(consumerStatsService); return consumerBuilder.build(); } @@ -157,6 +162,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { consumerBuilder.groupId("tb-rule-engine-notifications-node-" + serviceInfoProvider.getServiceId()); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(notificationAdmin); + consumerBuilder.statsService(consumerStatsService); return consumerBuilder.build(); } @@ -181,6 +187,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { } ); responseBuilder.admin(jsExecutorAdmin); + responseBuilder.statsService(consumerStatsService); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java index 0c64a57304..8c0c6999d5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java @@ -32,6 +32,7 @@ import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; @@ -54,6 +55,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -66,6 +68,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, + TbKafkaConsumerStatsService consumerStatsService, TbKafkaTopicConfigs kafkaTopicConfigs) { this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -73,6 +76,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; + this.consumerStatsService = consumerStatsService; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -95,6 +99,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { responseBuilder.groupId("transport-node-" + serviceInfoProvider.getServiceId()); responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); responseBuilder.admin(transportApiAdmin); + responseBuilder.statsService(consumerStatsService); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); @@ -136,6 +141,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { responseBuilder.groupId("transport-node-" + serviceInfoProvider.getServiceId()); responseBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); responseBuilder.admin(notificationAdmin); + responseBuilder.statsService(consumerStatsService); return responseBuilder.build(); } diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java index aad01c7c2f..13f38bfc80 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java @@ -17,9 +17,13 @@ package org.thingsboard.server.transport.http; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.coyote.ProtocolHandler; +import org.apache.coyote.http11.Http11NioProtocol; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.web.embedded.tomcat.TomcatConnectorCustomizer; +import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.transport.TransportContext; @@ -37,4 +41,18 @@ public class HttpTransportContext extends TransportContext { @Value("${transport.http.request_timeout}") private long defaultTimeout; + @Getter + @Value("${transport.http.max_request_timeout}") + private long maxRequestTimeout; + + @Bean + public TomcatConnectorCustomizer tomcatAsyncTimeoutConnectorCustomizer() { + return connector -> { + ProtocolHandler handler = connector.getProtocolHandler(); + if (handler instanceof Http11NioProtocol) { + log.trace("Setting async max request timeout {}", maxRequestTimeout); + connector.setAsyncTimeout(maxRequestTimeout); + } + }; + } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java index 362be1ed00..4bb1b96b0a 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java @@ -51,6 +51,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ValidateBasicMqttCre import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -245,12 +246,25 @@ public class JsonConverter { } private static void parseNumericValue(List result, Entry valueEntry, JsonPrimitive value) { - if (value.getAsString().contains(".")) { - result.add(new DoubleDataEntry(valueEntry.getKey(), value.getAsDouble())); + String valueAsString = value.getAsString(); + String key = valueEntry.getKey(); + if (valueAsString.contains("e") || valueAsString.contains("E")) { + var bd = new BigDecimal(valueAsString); + if (bd.stripTrailingZeros().scale() <= 0) { + try { + result.add(new LongDataEntry(key, bd.longValueExact())); + } catch (ArithmeticException e) { + result.add(new DoubleDataEntry(key, bd.doubleValue())); + } + } else { + result.add(new DoubleDataEntry(key, bd.doubleValue())); + } + } else if (valueAsString.contains(".")) { + result.add(new DoubleDataEntry(key, value.getAsDouble())); } else { try { long longValue = Long.parseLong(value.getAsString()); - result.add(new LongDataEntry(valueEntry.getKey(), longValue)); + result.add(new LongDataEntry(key, longValue)); } catch (NumberFormatException e) { throw new JsonSyntaxException("Big integer values are not supported!"); } @@ -285,7 +299,8 @@ public class JsonConverter { return result; } - public static JsonObject getJsonObjectForGateway(String deviceName, TransportProtos.GetAttributeResponseMsg responseMsg) { + public static JsonObject getJsonObjectForGateway(String deviceName, TransportProtos.GetAttributeResponseMsg + responseMsg) { JsonObject result = new JsonObject(); result.addProperty("id", responseMsg.getRequestId()); result.addProperty(DEVICE_PROPERTY, deviceName); @@ -298,7 +313,8 @@ public class JsonConverter { return result; } - public static JsonObject getJsonObjectForGateway(String deviceName, AttributeUpdateNotificationMsg notificationMsg) { + public static JsonObject getJsonObjectForGateway(String deviceName, AttributeUpdateNotificationMsg + notificationMsg) { JsonObject result = new JsonObject(); result.addProperty(DEVICE_PROPERTY, deviceName); result.add("data", toJson(notificationMsg)); @@ -446,7 +462,8 @@ public class JsonConverter { return result; } - public static JsonElement toGatewayJson(String deviceName, TransportProtos.ProvisionDeviceResponseMsg responseRequest) { + public static JsonElement toGatewayJson(String deviceName, TransportProtos.ProvisionDeviceResponseMsg + responseRequest) { JsonObject result = new JsonObject(); result.addProperty(DEVICE_PROPERTY, deviceName); result.add("data", JsonConverter.toJson(responseRequest)); @@ -496,15 +513,18 @@ public class JsonConverter { return result; } - public static Map> convertToTelemetry(JsonElement jsonElement, long systemTs) throws JsonSyntaxException { + public static Map> convertToTelemetry(JsonElement jsonElement, long systemTs) throws + JsonSyntaxException { return convertToTelemetry(jsonElement, systemTs, false); } - public static Map> convertToSortedTelemetry(JsonElement jsonElement, long systemTs) throws JsonSyntaxException { + public static Map> convertToSortedTelemetry(JsonElement jsonElement, long systemTs) throws + JsonSyntaxException { return convertToTelemetry(jsonElement, systemTs, true); } - public static Map> convertToTelemetry(JsonElement jsonElement, long systemTs, boolean sorted) throws JsonSyntaxException { + public static Map> convertToTelemetry(JsonElement jsonElement, long systemTs, boolean sorted) throws + JsonSyntaxException { Map> result = sorted ? new TreeMap<>() : new HashMap<>(); convertToTelemetry(jsonElement, systemTs, result, null); return result; @@ -574,7 +594,8 @@ public class JsonConverter { .build(); } - private static TransportProtos.ProvisionDeviceCredentialsMsg buildProvisionDeviceCredentialsMsg(String provisionKey, String provisionSecret) { + private static TransportProtos.ProvisionDeviceCredentialsMsg buildProvisionDeviceCredentialsMsg(String + provisionKey, String provisionSecret) { return TransportProtos.ProvisionDeviceCredentialsMsg.newBuilder() .setProvisionDeviceKey(provisionKey) .setProvisionDeviceSecret(provisionSecret) diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 8ef0187342..74dabe7cb6 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -763,8 +763,8 @@ public class DefaultTransportService implements TransportService { wrappedCallback); } - protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator()); + private void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tbMsg.getQueueName(), tenantId, tbMsg.getOriginator()); if (log.isTraceEnabled()) { log.trace("[{}][{}] Pushing to topic {} message {}", tenantId, tbMsg.getOriginator(), tpi.getFullTopicName(), tbMsg); } @@ -776,7 +776,7 @@ public class DefaultTransportService implements TransportService { ruleEngineMsgProducer.send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), wrappedCallback); } - protected void sendToRuleEngine(TenantId tenantId, DeviceId deviceId, TransportProtos.SessionInfoProto sessionInfo, JsonObject json, + private void sendToRuleEngine(TenantId tenantId, DeviceId deviceId, TransportProtos.SessionInfoProto sessionInfo, JsonObject json, TbMsgMetaData metaData, SessionMsgType sessionMsgType, TbQueueCallback callback) { DeviceProfileId deviceProfileId = new DeviceProfileId(new UUID(sessionInfo.getDeviceProfileIdMSB(), sessionInfo.getDeviceProfileIdLSB())); DeviceProfile deviceProfile = deviceProfileCache.get(deviceProfileId); diff --git a/common/transport/transport-api/src/test/java/JsonConverterTest.java b/common/transport/transport-api/src/test/java/JsonConverterTest.java new file mode 100644 index 0000000000..cedbef50c9 --- /dev/null +++ b/common/transport/transport-api/src/test/java/JsonConverterTest.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.google.gson.JsonParser; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; +import org.thingsboard.server.common.transport.adaptor.JsonConverter; + +@RunWith(MockitoJUnitRunner.class) +public class JsonConverterTest { + + private static final JsonParser JSON_PARSER = new JsonParser(); + + @Test + public void testParseBigDecimalAsLong() { + var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 1E+1}"), 0L); + Assert.assertEquals(10L, result.get(0L).get(0).getLongValue().get().longValue()); + } + + @Test + public void testParseBigDecimalAsDouble() { + var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 101E-1}"), 0L); + Assert.assertEquals(10.1, result.get(0L).get(0).getDoubleValue().get(), 0.0); + } + + @Test + public void testParseAsDouble() { + var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 1.1}"), 0L); + Assert.assertEquals(1.1, result.get(0L).get(0).getDoubleValue().get(), 0.0); + } + + @Test + public void testParseAsLong() { + var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 11}"), 0L); + Assert.assertEquals(11L, result.get(0L).get(0).getLongValue().get().longValue()); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCacheKey.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCacheKey.java new file mode 100644 index 0000000000..10771656e9 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCacheKey.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.attributes; + +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; + +import java.io.Serializable; + +@EqualsAndHashCode +@Getter +@AllArgsConstructor +public class AttributeCacheKey implements Serializable { + private static final long serialVersionUID = 2013369077925351881L; + + private final String scope; + private final EntityId entityId; + private final String key; + + @Override + public String toString() { + return entityId + "_" + scope + "_" + key; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java new file mode 100644 index 0000000000..b1fc82fe88 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.attributes; + +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.dao.exception.IncorrectParameterException; +import org.thingsboard.server.dao.service.Validator; + +public class AttributeUtils { + public static void validate(EntityId id, String scope) { + Validator.validateId(id.getId(), "Incorrect id " + id); + Validator.validateString(scope, "Incorrect scope " + scope); + } + + public static void validate(AttributeKvEntry kvEntry) { + if (kvEntry == null) { + throw new IncorrectParameterException("Key value entry can't be null"); + } else if (kvEntry.getDataType() == null) { + throw new IncorrectParameterException("Incorrect kvEntry. Data type can't be null"); + } else { + Validator.validateString(kvEntry.getKey(), "Incorrect kvEntry. Key can't be empty"); + Validator.validatePositiveNumber(kvEntry.getLastUpdateTs(), "Incorrect last update ts. Ts should be positive"); + } + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesCacheWrapper.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesCacheWrapper.java new file mode 100644 index 0000000000..196204cda1 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesCacheWrapper.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.attributes; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.Cache; +import org.springframework.cache.CacheManager; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; + +import static org.thingsboard.server.common.data.CacheConstants.ATTRIBUTES_CACHE; + +@Service +@ConditionalOnProperty(prefix = "cache.attributes", value = "enabled", havingValue = "true") +@Primary +@Slf4j +public class AttributesCacheWrapper { + private final Cache attributesCache; + + public AttributesCacheWrapper(CacheManager cacheManager) { + this.attributesCache = cacheManager.getCache(ATTRIBUTES_CACHE); + } + + public Cache.ValueWrapper get(AttributeCacheKey attributeCacheKey) { + try { + return attributesCache.get(attributeCacheKey); + } catch (Exception e) { + log.debug("Failed to retrieve element from cache for key {}. Reason - {}.", attributeCacheKey, e.getMessage()); + return null; + } + } + + public void put(AttributeCacheKey attributeCacheKey, AttributeKvEntry attributeKvEntry) { + try { + attributesCache.put(attributeCacheKey, attributeKvEntry); + } catch (Exception e) { + log.debug("Failed to put element from cache for key {}. Reason - {}.", attributeCacheKey, e.getMessage()); + } + } + + public void evict(AttributeCacheKey attributeCacheKey) { + try { + attributesCache.evict(attributeCacheKey); + } catch (Exception e) { + log.debug("Failed to evict element from cache for key {}. Reason - {}.", attributeCacheKey, e.getMessage()); + } + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index e70604353a..3eee300627 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -15,31 +15,39 @@ */ package org.thingsboard.server.dao.attributes; -import com.google.common.collect.Lists; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import org.springframework.beans.factory.annotation.Autowired; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.service.Validator; import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; + +import static org.thingsboard.server.dao.attributes.AttributeUtils.validate; /** * @author Andrew Shvayka */ @Service +@ConditionalOnProperty(prefix = "cache.attributes", value = "enabled", havingValue = "false", matchIfMissing = true) +@Primary +@Slf4j public class BaseAttributesService implements AttributesService { + private final AttributesDao attributesDao; - @Autowired - private AttributesDao attributesDao; + public BaseAttributesService(AttributesDao attributesDao) { + this.attributesDao = attributesDao; + } @Override public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, String attributeKey) { @@ -75,33 +83,14 @@ public class BaseAttributesService implements AttributesService { public ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes) { validate(entityId, scope); attributes.forEach(attribute -> validate(attribute)); - List> futures = Lists.newArrayListWithExpectedSize(attributes.size()); - for (AttributeKvEntry attribute : attributes) { - futures.add(attributesDao.save(tenantId, entityId, scope, attribute)); - } - return Futures.allAsList(futures); + + List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); + return Futures.allAsList(saveFutures); } @Override - public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String scope, List keys) { + public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String scope, List attributeKeys) { validate(entityId, scope); - return attributesDao.removeAll(tenantId, entityId, scope, keys); - } - - private static void validate(EntityId id, String scope) { - Validator.validateId(id.getId(), "Incorrect id " + id); - Validator.validateString(scope, "Incorrect scope " + scope); - } - - private static void validate(AttributeKvEntry kvEntry) { - if (kvEntry == null) { - throw new IncorrectParameterException("Key value entry can't be null"); - } else if (kvEntry.getDataType() == null) { - throw new IncorrectParameterException("Incorrect kvEntry. Data type can't be null"); - } else { - Validator.validateString(kvEntry.getKey(), "Incorrect kvEntry. Key can't be empty"); - Validator.validatePositiveNumber(kvEntry.getLastUpdateTs(), "Incorrect last update ts. Ts should be positive"); - } + return attributesDao.removeAll(tenantId, entityId, scope, attributeKeys); } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java new file mode 100644 index 0000000000..87ddfeee9a --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -0,0 +1,193 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.attributes; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.Cache; +import org.springframework.cache.CacheManager; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.stats.DefaultCounter; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.dao.service.Validator; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.thingsboard.server.common.data.CacheConstants.ATTRIBUTES_CACHE; +import static org.thingsboard.server.dao.attributes.AttributeUtils.validate; + +@Service +@ConditionalOnProperty(prefix = "cache.attributes", value = "enabled", havingValue = "true") +@Primary +@Slf4j +public class CachedAttributesService implements AttributesService { + private static final String STATS_NAME = "attributes.cache"; + + private final AttributesDao attributesDao; + private final AttributesCacheWrapper cacheWrapper; + private final DefaultCounter hitCounter; + private final DefaultCounter missCounter; + + public CachedAttributesService(AttributesDao attributesDao, + AttributesCacheWrapper cacheWrapper, + StatsFactory statsFactory) { + this.attributesDao = attributesDao; + this.cacheWrapper = cacheWrapper; + + this.hitCounter = statsFactory.createDefaultCounter(STATS_NAME, "result", "hit"); + this.missCounter = statsFactory.createDefaultCounter(STATS_NAME, "result", "miss"); + } + + @Override + public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, String attributeKey) { + validate(entityId, scope); + Validator.validateString(attributeKey, "Incorrect attribute key " + attributeKey); + + AttributeCacheKey attributeCacheKey = new AttributeCacheKey(scope, entityId, attributeKey); + Cache.ValueWrapper cachedAttributeValue = cacheWrapper.get(attributeCacheKey); + if (cachedAttributeValue != null) { + hitCounter.increment(); + AttributeKvEntry cachedAttributeKvEntry = (AttributeKvEntry) cachedAttributeValue.get(); + return Futures.immediateFuture(Optional.ofNullable(cachedAttributeKvEntry)); + } else { + missCounter.increment(); + ListenableFuture> result = attributesDao.find(tenantId, entityId, scope, attributeKey); + return Futures.transform(result, foundAttrKvEntry -> { + // TODO: think if it's a good idea to store 'empty' attributes + cacheWrapper.put(attributeCacheKey, foundAttrKvEntry.orElse(null)); + return foundAttrKvEntry; + }, MoreExecutors.directExecutor()); + } + } + + @Override + public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, Collection attributeKeys) { + validate(entityId, scope); + attributeKeys.forEach(attributeKey -> Validator.validateString(attributeKey, "Incorrect attribute key " + attributeKey)); + + Map wrappedCachedAttributes = findCachedAttributes(entityId, scope, attributeKeys); + + List cachedAttributes = wrappedCachedAttributes.values().stream() + .map(wrappedCachedAttribute -> (AttributeKvEntry) wrappedCachedAttribute.get()) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + if (wrappedCachedAttributes.size() == attributeKeys.size()) { + return Futures.immediateFuture(cachedAttributes); + } + + Set notFoundAttributeKeys = new HashSet<>(attributeKeys); + notFoundAttributeKeys.removeAll(wrappedCachedAttributes.keySet()); + + ListenableFuture> result = attributesDao.find(tenantId, entityId, scope, notFoundAttributeKeys); + return Futures.transform(result, foundInDbAttributes -> mergeDbAndCacheAttributes(entityId, scope, cachedAttributes, notFoundAttributeKeys, foundInDbAttributes), MoreExecutors.directExecutor()); + + } + + private Map findCachedAttributes(EntityId entityId, String scope, Collection attributeKeys) { + Map cachedAttributes = new HashMap<>(); + for (String attributeKey : attributeKeys) { + Cache.ValueWrapper cachedAttributeValue = cacheWrapper.get(new AttributeCacheKey(scope, entityId, attributeKey)); + if (cachedAttributeValue != null) { + hitCounter.increment(); + cachedAttributes.put(attributeKey, cachedAttributeValue); + } else { + missCounter.increment(); + } + } + return cachedAttributes; + } + + private List mergeDbAndCacheAttributes(EntityId entityId, String scope, List cachedAttributes, Set notFoundAttributeKeys, List foundInDbAttributes) { + for (AttributeKvEntry foundInDbAttribute : foundInDbAttributes) { + AttributeCacheKey attributeCacheKey = new AttributeCacheKey(scope, entityId, foundInDbAttribute.getKey()); + cacheWrapper.put(attributeCacheKey, foundInDbAttribute); + notFoundAttributeKeys.remove(foundInDbAttribute.getKey()); + } + for (String key : notFoundAttributeKeys){ + cacheWrapper.put(new AttributeCacheKey(scope, entityId, key), null); + } + List mergedAttributes = new ArrayList<>(cachedAttributes); + mergedAttributes.addAll(foundInDbAttributes); + return mergedAttributes; + } + + @Override + public ListenableFuture> findAll(TenantId tenantId, EntityId entityId, String scope) { + validate(entityId, scope); + return attributesDao.findAll(tenantId, entityId, scope); + } + + @Override + public List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId) { + return attributesDao.findAllKeysByDeviceProfileId(tenantId, deviceProfileId); + } + + @Override + public List findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List entityIds) { + return attributesDao.findAllKeysByEntityIds(tenantId, entityType, entityIds); + } + + @Override + public ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes) { + validate(entityId, scope); + attributes.forEach(AttributeUtils::validate); + + List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); + ListenableFuture> future = Futures.allAsList(saveFutures); + + // TODO: can do if (attributesCache.get() != null) attributesCache.put() instead, but will be more twice more requests to cache + List attributeKeys = attributes.stream().map(KvEntry::getKey).collect(Collectors.toList()); + future.addListener(() -> evictAttributesFromCache(tenantId, entityId, scope, attributeKeys), MoreExecutors.directExecutor()); + return future; + } + + @Override + public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String scope, List attributeKeys) { + validate(entityId, scope); + ListenableFuture> future = attributesDao.removeAll(tenantId, entityId, scope, attributeKeys); + future.addListener(() -> evictAttributesFromCache(tenantId, entityId, scope, attributeKeys), MoreExecutors.directExecutor()); + return future; + } + + private void evictAttributesFromCache(TenantId tenantId, EntityId entityId, String scope, List attributeKeys) { + try { + for (String attributeKey : attributeKeys) { + cacheWrapper.evict(new AttributeCacheKey(scope, entityId, attributeKey)); + } + } catch (Exception e) { + log.error("[{}][{}] Failed to remove values from cache.", tenantId, entityId, e); + } + } +} diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 39fd5c1db1..6c537a0d55 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -2188,7 +2188,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { Map params = new HashMap<>(); addPageLinkToParam(params, pageLink); return restTemplate.exchange( - baseURL + "/api/tenantProfiles" + getUrlParams(pageLink), + baseURL + "/api/tenantProfiles?" + getUrlParams(pageLink), HttpMethod.GET, HttpEntity.EMPTY, new ParameterizedTypeReference>() { @@ -2199,7 +2199,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { Map params = new HashMap<>(); addPageLinkToParam(params, pageLink); return restTemplate.exchange( - baseURL + "/api/tenantProfileInfos" + getUrlParams(pageLink), + baseURL + "/api/tenantProfileInfos?" + getUrlParams(pageLink), HttpMethod.GET, HttpEntity.EMPTY, new ParameterizedTypeReference>() { @@ -2256,7 +2256,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { Map params = new HashMap<>(); addPageLinkToParam(params, pageLink); return restTemplate.exchange( - baseURL + "/api/users" + getUrlParams(pageLink), + baseURL + "/api/users?" + getUrlParams(pageLink), HttpMethod.GET, HttpEntity.EMPTY, new ParameterizedTypeReference>() { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java index 0a97be3bd0..54be64b7fa 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java @@ -56,7 +56,7 @@ import java.security.spec.PKCS8EncodedKeySpec; public class CertPemCredentials implements ClientCredentials { private static final String TLS_VERSION = "TLSv1.2"; - private String caCert; + protected String caCert; private String cert; private String privateKey; private String password; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java index 69482a7b89..6047370193 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java @@ -104,9 +104,10 @@ public class TbMsgGeneratorNode implements TbNode { } }, t -> { - if (initialized) { + if (initialized && (config.getMsgCount() == TbMsgGeneratorNodeConfiguration.UNLIMITED_MSG_COUNT || currentMsgCount < config.getMsgCount())) { ctx.tellFailure(msg, t); scheduleTickMsg(ctx); + currentMsgCount++; } }); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/CalculateDeltaNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/CalculateDeltaNode.java index 1d5ae744c3..63b326a816 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/CalculateDeltaNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/CalculateDeltaNode.java @@ -45,11 +45,13 @@ import java.util.concurrent.ConcurrentHashMap; @Slf4j @RuleNode(type = ComponentType.ENRICHMENT, - name = "calculate delta", + name = "calculate delta", relationTypes = {"Success", "Failure", "Other"}, configClazz = CalculateDeltaNodeConfiguration.class, nodeDescription = "Calculates and adds 'delta' value into message based on the incoming and previous value", nodeDetails = "Calculates delta and period based on the previous time-series reading and current data. " + - "Delta calculation is done in scope of the message originator, e.g. device, asset or customer.", + "Delta calculation is done in scope of the message originator, e.g. device, asset or customer. " + + "If there is input key, the output relation will be 'Success' unless delta is negative and corresponding configuration parameter is set. " + + "If there is no input value key in the incoming message, the output relation will be 'Other'.", uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = "tbEnrichmentNodeCalculateDeltaConfig") public class CalculateDeltaNode implements TbNode { @@ -93,12 +95,17 @@ public class CalculateDeltaNode implements TbNode { return; } + if (config.getRound() != null) { delta = delta.setScale(config.getRound(), RoundingMode.HALF_UP); } ObjectNode result = (ObjectNode) json; - result.put(config.getOutputValueKey(), delta); + if (delta.stripTrailingZeros().scale() > 0) { + result.put(config.getOutputValueKey(), delta.doubleValue()); + } else { + result.put(config.getOutputValueKey(), delta.longValueExact()); + } if (config.isAddPeriodBetweenMsgs()) { long period = previousData != null ? currentTs - previousData.ts : 0; @@ -107,13 +114,11 @@ public class CalculateDeltaNode implements TbNode { ctx.tellSuccess(TbMsg.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), JacksonUtil.toString(result))); }, t -> ctx.tellFailure(msg, t), ctx.getDbCallbackExecutor()); - } else if (config.isTellFailureIfInputValueKeyIsAbsent()) { - ctx.tellNext(msg, TbRelationTypes.FAILURE); } else { - ctx.tellSuccess(msg); + ctx.tellNext(msg, "Other"); } } else { - ctx.tellSuccess(msg); + ctx.tellNext(msg, "Other"); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/CalculateDeltaNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/CalculateDeltaNodeConfiguration.java index b16a97caec..bec93a1d90 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/CalculateDeltaNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/CalculateDeltaNodeConfiguration.java @@ -15,10 +15,12 @@ */ package org.thingsboard.rule.engine.metadata; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Data; import org.thingsboard.rule.engine.api.NodeConfiguration; @Data +@JsonIgnoreProperties(ignoreUnknown = true) public class CalculateDeltaNodeConfiguration implements NodeConfiguration { private String inputValueKey; private String outputValueKey; @@ -26,7 +28,6 @@ public class CalculateDeltaNodeConfiguration implements NodeConfiguration entityKeys; + private final Set entityKeys; private PersistedAlarmRuleState state; private boolean updateFlag; private final DynamicPredicateValueCtx dynamicPredicateValueCtx; - AlarmRuleState(AlarmSeverity severity, AlarmRule alarmRule, Set entityKeys, PersistedAlarmRuleState state, DynamicPredicateValueCtx dynamicPredicateValueCtx) { + AlarmRuleState(AlarmSeverity severity, AlarmRule alarmRule, Set entityKeys, PersistedAlarmRuleState state, DynamicPredicateValueCtx dynamicPredicateValueCtx) { this.severity = severity; this.alarmRule = alarmRule; this.entityKeys = entityKeys; @@ -84,8 +89,8 @@ class AlarmRuleState { this.dynamicPredicateValueCtx = dynamicPredicateValueCtx; } - public boolean validateTsUpdate(Set changedKeys) { - for (EntityKey key : changedKeys) { + public boolean validateTsUpdate(Set changedKeys) { + for (AlarmConditionFilterKey key : changedKeys) { if (entityKeys.contains(key)) { return true; } @@ -93,18 +98,14 @@ class AlarmRuleState { return false; } - public boolean validateAttrUpdate(Set changedKeys) { + public boolean validateAttrUpdate(Set changedKeys) { //If the attribute was updated, but no new telemetry arrived - we ignore this until new telemetry is there. - for (EntityKey key : entityKeys) { - if (key.getType().equals(EntityKeyType.TIME_SERIES)) { + for (AlarmConditionFilterKey key : entityKeys) { + if (key.getType().equals(AlarmConditionKeyType.TIME_SERIES)) { return false; } } - for (EntityKey key : changedKeys) { - EntityKeyType keyType = key.getType(); - if (EntityKeyType.CLIENT_ATTRIBUTE.equals(keyType) || EntityKeyType.SERVER_ATTRIBUTE.equals(keyType) || EntityKeyType.SHARED_ATTRIBUTE.equals(keyType)) { - key = new EntityKey(EntityKeyType.ATTRIBUTE, key.getKey()); - } + for (AlarmConditionFilterKey key : changedKeys) { if (entityKeys.contains(key)) { return true; } @@ -259,16 +260,46 @@ class AlarmRuleState { private boolean eval(AlarmCondition condition, DataSnapshot data) { boolean eval = true; - for (KeyFilter keyFilter : condition.getCondition()) { - EntityKeyValue value = data.getValue(keyFilter.getKey()); + for (var filter : condition.getCondition()) { + EntityKeyValue value; + if (filter.getKey().getType().equals(AlarmConditionKeyType.CONSTANT)) { + try { + value = getConstantValue(filter); + } catch (RuntimeException e) { + log.warn("Failed to parse constant value from filter: {}", filter, e); + value = null; + } + } else { + value = data.getValue(filter.getKey()); + } if (value == null) { return false; } - eval = eval && eval(data, value, keyFilter.getPredicate()); + eval = eval && eval(data, value, filter.getPredicate()); } return eval; } + private EntityKeyValue getConstantValue(AlarmConditionFilter filter) { + EntityKeyValue value = new EntityKeyValue(); + String valueStr = filter.getValue().toString(); + switch (filter.getValueType()) { + case STRING: + value.setStrValue(valueStr); + break; + case DATE_TIME: + value.setLngValue(Long.valueOf(valueStr)); + break; + case NUMERIC: + value.setDblValue(Double.valueOf(valueStr)); + break; + case BOOLEAN: + value.setBoolValue(Boolean.valueOf(valueStr)); + break; + } + return value; + } + private boolean eval(DataSnapshot data, EntityKeyValue value, KeyFilterPredicate predicate) { switch (predicate.getType()) { case STRING: @@ -388,23 +419,18 @@ class AlarmRuleState { EntityKeyValue ekv = null; if (value.getDynamicValue() != null) { switch (value.getDynamicValue().getSourceType()) { - case CURRENT_TENANT: - ekv = dynamicPredicateValueCtx.getTenantValue(value.getDynamicValue().getSourceAttribute()); - break; + case CURRENT_DEVICE: + ekv = data.getValue(new AlarmConditionFilterKey(AlarmConditionKeyType.ATTRIBUTE, value.getDynamicValue().getSourceAttribute())); + if (ekv != null || !value.getDynamicValue().isInherit()) { + break; + } case CURRENT_CUSTOMER: ekv = dynamicPredicateValueCtx.getCustomerValue(value.getDynamicValue().getSourceAttribute()); - break; - case CURRENT_DEVICE: - ekv = data.getValue(new EntityKey(EntityKeyType.ATTRIBUTE, value.getDynamicValue().getSourceAttribute())); - if (ekv == null) { - ekv = data.getValue(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, value.getDynamicValue().getSourceAttribute())); - if (ekv == null) { - ekv = data.getValue(new EntityKey(EntityKeyType.SHARED_ATTRIBUTE, value.getDynamicValue().getSourceAttribute())); - if (ekv == null) { - ekv = data.getValue(new EntityKey(EntityKeyType.CLIENT_ATTRIBUTE, value.getDynamicValue().getSourceAttribute())); - } - } + if (ekv != null || !value.getDynamicValue().isInherit()) { + break; } + case CURRENT_TENANT: + ekv = dynamicPredicateValueCtx.getTenantValue(value.getDynamicValue().getSourceAttribute()); } } return ekv; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java index 7af0beb505..ca8628f86d 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmState.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmStatus; +import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType; import org.thingsboard.server.common.data.device.profile.DeviceProfileAlarm; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; @@ -138,9 +139,9 @@ class AlarmState { public boolean validateUpdate(SnapshotUpdate update, AlarmRuleState state) { if (update != null) { //Check that the update type and that keys match. - if (update.getType().equals(EntityKeyType.TIME_SERIES)) { + if (update.getType().equals(AlarmConditionKeyType.TIME_SERIES)) { return state.validateTsUpdate(update.getKeys()); - } else if (update.getType().equals(EntityKeyType.ATTRIBUTE)) { + } else if (update.getType().equals(AlarmConditionKeyType.ATTRIBUTE)) { return state.validateAttrUpdate(update.getKeys()); } } @@ -252,7 +253,7 @@ class AlarmState { String alarmDetailsStr = ruleState.getAlarmRule().getAlarmDetails(); if (StringUtils.isNotEmpty(alarmDetailsStr)) { - for (KeyFilter keyFilter : ruleState.getAlarmRule().getCondition().getCondition()) { + for (var keyFilter : ruleState.getAlarmRule().getCondition().getCondition()) { EntityKeyValue entityKeyValue = dataSnapshot.getValue(keyFilter.getKey()); alarmDetailsStr = alarmDetailsStr.replaceAll(String.format("\\$\\{%s}", keyFilter.getKey().getKey()), getValueAsString(entityKeyValue)); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DataSnapshot.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DataSnapshot.java index ed939bc3c1..c7db043dc9 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DataSnapshot.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DataSnapshot.java @@ -17,6 +17,8 @@ package org.thingsboard.rule.engine.profile; import lombok.Getter; import lombok.Setter; +import org.thingsboard.server.common.data.device.profile.AlarmConditionFilterKey; +import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; @@ -30,55 +32,42 @@ class DataSnapshot { @Getter @Setter private long ts; - private final Set keys; - private final Map values = new ConcurrentHashMap<>(); + private final Set keys; + private final Map values = new ConcurrentHashMap<>(); - DataSnapshot(Set entityKeysToFetch) { + DataSnapshot(Set entityKeysToFetch) { this.keys = entityKeysToFetch; } - void removeValue(EntityKey key) { - switch (key.getType()) { - case ATTRIBUTE: - values.remove(key); - values.remove(getAttrKey(key, EntityKeyType.CLIENT_ATTRIBUTE)); - values.remove(getAttrKey(key, EntityKeyType.SHARED_ATTRIBUTE)); - values.remove(getAttrKey(key, EntityKeyType.SERVER_ATTRIBUTE)); - break; - case CLIENT_ATTRIBUTE: - case SHARED_ATTRIBUTE: - case SERVER_ATTRIBUTE: - values.remove(key); - values.remove(getAttrKey(key, EntityKeyType.ATTRIBUTE)); - break; - default: - values.remove(key); - } + static AlarmConditionFilterKey toConditionKey(EntityKey key) { + return new AlarmConditionFilterKey(toConditionKeyType(key.getType()), key.getKey()); } - boolean putValue(EntityKey key, long newTs, EntityKeyValue value) { - boolean updateOfTs = ts != newTs; - boolean result = false; - switch (key.getType()) { + static AlarmConditionKeyType toConditionKeyType(EntityKeyType keyType) { + switch (keyType) { case ATTRIBUTE: - result |= putIfKeyExists(key, value, updateOfTs); - result |= putIfKeyExists(getAttrKey(key, EntityKeyType.CLIENT_ATTRIBUTE), value, updateOfTs); - result |= putIfKeyExists(getAttrKey(key, EntityKeyType.SHARED_ATTRIBUTE), value, updateOfTs); - result |= putIfKeyExists(getAttrKey(key, EntityKeyType.SERVER_ATTRIBUTE), value, updateOfTs); - break; - case CLIENT_ATTRIBUTE: - case SHARED_ATTRIBUTE: case SERVER_ATTRIBUTE: - result |= putIfKeyExists(key, value, updateOfTs); - result |= putIfKeyExists(getAttrKey(key, EntityKeyType.ATTRIBUTE), value, updateOfTs); - break; + case SHARED_ATTRIBUTE: + case CLIENT_ATTRIBUTE: + return AlarmConditionKeyType.ATTRIBUTE; + case TIME_SERIES: + return AlarmConditionKeyType.TIME_SERIES; + case ENTITY_FIELD: + return AlarmConditionKeyType.ENTITY_FIELD; default: - result |= putIfKeyExists(key, value, updateOfTs); + throw new RuntimeException("Not supported entity key: " + keyType.name()); } - return result; } - private boolean putIfKeyExists(EntityKey key, EntityKeyValue value, boolean updateOfTs) { + void removeValue(EntityKey key) { + values.remove(toConditionKey(key)); + } + + boolean putValue(AlarmConditionFilterKey key, long newTs, EntityKeyValue value) { + return putIfKeyExists(key, value, ts != newTs); + } + + private boolean putIfKeyExists(AlarmConditionFilterKey key, EntityKeyValue value, boolean updateOfTs) { if (keys.contains(key)) { EntityKeyValue oldValue = values.put(key, value); if (updateOfTs) { @@ -91,25 +80,7 @@ class DataSnapshot { } } - EntityKeyValue getValue(EntityKey key) { - if (EntityKeyType.ATTRIBUTE.equals(key.getType())) { - EntityKeyValue value = values.get(key); - if (value == null) { - value = values.get(getAttrKey(key, EntityKeyType.CLIENT_ATTRIBUTE)); - if (value == null) { - value = values.get(getAttrKey(key, EntityKeyType.SHARED_ATTRIBUTE)); - if (value == null) { - value = values.get(getAttrKey(key, EntityKeyType.SERVER_ATTRIBUTE)); - } - } - } - return value; - } else { - return values.get(key); - } - } - - private EntityKey getAttrKey(EntityKey key, EntityKeyType clientAttribute) { - return new EntityKey(clientAttribute, key.getKey()); + EntityKeyValue getValue(AlarmConditionFilterKey key) { + return values.get(key); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java index bb7d5f8862..827aa9eeb0 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java @@ -26,6 +26,8 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.device.profile.AlarmConditionFilterKey; +import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType; import org.thingsboard.server.common.data.device.profile.DeviceProfileAlarm; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -97,10 +99,10 @@ class DeviceState { } public void updateProfile(TbContext ctx, DeviceProfile deviceProfile) throws ExecutionException, InterruptedException { - Set oldKeys = this.deviceProfile.getEntityKeys(); + Set oldKeys = this.deviceProfile.getEntityKeys(); this.deviceProfile.updateDeviceProfile(deviceProfile); if (latestValues != null) { - Set keysToFetch = new HashSet<>(this.deviceProfile.getEntityKeys()); + Set keysToFetch = new HashSet<>(this.deviceProfile.getEntityKeys()); keysToFetch.removeAll(oldKeys); if (!keysToFetch.isEmpty()) { addEntityKeysToSnapshot(ctx, deviceId, keysToFetch, latestValues); @@ -260,29 +262,29 @@ class DeviceState { } private SnapshotUpdate merge(DataSnapshot latestValues, Long newTs, List data) { - Set keys = new HashSet<>(); + Set keys = new HashSet<>(); for (KvEntry entry : data) { - EntityKey entityKey = new EntityKey(EntityKeyType.TIME_SERIES, entry.getKey()); + AlarmConditionFilterKey entityKey = new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, entry.getKey()); if (latestValues.putValue(entityKey, newTs, toEntityValue(entry))) { keys.add(entityKey); } } latestValues.setTs(newTs); - return new SnapshotUpdate(EntityKeyType.TIME_SERIES, keys); + return new SnapshotUpdate(AlarmConditionKeyType.TIME_SERIES, keys); } private SnapshotUpdate merge(DataSnapshot latestValues, Set attributes, String scope) { long newTs = 0; - Set keys = new HashSet<>(); + Set keys = new HashSet<>(); for (AttributeKvEntry entry : attributes) { newTs = Math.max(newTs, entry.getLastUpdateTs()); - EntityKey entityKey = new EntityKey(getKeyTypeFromScope(scope), entry.getKey()); + AlarmConditionFilterKey entityKey = new AlarmConditionFilterKey(AlarmConditionKeyType.ATTRIBUTE, entry.getKey()); if (latestValues.putValue(entityKey, newTs, toEntityValue(entry))) { keys.add(entityKey); } } latestValues.setTs(newTs); - return new SnapshotUpdate(EntityKeyType.ATTRIBUTE, keys); + return new SnapshotUpdate(AlarmConditionKeyType.ATTRIBUTE, keys); } private static EntityKeyType getKeyTypeFromScope(String scope) { @@ -298,37 +300,22 @@ class DeviceState { } private DataSnapshot fetchLatestValues(TbContext ctx, EntityId originator) throws ExecutionException, InterruptedException { - Set entityKeysToFetch = deviceProfile.getEntityKeys(); + Set entityKeysToFetch = deviceProfile.getEntityKeys(); DataSnapshot result = new DataSnapshot(entityKeysToFetch); addEntityKeysToSnapshot(ctx, originator, entityKeysToFetch, result); return result; } - private void addEntityKeysToSnapshot(TbContext ctx, EntityId originator, Set entityKeysToFetch, DataSnapshot result) throws InterruptedException, ExecutionException { - Set serverAttributeKeys = new HashSet<>(); - Set clientAttributeKeys = new HashSet<>(); - Set sharedAttributeKeys = new HashSet<>(); - Set commonAttributeKeys = new HashSet<>(); + private void addEntityKeysToSnapshot(TbContext ctx, EntityId originator, Set entityKeysToFetch, DataSnapshot result) throws InterruptedException, ExecutionException { + Set attributeKeys = new HashSet<>(); Set latestTsKeys = new HashSet<>(); Device device = null; - for (EntityKey entityKey : entityKeysToFetch) { + for (AlarmConditionFilterKey entityKey : entityKeysToFetch) { String key = entityKey.getKey(); switch (entityKey.getType()) { - case SERVER_ATTRIBUTE: - serverAttributeKeys.add(key); - break; - case CLIENT_ATTRIBUTE: - clientAttributeKeys.add(key); - break; - case SHARED_ATTRIBUTE: - sharedAttributeKeys.add(key); - break; case ATTRIBUTE: - serverAttributeKeys.add(key); - clientAttributeKeys.add(key); - sharedAttributeKeys.add(key); - commonAttributeKeys.add(key); + attributeKeys.add(key); break; case TIME_SERIES: latestTsKeys.add(key); @@ -361,32 +348,22 @@ class DeviceState { List data = ctx.getTimeseriesService().findLatest(ctx.getTenantId(), originator, latestTsKeys).get(); for (TsKvEntry entry : data) { if (entry.getValue() != null) { - result.putValue(new EntityKey(EntityKeyType.TIME_SERIES, entry.getKey()), entry.getTs(), toEntityValue(entry)); + result.putValue(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, entry.getKey()), entry.getTs(), toEntityValue(entry)); } } } - if (!clientAttributeKeys.isEmpty()) { - addToSnapshot(result, commonAttributeKeys, - ctx.getAttributesService().find(ctx.getTenantId(), originator, DataConstants.CLIENT_SCOPE, clientAttributeKeys).get()); - } - if (!sharedAttributeKeys.isEmpty()) { - addToSnapshot(result, commonAttributeKeys, - ctx.getAttributesService().find(ctx.getTenantId(), originator, DataConstants.SHARED_SCOPE, sharedAttributeKeys).get()); - } - if (!serverAttributeKeys.isEmpty()) { - addToSnapshot(result, commonAttributeKeys, - ctx.getAttributesService().find(ctx.getTenantId(), originator, DataConstants.SERVER_SCOPE, serverAttributeKeys).get()); + if (!attributeKeys.isEmpty()) { + addToSnapshot(result, ctx.getAttributesService().find(ctx.getTenantId(), originator, DataConstants.CLIENT_SCOPE, attributeKeys).get()); + addToSnapshot(result, ctx.getAttributesService().find(ctx.getTenantId(), originator, DataConstants.SHARED_SCOPE, attributeKeys).get()); + addToSnapshot(result, ctx.getAttributesService().find(ctx.getTenantId(), originator, DataConstants.SERVER_SCOPE, attributeKeys).get()); } } - private void addToSnapshot(DataSnapshot snapshot, Set commonAttributeKeys, List data) { + private void addToSnapshot(DataSnapshot snapshot, List data) { for (AttributeKvEntry entry : data) { if (entry.getValue() != null) { EntityKeyValue value = toEntityValue(entry); - snapshot.putValue(new EntityKey(EntityKeyType.CLIENT_ATTRIBUTE, entry.getKey()), entry.getLastUpdateTs(), value); - if (commonAttributeKeys.contains(entry.getKey())) { - snapshot.putValue(new EntityKey(EntityKeyType.ATTRIBUTE, entry.getKey()), entry.getLastUpdateTs(), value); - } + snapshot.putValue(new AlarmConditionFilterKey(AlarmConditionKeyType.ATTRIBUTE, entry.getKey()), entry.getLastUpdateTs(), value); } } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java index 08a230d716..8fbacc120d 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/ProfileState.java @@ -19,6 +19,9 @@ import lombok.AccessLevel; import lombok.Getter; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.alarm.AlarmSeverity; +import org.thingsboard.server.common.data.device.profile.AlarmConditionFilter; +import org.thingsboard.server.common.data.device.profile.AlarmConditionFilterKey; +import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType; import org.thingsboard.server.common.data.device.profile.AlarmRule; import org.thingsboard.server.common.data.device.profile.DeviceProfileAlarm; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -50,10 +53,10 @@ class ProfileState { @Getter(AccessLevel.PACKAGE) private final List alarmSettings = new CopyOnWriteArrayList<>(); @Getter(AccessLevel.PACKAGE) - private final Set entityKeys = ConcurrentHashMap.newKeySet(); + private final Set entityKeys = ConcurrentHashMap.newKeySet(); - private final Map>> alarmCreateKeys = new HashMap<>(); - private final Map> alarmClearKeys = new HashMap<>(); + private final Map>> alarmCreateKeys = new HashMap<>(); + private final Map> alarmClearKeys = new HashMap<>(); ProfileState(DeviceProfile deviceProfile) { updateDeviceProfile(deviceProfile); @@ -68,18 +71,18 @@ class ProfileState { if (deviceProfile.getProfileData().getAlarms() != null) { alarmSettings.addAll(deviceProfile.getProfileData().getAlarms()); for (DeviceProfileAlarm alarm : deviceProfile.getProfileData().getAlarms()) { - Map> createAlarmKeys = alarmCreateKeys.computeIfAbsent(alarm.getId(), id -> new HashMap<>()); + Map> createAlarmKeys = alarmCreateKeys.computeIfAbsent(alarm.getId(), id -> new HashMap<>()); alarm.getCreateRules().forEach(((severity, alarmRule) -> { - Set ruleKeys = createAlarmKeys.computeIfAbsent(severity, id -> new HashSet<>()); - for (KeyFilter keyFilter : alarmRule.getCondition().getCondition()) { + var ruleKeys = createAlarmKeys.computeIfAbsent(severity, id -> new HashSet<>()); + for (var keyFilter : alarmRule.getCondition().getCondition()) { entityKeys.add(keyFilter.getKey()); ruleKeys.add(keyFilter.getKey()); addDynamicValuesRecursively(keyFilter.getPredicate(), entityKeys, ruleKeys); } })); if (alarm.getClearRule() != null) { - Set clearAlarmKeys = alarmClearKeys.computeIfAbsent(alarm.getId(), id -> new HashSet<>()); - for (KeyFilter keyFilter : alarm.getClearRule().getCondition().getCondition()) { + var clearAlarmKeys = alarmClearKeys.computeIfAbsent(alarm.getId(), id -> new HashSet<>()); + for (var keyFilter : alarm.getClearRule().getCondition().getCondition()) { entityKeys.add(keyFilter.getKey()); clearAlarmKeys.add(keyFilter.getKey()); addDynamicValuesRecursively(keyFilter.getPredicate(), entityKeys, clearAlarmKeys); @@ -89,7 +92,7 @@ class ProfileState { } } - private void addDynamicValuesRecursively(KeyFilterPredicate predicate, Set entityKeys, Set ruleKeys) { + private void addDynamicValuesRecursively(KeyFilterPredicate predicate, Set entityKeys, Set ruleKeys) { switch (predicate.getType()) { case STRING: case NUMERIC: @@ -98,7 +101,7 @@ class ProfileState { if (value != null && (value.getSourceType() == DynamicValueSourceType.CURRENT_TENANT || value.getSourceType() == DynamicValueSourceType.CURRENT_CUSTOMER || value.getSourceType() == DynamicValueSourceType.CURRENT_DEVICE)) { - EntityKey entityKey = new EntityKey(EntityKeyType.ATTRIBUTE, value.getSourceAttribute()); + AlarmConditionFilterKey entityKey = new AlarmConditionFilterKey(AlarmConditionKeyType.ATTRIBUTE, value.getSourceAttribute()); entityKeys.add(entityKey); ruleKeys.add(entityKey); } @@ -115,12 +118,12 @@ class ProfileState { return deviceProfile.getId(); } - Set getCreateAlarmKeys(String id, AlarmSeverity severity) { - Map> sKeys = alarmCreateKeys.get(id); + Set getCreateAlarmKeys(String id, AlarmSeverity severity) { + Map> sKeys = alarmCreateKeys.get(id); if (sKeys == null) { return Collections.emptySet(); } else { - Set keys = sKeys.get(severity); + Set keys = sKeys.get(severity); if (keys == null) { return Collections.emptySet(); } else { @@ -129,8 +132,8 @@ class ProfileState { } } - Set getClearAlarmKeys(String id) { - Set keys = alarmClearKeys.get(id); + Set getClearAlarmKeys(String id) { + Set keys = alarmClearKeys.get(id); if (keys == null) { return Collections.emptySet(); } else { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/SnapshotUpdate.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/SnapshotUpdate.java index 1ad1860d5a..7a3ce5debc 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/SnapshotUpdate.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/SnapshotUpdate.java @@ -16,6 +16,8 @@ package org.thingsboard.rule.engine.profile; import lombok.Getter; +import org.thingsboard.server.common.data.device.profile.AlarmConditionFilterKey; +import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; @@ -24,11 +26,11 @@ import java.util.Set; class SnapshotUpdate { @Getter - private final EntityKeyType type; + private final AlarmConditionKeyType type; @Getter - private final Set keys; + private final Set keys; - SnapshotUpdate(EntityKeyType type, Set keys) { + SnapshotUpdate(AlarmConditionKeyType type, Set keys) { this.type = type; this.keys = keys; } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java index 9282d2c357..b980423c08 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java @@ -36,6 +36,9 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.device.profile.AlarmCondition; +import org.thingsboard.server.common.data.device.profile.AlarmConditionFilter; +import org.thingsboard.server.common.data.device.profile.AlarmConditionFilterKey; +import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType; import org.thingsboard.server.common.data.device.profile.AlarmRule; import org.thingsboard.server.common.data.device.profile.DeviceProfileAlarm; import org.thingsboard.server.common.data.device.profile.DeviceProfileData; @@ -44,6 +47,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.query.BooleanFilterPredicate; import org.thingsboard.server.common.data.query.DynamicValue; import org.thingsboard.server.common.data.query.DynamicValueSourceType; import org.thingsboard.server.common.data.query.EntityKey; @@ -62,6 +66,7 @@ import org.thingsboard.server.dao.model.sql.AttributeKvCompositeKey; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -69,6 +74,7 @@ import java.util.TreeMap; import java.util.UUID; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @RunWith(MockitoJUnitRunner.class) @@ -141,8 +147,8 @@ public class TbDeviceProfileNodeTest { DeviceProfile deviceProfile = new DeviceProfile(); DeviceProfileData deviceProfileData = new DeviceProfileData(); - KeyFilter highTempFilter = new KeyFilter(); - highTempFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); + AlarmConditionFilter highTempFilter = new AlarmConditionFilter(); + highTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); highTempFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate highTemperaturePredicate = new NumericFilterPredicate(); highTemperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); @@ -157,8 +163,8 @@ public class TbDeviceProfileNodeTest { dpa.setAlarmType("highTemperatureAlarm"); dpa.setCreateRules(new TreeMap<>(Collections.singletonMap(AlarmSeverity.CRITICAL, alarmRule))); - KeyFilter lowTempFilter = new KeyFilter(); - lowTempFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); + AlarmConditionFilter lowTempFilter = new AlarmConditionFilter(); + lowTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); lowTempFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate lowTemperaturePredicate = new NumericFilterPredicate(); lowTemperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS); @@ -203,6 +209,175 @@ public class TbDeviceProfileNodeTest { } + @Test + public void testConstantKeyFilterSimple() throws Exception { + init(); + + DeviceProfile deviceProfile = new DeviceProfile(); + deviceProfile.setId(deviceProfileId); + DeviceProfileData deviceProfileData = new DeviceProfileData(); + + Device device = new Device(); + device.setId(deviceId); + device.setCustomerId(customerId); + + AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey( + EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "alarmEnabled" + ); + + AttributeKvEntity attributeKvEntity = new AttributeKvEntity(); + attributeKvEntity.setId(compositeKey); + attributeKvEntity.setBooleanValue(Boolean.TRUE); + attributeKvEntity.setLastUpdateTs(System.currentTimeMillis()); + + AttributeKvEntry entry = attributeKvEntity.toData(); + ListenableFuture> attrListListenableFuture = Futures.immediateFuture(Collections.singletonList(entry)); + + AlarmConditionFilter alarmEnabledFilter = new AlarmConditionFilter(); + alarmEnabledFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.CONSTANT, "alarmEnabled")); + alarmEnabledFilter.setValue(Boolean.TRUE); + alarmEnabledFilter.setValueType(EntityKeyValueType.BOOLEAN); + BooleanFilterPredicate alarmEnabledPredicate = new BooleanFilterPredicate(); + alarmEnabledPredicate.setOperation(BooleanFilterPredicate.BooleanOperation.EQUAL); + alarmEnabledPredicate.setValue(new FilterPredicateValue<>( + Boolean.FALSE, + null, + new DynamicValue<>(DynamicValueSourceType.CURRENT_DEVICE, "alarmEnabled") + )); + alarmEnabledFilter.setPredicate(alarmEnabledPredicate); + + AlarmConditionFilter temperatureFilter = new AlarmConditionFilter(); + temperatureFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + temperatureFilter.setValueType(EntityKeyValueType.NUMERIC); + NumericFilterPredicate temperaturePredicate = new NumericFilterPredicate(); + temperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + temperaturePredicate.setValue(new FilterPredicateValue<>(20.0, null, null)); + temperatureFilter.setPredicate(temperaturePredicate); + + AlarmCondition alarmCondition = new AlarmCondition(); + alarmCondition.setCondition(Arrays.asList(alarmEnabledFilter, temperatureFilter)); + AlarmRule alarmRule = new AlarmRule(); + alarmRule.setCondition(alarmCondition); + DeviceProfileAlarm dpa = new DeviceProfileAlarm(); + dpa.setId("alarmEnabledAlarmID"); + dpa.setAlarmType("alarmEnabledAlarm"); + dpa.setCreateRules(new TreeMap<>(Collections.singletonMap(AlarmSeverity.CRITICAL, alarmRule))); + + deviceProfileData.setAlarms(Collections.singletonList(dpa)); + deviceProfile.setProfileData(deviceProfileData); + + Mockito.when(cache.get(tenantId, deviceId)).thenReturn(deviceProfile); + Mockito.when(timeseriesService.findLatest(tenantId, deviceId, Collections.singleton("temperature"))) + .thenReturn(Futures.immediateFuture(Collections.emptyList())); + Mockito.when(alarmService.findLatestByOriginatorAndType(tenantId, deviceId, "alarmEnabledAlarm")) + .thenReturn(Futures.immediateFuture(null)); + Mockito.when(alarmService.createOrUpdateAlarm(Mockito.any())).thenAnswer(AdditionalAnswers.returnsFirstArg()); + Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + .thenReturn(attrListListenableFuture); + + TbMsg theMsg = TbMsg.newMsg("ALARM", deviceId, new TbMsgMetaData(), ""); + Mockito.when(ctx.newMsg(Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.anyString())) + .thenReturn(theMsg); + + ObjectNode data = mapper.createObjectNode(); + data.put("temperature", 21); + TbMsg msg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, new TbMsgMetaData(), + TbMsgDataType.JSON, mapper.writeValueAsString(data), null, null); + + node.onMsg(ctx, msg); + verify(ctx).tellSuccess(msg); + verify(ctx).tellNext(theMsg, "Alarm Created"); + verify(ctx, Mockito.never()).tellFailure(Mockito.any(), Mockito.any()); + } + + @Test + public void testConstantKeyFilterInherited() throws Exception { + init(); + + DeviceProfile deviceProfile = new DeviceProfile(); + deviceProfile.setId(deviceProfileId); + DeviceProfileData deviceProfileData = new DeviceProfileData(); + + Device device = new Device(); + device.setId(deviceId); + device.setCustomerId(customerId); + + AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey( + EntityType.TENANT, tenantId.getId(), "SERVER_SCOPE", "alarmEnabled" + ); + + AttributeKvEntity attributeKvEntity = new AttributeKvEntity(); + attributeKvEntity.setId(compositeKey); + attributeKvEntity.setBooleanValue(Boolean.TRUE); + attributeKvEntity.setLastUpdateTs(System.currentTimeMillis()); + + AttributeKvEntry entry = attributeKvEntity.toData(); + ListenableFuture> attrListListenableFuture = Futures.immediateFuture(Optional.of(entry)); + + AlarmConditionFilter alarmEnabledFilter = new AlarmConditionFilter(); + alarmEnabledFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.CONSTANT, "alarmEnabled")); + alarmEnabledFilter.setValue(Boolean.TRUE); + alarmEnabledFilter.setValueType(EntityKeyValueType.BOOLEAN); + BooleanFilterPredicate alarmEnabledPredicate = new BooleanFilterPredicate(); + alarmEnabledPredicate.setOperation(BooleanFilterPredicate.BooleanOperation.EQUAL); + alarmEnabledPredicate.setValue(new FilterPredicateValue<>( + Boolean.FALSE, + null, + new DynamicValue<>(DynamicValueSourceType.CURRENT_DEVICE, "alarmEnabled", true) + )); + alarmEnabledFilter.setPredicate(alarmEnabledPredicate); + + AlarmConditionFilter temperatureFilter = new AlarmConditionFilter(); + temperatureFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + temperatureFilter.setValueType(EntityKeyValueType.NUMERIC); + NumericFilterPredicate temperaturePredicate = new NumericFilterPredicate(); + temperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + temperaturePredicate.setValue(new FilterPredicateValue<>(20.0, null, null)); + temperatureFilter.setPredicate(temperaturePredicate); + + AlarmCondition alarmCondition = new AlarmCondition(); + alarmCondition.setCondition(Arrays.asList(alarmEnabledFilter, temperatureFilter)); + AlarmRule alarmRule = new AlarmRule(); + alarmRule.setCondition(alarmCondition); + DeviceProfileAlarm dpa = new DeviceProfileAlarm(); + dpa.setId("alarmEnabledAlarmID"); + dpa.setAlarmType("alarmEnabledAlarm"); + dpa.setCreateRules(new TreeMap<>(Collections.singletonMap(AlarmSeverity.CRITICAL, alarmRule))); + + deviceProfileData.setAlarms(Collections.singletonList(dpa)); + deviceProfile.setProfileData(deviceProfileData); + + Mockito.when(deviceService.findDeviceById(tenantId, deviceId)).thenReturn(device); + Mockito.when(cache.get(tenantId, deviceId)).thenReturn(deviceProfile); + Mockito.when(timeseriesService.findLatest(tenantId, deviceId, Collections.singleton("temperature"))) + .thenReturn(Futures.immediateFuture(Collections.emptyList())); + Mockito.when(alarmService.findLatestByOriginatorAndType(tenantId, deviceId, "alarmEnabledAlarm")) + .thenReturn(Futures.immediateFuture(null)); + Mockito.when(alarmService.createOrUpdateAlarm(Mockito.any())).thenAnswer(AdditionalAnswers.returnsFirstArg()); + Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + .thenReturn(Futures.immediateFuture(Collections.emptyList())); + Mockito.when(attributesService.find(eq(tenantId), eq(customerId), Mockito.anyString(), Mockito.anyString())) + .thenReturn(Futures.immediateFuture(Optional.empty())); + Mockito.when(attributesService.find(eq(tenantId), eq(tenantId), Mockito.anyString(), Mockito.anyString())) + .thenReturn(attrListListenableFuture); + + TbMsg theMsg = TbMsg.newMsg("ALARM", deviceId, new TbMsgMetaData(), ""); + Mockito.when(ctx.newMsg(Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.anyString())) + .thenReturn(theMsg); + + ObjectNode data = mapper.createObjectNode(); + data.put("temperature", 21); + TbMsg msg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, new TbMsgMetaData(), + TbMsgDataType.JSON, mapper.writeValueAsString(data), null, null); + + node.onMsg(ctx, msg); + verify(ctx).tellSuccess(msg); + verify(ctx).tellNext(theMsg, "Alarm Created"); + verify(ctx, Mockito.never()).tellFailure(Mockito.any(), Mockito.any()); + } + @Test public void testCurrentDeviceAttributeForDynamicValue() throws Exception { init(); @@ -228,8 +403,8 @@ public class TbDeviceProfileNodeTest { ListenableFuture> listListenableFutureWithLess = Futures.immediateFuture(Collections.singletonList(entry)); - KeyFilter highTempFilter = new KeyFilter(); - highTempFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); + AlarmConditionFilter highTempFilter = new AlarmConditionFilter(); + highTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); highTempFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate highTemperaturePredicate = new NumericFilterPredicate(); highTemperaturePredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); @@ -303,8 +478,8 @@ public class TbDeviceProfileNodeTest { ListenableFuture> optionalListenableFutureWithLess = Futures.immediateFuture(Optional.of(entry)); - KeyFilter lowTempFilter = new KeyFilter(); - lowTempFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); + AlarmConditionFilter lowTempFilter = new AlarmConditionFilter(); + lowTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); lowTempFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate lowTempPredicate = new NumericFilterPredicate(); lowTempPredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS); @@ -382,8 +557,8 @@ public class TbDeviceProfileNodeTest { ListenableFuture> optionalListenableFutureWithLess = Futures.immediateFuture(Optional.of(entry)); - KeyFilter lowTempFilter = new KeyFilter(); - lowTempFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); + AlarmConditionFilter lowTempFilter = new AlarmConditionFilter(); + lowTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); lowTempFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate lowTempPredicate = new NumericFilterPredicate(); lowTempPredicate.setOperation(NumericFilterPredicate.NumericOperation.LESS); @@ -416,7 +591,7 @@ public class TbDeviceProfileNodeTest { Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) .thenReturn(listListenableFutureWithLess); - Mockito.when(attributesService.find(eq(tenantId), eq(tenantId), eq(DataConstants.SERVER_SCOPE), Mockito.anyString())) + Mockito.when(attributesService.find(eq(tenantId), eq(tenantId), eq(DataConstants.SERVER_SCOPE), Mockito.anyString())) .thenReturn(optionalListenableFutureWithLess); TbMsg theMsg = TbMsg.newMsg("ALARM", deviceId, new TbMsgMetaData(), ""); @@ -434,11 +609,152 @@ public class TbDeviceProfileNodeTest { verify(ctx, Mockito.never()).tellFailure(Mockito.any(), Mockito.any()); } - private void init() throws TbNodeException { + @Test + public void testTenantInheritModeForDynamicValues() throws Exception { + init(); - UUID uuid = new UUID(6041557255264276971L, -9019477126543226049L); - System.out.println(uuid); + DeviceProfile deviceProfile = new DeviceProfile(); + DeviceProfileData deviceProfileData = new DeviceProfileData(); + AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey( + EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "tenantAttribute" + ); + + AttributeKvEntity attributeKvEntity = new AttributeKvEntity(); + attributeKvEntity.setId(compositeKey); + attributeKvEntity.setLongValue(100L); + attributeKvEntity.setLastUpdateTs(0L); + + AttributeKvEntry entry = attributeKvEntity.toData(); + ListenableFuture> listListenableFutureWithLess = + Futures.immediateFuture(Collections.singletonList(entry)); + + AlarmConditionFilter lowTempFilter = new AlarmConditionFilter(); + lowTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + lowTempFilter.setValueType(EntityKeyValueType.NUMERIC); + NumericFilterPredicate lowTempPredicate = new NumericFilterPredicate(); + lowTempPredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + lowTempPredicate.setValue( + new FilterPredicateValue<>( + 0.0, + null, + new DynamicValue<>(DynamicValueSourceType.CURRENT_DEVICE, "tenantAttribute", true)) + ); + lowTempFilter.setPredicate(lowTempPredicate); + AlarmCondition alarmCondition = new AlarmCondition(); + alarmCondition.setCondition(Collections.singletonList(lowTempFilter)); + AlarmRule alarmRule = new AlarmRule(); + alarmRule.setCondition(alarmCondition); + DeviceProfileAlarm dpa = new DeviceProfileAlarm(); + dpa.setId("lesstempID"); + dpa.setAlarmType("lessTemperatureAlarm"); + dpa.setCreateRules(new TreeMap<>(Collections.singletonMap(AlarmSeverity.CRITICAL, alarmRule))); + + deviceProfileData.setAlarms(Collections.singletonList(dpa)); + deviceProfile.setProfileData(deviceProfileData); + + Mockito.when(cache.get(tenantId, deviceId)).thenReturn(deviceProfile); + Mockito.when(timeseriesService.findLatest(tenantId, deviceId, Collections.singleton("temperature"))) + .thenReturn(Futures.immediateFuture(Collections.emptyList())); + Mockito.when(alarmService.findLatestByOriginatorAndType(tenantId, deviceId, "lessTemperatureAlarm")) + .thenReturn(Futures.immediateFuture(null)); + Mockito.when(alarmService.createOrUpdateAlarm(Mockito.any())) + .thenAnswer(AdditionalAnswers.returnsFirstArg()); + Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + .thenReturn(listListenableFutureWithLess); + + TbMsg theMsg = TbMsg.newMsg("ALARM", deviceId, new TbMsgMetaData(), ""); + Mockito.when(ctx.newMsg(Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.anyString())) + .thenReturn(theMsg); + + ObjectNode data = mapper.createObjectNode(); + data.put("temperature", 150L); + TbMsg msg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, new TbMsgMetaData(), + TbMsgDataType.JSON, mapper.writeValueAsString(data), null, null); + + node.onMsg(ctx, msg); + verify(ctx).tellSuccess(msg); + verify(ctx).tellNext(theMsg, "Alarm Created"); + verify(ctx, Mockito.never()).tellFailure(Mockito.any(), Mockito.any()); + } + + + @Test + public void testCustomerInheritModeForDynamicValues() throws Exception { + init(); + + DeviceProfile deviceProfile = new DeviceProfile(); + DeviceProfileData deviceProfileData = new DeviceProfileData(); + + AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey( + EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "customerAttribute" + ); + + AttributeKvEntity attributeKvEntity = new AttributeKvEntity(); + attributeKvEntity.setId(compositeKey); + attributeKvEntity.setLongValue(100L); + attributeKvEntity.setLastUpdateTs(0L); + + AttributeKvEntry entry = attributeKvEntity.toData(); + ListenableFuture> listListenableFutureWithLess = + Futures.immediateFuture(Collections.singletonList(entry)); + ListenableFuture> optionalListenableFutureWithLess = + Futures.immediateFuture(Optional.of(entry)); + + AlarmConditionFilter lowTempFilter = new AlarmConditionFilter(); + lowTempFilter.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.TIME_SERIES, "temperature")); + lowTempFilter.setValueType(EntityKeyValueType.NUMERIC); + NumericFilterPredicate lowTempPredicate = new NumericFilterPredicate(); + lowTempPredicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); + lowTempPredicate.setValue( + new FilterPredicateValue<>( + 0.0, + null, + new DynamicValue<>(DynamicValueSourceType.CURRENT_CUSTOMER, "customerAttribute", true)) + ); + lowTempFilter.setPredicate(lowTempPredicate); + AlarmCondition alarmCondition = new AlarmCondition(); + alarmCondition.setCondition(Collections.singletonList(lowTempFilter)); + AlarmRule alarmRule = new AlarmRule(); + alarmRule.setCondition(alarmCondition); + DeviceProfileAlarm dpa = new DeviceProfileAlarm(); + dpa.setId("lesstempID"); + dpa.setAlarmType("lessTemperatureAlarm"); + dpa.setCreateRules(new TreeMap<>(Collections.singletonMap(AlarmSeverity.CRITICAL, alarmRule))); + + deviceProfileData.setAlarms(Collections.singletonList(dpa)); + deviceProfile.setProfileData(deviceProfileData); + + Mockito.when(cache.get(tenantId, deviceId)).thenReturn(deviceProfile); + Mockito.when(timeseriesService.findLatest(tenantId, deviceId, Collections.singleton("temperature"))) + .thenReturn(Futures.immediateFuture(Collections.emptyList())); + Mockito.when(alarmService.findLatestByOriginatorAndType(tenantId, deviceId, "lessTemperatureAlarm")) + .thenReturn(Futures.immediateFuture(null)); + Mockito.when(alarmService.createOrUpdateAlarm(Mockito.any())) + .thenAnswer(AdditionalAnswers.returnsFirstArg()); + Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + .thenReturn(listListenableFutureWithLess); + Mockito.when(attributesService.find(eq(tenantId), eq(tenantId), eq(DataConstants.SERVER_SCOPE), Mockito.anyString())) + .thenReturn(optionalListenableFutureWithLess); + + TbMsg theMsg = TbMsg.newMsg("ALARM", deviceId, new TbMsgMetaData(), ""); + Mockito.when(ctx.newMsg(Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.anyString())) + .thenReturn(theMsg); + + ObjectNode data = mapper.createObjectNode(); + data.put("temperature", 150L); + TbMsg msg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, new TbMsgMetaData(), + TbMsgDataType.JSON, mapper.writeValueAsString(data), null, null); + + node.onMsg(ctx, msg); + verify(ctx).tellSuccess(msg); + verify(ctx).tellNext(theMsg, "Alarm Created"); + verify(ctx, Mockito.never()).tellFailure(Mockito.any(), Mockito.any()); + } + + private void init() throws TbNodeException { Mockito.when(ctx.getTenantId()).thenReturn(tenantId); Mockito.when(ctx.getDeviceProfileCache()).thenReturn(cache); Mockito.when(ctx.getTimeseriesService()).thenReturn(timeseriesService); diff --git a/tools/src/main/shell/keygen.properties b/tools/src/main/shell/keygen.properties index 0fb36d4524..d2733e5b7b 100644 --- a/tools/src/main/shell/keygen.properties +++ b/tools/src/main/shell/keygen.properties @@ -15,6 +15,7 @@ # DOMAIN_SUFFIX="$(hostname)" +SUBJECT_ALTERNATIVE_NAMES="ip:127.0.0.1" ORGANIZATIONAL_UNIT=Thingsboard ORGANIZATION=Thingsboard CITY=SF diff --git a/tools/src/main/shell/server.keygen.sh b/tools/src/main/shell/server.keygen.sh index e01b17b8b2..7679cbd812 100755 --- a/tools/src/main/shell/server.keygen.sh +++ b/tools/src/main/shell/server.keygen.sh @@ -86,6 +86,12 @@ fi echo "Generating SSL Key Pair..." +EXT="" + +if [[ ! -z "$SUBJECT_ALTERNATIVE_NAMES" ]]; then + EXT="-ext san=$SUBJECT_ALTERNATIVE_NAMES " +fi + keytool -genkeypair -v \ -alias $SERVER_KEY_ALIAS \ -dname "CN=$DOMAIN_SUFFIX, OU=$ORGANIZATIONAL_UNIT, O=$ORGANIZATION, L=$CITY, ST=$STATE_OR_PROVINCE, C=$TWO_LETTER_COUNTRY_CODE" \ @@ -94,7 +100,8 @@ keytool -genkeypair -v \ -storepass $SERVER_KEYSTORE_PASSWORD \ -keyalg $SERVER_KEY_ALG \ -keysize $SERVER_KEY_SIZE \ - -validity 9999 + -validity 9999 \ + $EXT status=$? if [[ $status != 0 ]]; then diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index a7ed12faab..7fc93371f6 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -39,6 +39,7 @@ zk: transport: http: request_timeout: "${HTTP_REQUEST_TIMEOUT:60000}" + max_request_timeout: "${HTTP_MAX_REQUEST_TIMEOUT:300000}" sessions: inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}" report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:30000}" diff --git a/ui-ngx/src/app/core/http/entity.service.ts b/ui-ngx/src/app/core/http/entity.service.ts index 463183d06e..1118307d06 100644 --- a/ui-ngx/src/app/core/http/entity.service.ts +++ b/ui-ngx/src/app/core/http/entity.service.ts @@ -414,6 +414,7 @@ export class EntityService { { key: nameField, valueType: EntityKeyValueType.STRING, + value: null, predicate: { type: FilterPredicateType.STRING, operation: StringOperation.STARTS_WITH, diff --git a/ui-ngx/src/app/modules/home/components/alarm/alarm-table-config.ts b/ui-ngx/src/app/modules/home/components/alarm/alarm-table-config.ts index db12d8ff65..743b49c286 100644 --- a/ui-ngx/src/app/modules/home/components/alarm/alarm-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/alarm/alarm-table-config.ts @@ -43,6 +43,7 @@ import { AlarmDetailsDialogComponent, AlarmDetailsDialogData } from '@home/components/alarm/alarm-details-dialog.component'; +import { DAY, historyInterval } from '@shared/models/time/time.models'; export class AlarmTableConfig extends EntityTableConfig { @@ -59,6 +60,7 @@ export class AlarmTableConfig extends EntityTableConfig this.loadDataOnInit = false; this.tableTitle = ''; this.useTimePageLink = true; + this.defaultTimewindowInterval = historyInterval(DAY * 30); this.detailsPanelEnabled = false; this.selectionEnabled = false; this.searchEnabled = true; diff --git a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts index dedaced162..da735c1567 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts @@ -55,11 +55,11 @@ import { EntityTypeTranslation } from '@shared/models/entity-type.models'; import { DialogService } from '@core/services/dialog.service'; import { AddEntityDialogComponent } from './add-entity-dialog.component'; import { AddEntityDialogData, EntityAction } from '@home/models/entity/entity-component.models'; -import { DAY, historyInterval, HistoryWindowType, Timewindow } from '@shared/models/time/time.models'; +import { HistoryWindowType, Timewindow } from '@shared/models/time/time.models'; import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; import { TbAnchorComponent } from '@shared/components/tb-anchor.component'; import { isDefined, isUndefined } from '@core/utils'; -import { HasUUID } from '../../../../shared/models/id/has-uuid'; +import { HasUUID } from '@shared/models/id/has-uuid'; @Component({ selector: 'tb-entities-table', @@ -202,7 +202,7 @@ export class EntitiesTableComponent extends PageComponent implements AfterViewIn this.pageSizeOptions = [this.defaultPageSize, this.defaultPageSize * 2, this.defaultPageSize * 3]; if (this.entitiesTableConfig.useTimePageLink) { - this.timewindow = historyInterval(DAY); + this.timewindow = this.entitiesTableConfig.defaultTimewindowInterval; const currentTime = Date.now(); this.pageLink = new TimePageLink(10, 0, null, sortOrder, currentTime - this.timewindow.history.timewindowMs, currentTime); @@ -446,7 +446,7 @@ export class EntitiesTableComponent extends PageComponent implements AfterViewIn resetSortAndFilter(update: boolean = true, preserveTimewindow: boolean = false) { this.pageLink.textSearch = null; if (this.entitiesTableConfig.useTimePageLink && !preserveTimewindow) { - this.timewindow = historyInterval(DAY); + this.timewindow = this.entitiesTableConfig.defaultTimewindowInterval; } if (this.displayPagination) { this.paginator.pageIndex = 0; diff --git a/ui-ngx/src/app/modules/home/components/entity/entity-details-panel.component.ts b/ui-ngx/src/app/modules/home/components/entity/entity-details-panel.component.ts index 1efcc9211e..0eebbc353e 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entity-details-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/entity-details-panel.component.ts @@ -280,7 +280,11 @@ export class EntityDetailsPanelComponent extends PageComponent implements OnInit saveEntity() { if (this.detailsForm.valid) { - const editingEntity = mergeDeep(this.editingEntity, this.entityComponent.entityFormValue()); + const editingEntity = {...this.editingEntity, ...this.entityComponent.entityFormValue()}; + if (this.editingEntity.hasOwnProperty('additionalInfo')) { + editingEntity.additionalInfo = + mergeDeep((this.editingEntity as any).additionalInfo, this.entityComponent.entityFormValue()?.additionalInfo); + } this.entitiesTableConfig.saveEntity(editingEntity).subscribe( (entity) => { this.entity = entity; diff --git a/ui-ngx/src/app/modules/home/components/filter/boolean-filter-predicate.component.html b/ui-ngx/src/app/modules/home/components/filter/boolean-filter-predicate.component.html index 6194581625..f7bae7d797 100644 --- a/ui-ngx/src/app/modules/home/components/filter/boolean-filter-predicate.component.html +++ b/ui-ngx/src/app/modules/home/components/filter/boolean-filter-predicate.component.html @@ -16,7 +16,7 @@ -->
- + @@ -25,7 +25,8 @@ diff --git a/ui-ngx/src/app/modules/home/components/filter/boolean-filter-predicate.component.ts b/ui-ngx/src/app/modules/home/components/filter/boolean-filter-predicate.component.ts index fbd329dc16..c6c8a5ed51 100644 --- a/ui-ngx/src/app/modules/home/components/filter/boolean-filter-predicate.component.ts +++ b/ui-ngx/src/app/modules/home/components/filter/boolean-filter-predicate.component.ts @@ -41,6 +41,8 @@ export class BooleanFilterPredicateComponent implements ControlValueAccessor, On @Input() allowUserDynamicSource = true; + @Input() onlyUserDynamicSource = false; + valueTypeEnum = EntityKeyValueType; booleanFilterPredicateFormGroup: FormGroup; diff --git a/ui-ngx/src/app/modules/home/components/filter/complex-filter-predicate-dialog.component.html b/ui-ngx/src/app/modules/home/components/filter/complex-filter-predicate-dialog.component.html index bbf4887e5f..e2bee07a3f 100644 --- a/ui-ngx/src/app/modules/home/components/filter/complex-filter-predicate-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/filter/complex-filter-predicate-dialog.component.html @@ -39,6 +39,7 @@ [valueType]="data.valueType" [displayUserParameters]="data.displayUserParameters" [allowUserDynamicSource]="data.allowUserDynamicSource" + [onlyUserDynamicSource]="data.onlyUserDynamicSource" [operation]="complexFilterFormGroup.get('operation').value" [key]="data.key" formControlName="predicates"> diff --git a/ui-ngx/src/app/modules/home/components/filter/complex-filter-predicate-dialog.component.ts b/ui-ngx/src/app/modules/home/components/filter/complex-filter-predicate-dialog.component.ts index 447546ba2f..3f1bb3cace 100644 --- a/ui-ngx/src/app/modules/home/components/filter/complex-filter-predicate-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/filter/complex-filter-predicate-dialog.component.ts @@ -37,6 +37,7 @@ export interface ComplexFilterPredicateDialogData { valueType: EntityKeyValueType; displayUserParameters: boolean; allowUserDynamicSource: boolean; + onlyUserDynamicSource: boolean; } @Component({ diff --git a/ui-ngx/src/app/modules/home/components/filter/complex-filter-predicate.component.ts b/ui-ngx/src/app/modules/home/components/filter/complex-filter-predicate.component.ts index 0843583a52..9f4acfc3c6 100644 --- a/ui-ngx/src/app/modules/home/components/filter/complex-filter-predicate.component.ts +++ b/ui-ngx/src/app/modules/home/components/filter/complex-filter-predicate.component.ts @@ -52,6 +52,8 @@ export class ComplexFilterPredicateComponent implements ControlValueAccessor, On @Input() allowUserDynamicSource = true; + @Input() onlyUserDynamicSource = false; + private propagateChange = null; private complexFilterPredicate: ComplexFilterPredicateInfo; @@ -89,7 +91,8 @@ export class ComplexFilterPredicateComponent implements ControlValueAccessor, On isAdd: false, key: this.key, displayUserParameters: this.displayUserParameters, - allowUserDynamicSource: this.allowUserDynamicSource + allowUserDynamicSource: this.allowUserDynamicSource, + onlyUserDynamicSource: this.onlyUserDynamicSource } }).afterClosed().subscribe( (result) => { diff --git a/ui-ngx/src/app/modules/home/components/filter/filter-predicate-list.component.html b/ui-ngx/src/app/modules/home/components/filter/filter-predicate-list.component.html index 41c04ea238..e5bb27f20c 100644 --- a/ui-ngx/src/app/modules/home/components/filter/filter-predicate-list.component.html +++ b/ui-ngx/src/app/modules/home/components/filter/filter-predicate-list.component.html @@ -26,12 +26,12 @@
-
+
- +
@@ -53,6 +53,7 @@ [valueType]="valueType" [displayUserParameters]="displayUserParameters" [allowUserDynamicSource]="allowUserDynamicSource" + [onlyUserDynamicSource]="onlyUserDynamicSource" [key]="key" [formControl]="predicateControl"> diff --git a/ui-ngx/src/app/modules/home/components/filter/filter-predicate-list.component.ts b/ui-ngx/src/app/modules/home/components/filter/filter-predicate-list.component.ts index e89489e6bb..3c16b01319 100644 --- a/ui-ngx/src/app/modules/home/components/filter/filter-predicate-list.component.ts +++ b/ui-ngx/src/app/modules/home/components/filter/filter-predicate-list.component.ts @@ -66,6 +66,8 @@ export class FilterPredicateListComponent implements ControlValueAccessor, OnIni @Input() allowUserDynamicSource = true; + @Input() onlyUserDynamicSource = false; + filterListFormGroup: FormGroup; valueTypeEnum = EntityKeyValueType; @@ -159,7 +161,8 @@ export class FilterPredicateListComponent implements ControlValueAccessor, OnIni key: this.key, isAdd: true, displayUserParameters: this.displayUserParameters, - allowUserDynamicSource: this.allowUserDynamicSource + allowUserDynamicSource: this.allowUserDynamicSource, + onlyUserDynamicSource: this.onlyUserDynamicSource } }).afterClosed().pipe( map((result) => { diff --git a/ui-ngx/src/app/modules/home/components/filter/filter-predicate-value.component.html b/ui-ngx/src/app/modules/home/components/filter/filter-predicate-value.component.html index adda4bea1a..4bc249837a 100644 --- a/ui-ngx/src/app/modules/home/components/filter/filter-predicate-value.component.html +++ b/ui-ngx/src/app/modules/home/components/filter/filter-predicate-value.component.html @@ -16,7 +16,7 @@ -->
-
+
@@ -45,9 +45,9 @@
filter.default-value
-
+
-
+
@@ -68,9 +68,17 @@
filter.source-attribute
+
+ + {{ 'filter.inherit-owner' | translate}} + +
filter.source-attribute-not-set
+
-