Browse Source

Merge remote-tracking branch 'upstream/master' into feature/edge-rpc-call-from-device

pull/7592/head
Volodymyr Babak 4 years ago
parent
commit
8b46b30747
  1. 65
      application/src/main/data/upgrade/3.4.1/schema_update.sql
  2. 9
      application/src/main/data/upgrade/3.4.1/schema_update_before.sql
  3. 12
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  4. 26
      application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java
  5. 5
      application/src/main/java/org/thingsboard/server/controller/AssetController.java
  6. 66
      application/src/main/java/org/thingsboard/server/controller/AuthController.java
  7. 11
      application/src/main/java/org/thingsboard/server/controller/UserController.java
  8. 8
      application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
  9. 5
      application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java
  10. 4
      application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java
  11. 3
      application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java
  12. 9
      application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java
  13. 14
      application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java
  14. 37
      application/src/main/java/org/thingsboard/server/service/install/CassandraKeyspaceService.java
  15. 26
      application/src/main/java/org/thingsboard/server/service/install/DbUpgradeExecutorService.java
  16. 19
      application/src/main/java/org/thingsboard/server/service/install/NoSqlKeyspaceService.java
  17. 68
      application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java
  18. 5
      application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java
  19. 13
      application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java
  20. 14
      application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java
  21. 71
      application/src/main/java/org/thingsboard/server/service/security/auth/DefaultTokenOutdatingService.java
  22. 60
      application/src/main/java/org/thingsboard/server/service/security/auth/TokenOutdatingService.java
  23. 2
      application/src/main/java/org/thingsboard/server/service/security/auth/jwt/JwtAuthenticationProvider.java
  24. 4
      application/src/main/java/org/thingsboard/server/service/security/auth/jwt/RefreshTokenAuthenticationProvider.java
  25. 6
      application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/AbstractOAuth2ClientMapper.java
  26. 14
      application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java
  27. 5
      application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java
  28. 10
      application/src/main/java/org/thingsboard/server/service/security/model/SecurityUser.java
  29. 10
      application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java
  30. 41
      application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java
  31. 1
      application/src/main/java/org/thingsboard/server/service/security/system/SystemSecurityService.java
  32. 24
      application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java
  33. 5
      application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java
  34. 22
      application/src/main/java/org/thingsboard/server/service/ttl/EdgeEventsCleanUpService.java
  35. 21
      application/src/main/resources/thingsboard.yml
  36. 14
      application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java
  37. 4
      application/src/test/java/org/thingsboard/server/controller/BaseAlarmControllerTest.java
  38. 12
      application/src/test/java/org/thingsboard/server/controller/BaseAuthControllerTest.java
  39. 160
      application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java
  40. 69
      application/src/test/java/org/thingsboard/server/controller/BaseEdgeEventControllerTest.java
  41. 8
      application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java
  42. 3
      application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java
  43. 115
      application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java
  44. 167
      application/src/test/java/org/thingsboard/server/service/security/auth/TokenOutdatingTest.java
  45. 48
      application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java
  46. 12
      application/src/test/java/org/thingsboard/server/system/BaseRestApiLimitsTest.java
  47. 1
      application/src/test/java/org/thingsboard/server/transport/TransportNoSqlTestSuite.java
  48. 6
      application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java
  49. 9
      application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java
  50. 42
      application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesProtoIntegrationTest.java
  51. 22
      application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java
  52. 5
      application/src/test/java/org/thingsboard/server/transport/mqtt/attributes/AbstractMqttAttributesIntegrationTest.java
  53. 9
      application/src/test/java/org/thingsboard/server/transport/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java
  54. 6
      application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/attributes/MqttAttributesProtoIntegrationTest.java
  55. 16
      application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/timeseries/AbstractMqttTimeseriesProtoIntegrationTest.java
  56. 5
      application/src/test/resources/application-test.properties
  57. 16
      common/cache/src/main/java/org/thingsboard/server/cache/CacheSpecsMap.java
  58. 2
      common/cache/src/main/java/org/thingsboard/server/cache/TbCaffeineCacheConfiguration.java
  59. 34
      common/cache/src/main/java/org/thingsboard/server/cache/usersUpdateTime/UsersSessionInvalidationCaffeineCache.java
  60. 36
      common/cache/src/main/java/org/thingsboard/server/cache/usersUpdateTime/UsersSessionInvalidationRedisCache.java
  61. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java
  62. 16
      common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/AbstractCassandraCluster.java
  63. 22
      common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/guava/GuavaDriverContext.java
  64. 14
      common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/guava/GuavaSessionBuilder.java
  65. 27
      common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDaoNonCloud.java
  66. 4
      common/data/pom.xml
  67. 2
      common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java
  68. 301
      common/data/src/main/java/org/thingsboard/server/common/data/DynamicProtoUtils.java
  69. 4
      common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java
  70. 169
      common/data/src/main/java/org/thingsboard/server/common/data/device/profile/ProtoTransportPayloadConfiguration.java
  71. 16
      common/data/src/main/java/org/thingsboard/server/common/data/security/event/UserAuthDataChangedEvent.java
  72. 40
      common/data/src/main/java/org/thingsboard/server/common/data/security/event/UserCredentialsInvalidationEvent.java
  73. 39
      common/data/src/main/java/org/thingsboard/server/common/data/security/event/UserSessionInvalidationEvent.java
  74. 26
      common/data/src/main/java/org/thingsboard/server/common/data/util/TbPair.java
  75. 169
      common/data/src/test/java/org/thingsboard/server/common/data/DynamicProtoUtilsTest.java
  76. 3
      common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgProcessingStackItem.java
  77. 37
      common/message/src/test/java/org/thingsboard/server/common/msg/TbMsgProcessingStackItemTest.java
  78. 4
      common/queue/pom.xml
  79. 18
      common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java
  80. 95
      common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/TbDate.java
  81. 80
      common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/TbUtils.java
  82. 17
      common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java
  83. 7
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java
  84. 37
      common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java
  85. 5
      common/util/src/main/java/org/thingsboard/common/util/ThingsBoardExecutors.java
  86. 35
      common/util/src/test/java/org/thingsboard/common/util/JacksonUtilTest.java
  87. 5
      dao/pom.xml
  88. 6
      dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java
  89. 62
      dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java
  90. 21
      dao/src/main/java/org/thingsboard/server/dao/aspect/SqlDaoCallsAspect.java
  91. 3
      dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java
  92. 6
      dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogServiceImpl.java
  93. 9
      dao/src/main/java/org/thingsboard/server/dao/edge/BaseEdgeEventService.java
  94. 2
      dao/src/main/java/org/thingsboard/server/dao/edge/EdgeEventDao.java
  95. 146
      dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java
  96. 171
      dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java
  97. 24
      dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java
  98. 18
      dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java
  99. 9
      dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java
  100. 83
      dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java

65
application/src/main/data/upgrade/3.4.1/schema_update.sql

@ -14,6 +14,7 @@
-- limitations under the License.
--
-- AUDIT LOGS MIGRATION START
DO
$$
DECLARE table_partition RECORD;
@ -68,7 +69,69 @@ BEGIN
END LOOP;
INSERT INTO audit_log
SELECT * FROM old_audit_log
SELECT id, created_time, tenant_id, customer_id, entity_id, entity_type, entity_name, user_id, user_name, action_type, action_data, action_status, action_failure_details
FROM old_audit_log
WHERE created_time >= start_time_ms AND created_time < end_time_ms;
END;
$$;
-- AUDIT LOGS MIGRATION END
-- EDGE EVENTS MIGRATION START
DO
$$
DECLARE table_partition RECORD;
BEGIN
-- in case of running the upgrade script a second time:
IF NOT (SELECT exists(SELECT FROM pg_tables WHERE tablename = 'old_edge_event')) THEN
ALTER TABLE edge_event RENAME TO old_edge_event;
ALTER INDEX IF EXISTS idx_edge_event_tenant_id_and_created_time RENAME TO idx_old_edge_event_tenant_id_and_created_time;
FOR table_partition IN SELECT tablename AS name, split_part(tablename, '_', 3) AS partition_ts
FROM pg_tables WHERE tablename LIKE 'edge_event_%'
LOOP
EXECUTE format('ALTER TABLE %s RENAME TO old_edge_event_%s', table_partition.name, table_partition.partition_ts);
END LOOP;
ELSE
RAISE NOTICE 'Table old_edge_event already exists, leaving as is';
END IF;
END;
$$;
CREATE TABLE IF NOT EXISTS edge_event (
id uuid NOT NULL,
created_time bigint NOT NULL,
edge_id uuid,
edge_event_type varchar(255),
edge_event_uid varchar(255),
entity_id uuid,
edge_event_action varchar(255),
body varchar(10000000),
tenant_id uuid,
ts bigint NOT NULL
) PARTITION BY RANGE (created_time);
CREATE INDEX IF NOT EXISTS idx_edge_event_tenant_id_and_created_time ON edge_event(tenant_id, created_time DESC);
CREATE OR REPLACE PROCEDURE migrate_edge_event(IN start_time_ms BIGINT, IN end_time_ms BIGINT, IN partition_size_ms BIGINT)
LANGUAGE plpgsql AS
$$
DECLARE
p RECORD;
partition_end_ts BIGINT;
BEGIN
FOR p IN SELECT DISTINCT (created_time - created_time % partition_size_ms) AS partition_ts FROM old_edge_event
WHERE created_time >= start_time_ms AND created_time < end_time_ms
LOOP
partition_end_ts = p.partition_ts + partition_size_ms;
RAISE NOTICE '[edge_event] Partition to create : [%-%]', p.partition_ts, partition_end_ts;
EXECUTE format('CREATE TABLE IF NOT EXISTS edge_event_%s PARTITION OF edge_event ' ||
'FOR VALUES FROM ( %s ) TO ( %s )', p.partition_ts, p.partition_ts, partition_end_ts);
END LOOP;
INSERT INTO edge_event
SELECT id, created_time, edge_id, edge_event_type, edge_event_uid, entity_id, edge_event_action, body, tenant_id, ts
FROM old_edge_event
WHERE created_time >= start_time_ms AND created_time < end_time_ms;
END;
$$;
-- EDGE EVENTS MIGRATION END

9
application/src/main/data/upgrade/3.4.1/schema_update_before.sql

@ -37,9 +37,10 @@ CREATE OR REPLACE PROCEDURE update_asset_profiles()
LANGUAGE plpgsql AS
$$
BEGIN
UPDATE asset as a SET asset_profile_id = p.id
FROM
(SELECT id, tenant_id, name from asset_profile) as p
WHERE a.asset_profile_id IS NULL AND p.tenant_id = a.tenant_id AND a.type = p.name;
UPDATE asset a SET asset_profile_id = COALESCE(
(SELECT id from asset_profile p WHERE p.tenant_id = a.tenant_id AND a.type = p.name),
(SELECT id from asset_profile p WHERE p.tenant_id = a.tenant_id AND p.name = 'default')
)
WHERE a.asset_profile_id IS NULL;
END;
$$;

12
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@ -76,6 +76,8 @@ import org.thingsboard.server.dao.tenant.TenantProfileService;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.dao.widget.WidgetTypeService;
import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.util.DataDecodingEncodingService;
@ -379,6 +381,16 @@ public class ActorSystemContext {
@Getter
private QueueService queueService;
@Lazy
@Autowired(required = false)
@Getter
private WidgetsBundleService widgetsBundleService;
@Lazy
@Autowired(required = false)
@Getter
private WidgetTypeService widgetTypeService;
@Value("${actors.session.max_concurrent_sessions_per_device:1}")
@Getter
private long maxConcurrentSessionsPerDevice;

26
application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java

@ -26,6 +26,7 @@ import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ListeningExecutor;
import org.thingsboard.rule.engine.api.MailService;
import org.thingsboard.rule.engine.api.RuleEngineAlarmService;
import org.thingsboard.rule.engine.api.RuleEngineApiUsageStateService;
import org.thingsboard.rule.engine.api.RuleEngineAssetProfileCache;
import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache;
import org.thingsboard.rule.engine.api.RuleEngineRpcService;
@ -35,6 +36,7 @@ import org.thingsboard.rule.engine.api.SmsService;
import org.thingsboard.rule.engine.api.TbContext;
import org.thingsboard.rule.engine.api.TbRelationTypes;
import org.thingsboard.rule.engine.api.sms.SmsSenderFactory;
import org.thingsboard.rule.engine.util.TenantIdLoader;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.actors.TbActorRef;
import org.thingsboard.server.cluster.TbClusterService;
@ -89,6 +91,8 @@ import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.dao.widget.WidgetTypeService;
import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.TbQueueCallback;
import org.thingsboard.server.queue.TbQueueMsgMetadata;
@ -751,12 +755,34 @@ class DefaultTbContext implements TbContext {
return mainCtx.getTenantProfileCache().get(getTenantId());
}
@Override
public WidgetsBundleService getWidgetBundleService() {
return mainCtx.getWidgetsBundleService();
}
@Override
public WidgetTypeService getWidgetTypeService() {
return mainCtx.getWidgetTypeService();
}
@Override
public RuleEngineApiUsageStateService getRuleEngineApiUsageStateService() {
return mainCtx.getApiUsageStateService();
}
private TbMsgMetaData getActionMetaData(RuleNodeId ruleNodeId) {
TbMsgMetaData metaData = new TbMsgMetaData();
metaData.putValue("ruleNodeId", ruleNodeId.toString());
return metaData;
}
@Override
public void checkTenantEntity(EntityId entityId) {
if (!this.getTenantId().equals(TenantIdLoader.findTenantId(this, entityId))) {
throw new RuntimeException("Entity with id: '" + entityId + "' specified in the configuration doesn't belong to the current tenant.");
}
}
private class SimpleTbQueueCallback implements TbQueueCallback {
private final Runnable onSuccess;
private final Consumer<Throwable> onFailure;

5
application/src/main/java/org/thingsboard/server/controller/AssetController.java

@ -38,7 +38,6 @@ import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.asset.AssetInfo;
import org.thingsboard.server.common.data.asset.AssetSearchQuery;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.AssetProfileId;
@ -86,7 +85,6 @@ import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHO
import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH;
import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK;
import static org.thingsboard.server.controller.EdgeController.EDGE_ID;
import static org.thingsboard.server.dao.asset.BaseAssetService.TB_SERVICE_QUEUE;
@RestController
@TbCoreComponent
@ -148,9 +146,6 @@ public class AssetController extends BaseController {
@RequestMapping(value = "/asset", method = RequestMethod.POST)
@ResponseBody
public Asset saveAsset(@ApiParam(value = "A JSON value representing the asset.") @RequestBody Asset asset) throws Exception {
if (TB_SERVICE_QUEUE.equals(asset.getType())) {
throw new ThingsboardException("Unable to save asset with type " + TB_SERVICE_QUEUE, ThingsboardErrorCode.BAD_REQUEST_PARAMS);
}
asset.setTenantId(getTenantId());
checkEntity(asset.getId(), asset, Resource.ASSET);
return tbAssetService.save(asset, getCurrentUser());

66
application/src/main/java/org/thingsboard/server/controller/AuthController.java

@ -42,13 +42,12 @@ import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.security.event.UserAuthDataChangedEvent;
import org.thingsboard.server.common.data.security.model.JwtToken;
import org.thingsboard.server.common.data.security.event.UserCredentialsInvalidationEvent;
import org.thingsboard.server.common.data.security.event.UserSessionInvalidationEvent;
import org.thingsboard.server.common.data.security.model.SecuritySettings;
import org.thingsboard.server.common.data.security.model.UserPasswordPolicy;
import org.thingsboard.server.dao.audit.AuditLogService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository;
import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails;
import org.thingsboard.server.service.security.model.ActivateUserRequest;
import org.thingsboard.server.service.security.model.ChangePasswordRequest;
@ -59,7 +58,6 @@ import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.model.UserPrincipal;
import org.thingsboard.server.service.security.model.token.JwtTokenFactory;
import org.thingsboard.server.service.security.system.SystemSecurityService;
import ua_parser.Client;
import javax.servlet.http.HttpServletRequest;
import java.net.URI;
@ -73,7 +71,6 @@ import java.net.URISyntaxException;
public class AuthController extends BaseController {
private final BCryptPasswordEncoder passwordEncoder;
private final JwtTokenFactory tokenFactory;
private final RefreshTokenRepository refreshTokenRepository;
private final MailService mailService;
private final SystemSecurityService systemSecurityService;
private final AuditLogService auditLogService;
@ -128,7 +125,7 @@ public class AuthController extends BaseController {
sendEntityNotificationMsg(getTenantId(), userCredentials.getUserId(), EdgeEventActionType.CREDENTIALS_UPDATED);
eventPublisher.publishEvent(new UserAuthDataChangedEvent(securityUser.getId()));
eventPublisher.publishEvent(new UserCredentialsInvalidationEvent(securityUser.getId()));
ObjectNode response = JacksonUtil.newObjectNode();
response.put("token", tokenFactory.createAccessJwtToken(securityUser).getToken());
response.put("refreshToken", tokenFactory.createRefreshToken(securityUser).getToken());
@ -268,10 +265,7 @@ public class AuthController extends BaseController {
sendEntityNotificationMsg(user.getTenantId(), user.getId(), EdgeEventActionType.CREDENTIALS_UPDATED);
JwtToken accessToken = tokenFactory.createAccessJwtToken(securityUser);
JwtToken refreshToken = refreshTokenRepository.requestRefreshToken(securityUser);
return new JwtTokenPair(accessToken.getToken(), refreshToken.getToken());
return tokenFactory.createTokenPair(securityUser);
} catch (Exception e) {
throw handleException(e);
}
@ -309,11 +303,9 @@ public class AuthController extends BaseController {
String email = user.getEmail();
mailService.sendPasswordWasResetEmail(loginUrl, email);
eventPublisher.publishEvent(new UserAuthDataChangedEvent(securityUser.getId()));
JwtToken accessToken = tokenFactory.createAccessJwtToken(securityUser);
JwtToken refreshToken = refreshTokenRepository.requestRefreshToken(securityUser);
eventPublisher.publishEvent(new UserCredentialsInvalidationEvent(securityUser.getId()));
return new JwtTokenPair(accessToken.getToken(), refreshToken.getToken());
return tokenFactory.createTokenPair(securityUser);
} else {
throw new ThingsboardException("Invalid reset token!", ThingsboardErrorCode.BAD_REQUEST_PARAMS);
}
@ -324,49 +316,9 @@ public class AuthController extends BaseController {
private void logLogoutAction(HttpServletRequest request) throws ThingsboardException {
try {
SecurityUser user = getCurrentUser();
RestAuthenticationDetails details = new RestAuthenticationDetails(request);
String clientAddress = details.getClientAddress();
String browser = "Unknown";
String os = "Unknown";
String device = "Unknown";
if (details.getUserAgent() != null) {
Client userAgent = details.getUserAgent();
if (userAgent.userAgent != null) {
browser = userAgent.userAgent.family;
if (userAgent.userAgent.major != null) {
browser += " " + userAgent.userAgent.major;
if (userAgent.userAgent.minor != null) {
browser += "." + userAgent.userAgent.minor;
if (userAgent.userAgent.patch != null) {
browser += "." + userAgent.userAgent.patch;
}
}
}
}
if (userAgent.os != null) {
os = userAgent.os.family;
if (userAgent.os.major != null) {
os += " " + userAgent.os.major;
if (userAgent.os.minor != null) {
os += "." + userAgent.os.minor;
if (userAgent.os.patch != null) {
os += "." + userAgent.os.patch;
if (userAgent.os.patchMinor != null) {
os += "." + userAgent.os.patchMinor;
}
}
}
}
}
if (userAgent.device != null) {
device = userAgent.device.family;
}
}
auditLogService.logEntityAction(
user.getTenantId(), user.getCustomerId(), user.getId(),
user.getName(), user.getId(), null, ActionType.LOGOUT, null, clientAddress, browser, os, device);
var user = getCurrentUser();
systemSecurityService.logLoginAction(user, new RestAuthenticationDetails(request), ActionType.LOGOUT, null);
eventPublisher.publishEvent(new UserSessionInvalidationEvent(user.getSessionId()));
} catch (Exception e) {
throw handleException(e);
}

11
application/src/main/java/org/thingsboard/server/controller/UserController.java

@ -43,11 +43,9 @@ import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.security.event.UserAuthDataChangedEvent;
import org.thingsboard.server.common.data.security.model.JwtToken;
import org.thingsboard.server.common.data.security.event.UserCredentialsInvalidationEvent;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.user.TbUserService;
import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository;
import org.thingsboard.server.service.security.model.JwtTokenPair;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.model.UserPrincipal;
@ -95,7 +93,6 @@ public class UserController extends BaseController {
private final MailService mailService;
private final JwtTokenFactory tokenFactory;
private final RefreshTokenRepository refreshTokenRepository;
private final SystemSecurityService systemSecurityService;
private final ApplicationEventPublisher eventPublisher;
private final TbUserService tbUserService;
@ -163,9 +160,7 @@ public class UserController extends BaseController {
UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail());
UserCredentials credentials = userService.findUserCredentialsByUserId(authUser.getTenantId(), userId);
SecurityUser securityUser = new SecurityUser(user, credentials.isEnabled(), principal);
JwtToken accessToken = tokenFactory.createAccessJwtToken(securityUser);
JwtToken refreshToken = refreshTokenRepository.requestRefreshToken(securityUser);
return new JwtTokenPair(accessToken.getToken(), refreshToken.getToken());
return tokenFactory.createTokenPair(securityUser);
} catch (Exception e) {
throw handleException(e);
}
@ -376,7 +371,7 @@ public class UserController extends BaseController {
userService.setUserCredentialsEnabled(tenantId, userId, userCredentialsEnabled);
if (!userCredentialsEnabled) {
eventPublisher.publishEvent(new UserAuthDataChangedEvent(userId));
eventPublisher.publishEvent(new UserCredentialsInvalidationEvent(userId));
}
} catch (Exception e) {
throw handleException(e);

8
application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java

@ -26,6 +26,7 @@ import org.thingsboard.server.service.component.ComponentDiscoveryService;
import org.thingsboard.server.service.install.DatabaseEntitiesUpgradeService;
import org.thingsboard.server.service.install.DatabaseTsUpgradeService;
import org.thingsboard.server.service.install.EntityDatabaseSchemaService;
import org.thingsboard.server.service.install.NoSqlKeyspaceService;
import org.thingsboard.server.service.install.SystemDataLoaderService;
import org.thingsboard.server.service.install.TsDatabaseSchemaService;
import org.thingsboard.server.service.install.TsLatestDatabaseSchemaService;
@ -51,6 +52,9 @@ public class ThingsboardInstallService {
@Autowired
private EntityDatabaseSchemaService entityDatabaseSchemaService;
@Autowired(required = false)
private NoSqlKeyspaceService noSqlKeyspaceService;
@Autowired
private TsDatabaseSchemaService tsDatabaseSchemaService;
@ -252,6 +256,10 @@ public class ThingsboardInstallService {
log.info("Installing DataBase schema for timeseries...");
if (noSqlKeyspaceService != null) {
noSqlKeyspaceService.createDatabaseSchema();
}
tsDatabaseSchemaService.createDatabaseSchema();
if (tsLatestDatabaseSchemaService != null) {

5
application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java

@ -395,6 +395,11 @@ public class DefaultTbApiUsageStateService extends AbstractPartitionBasedService
}
}
@Override
public ApiUsageState findApiUsageStateById(TenantId tenantId, ApiUsageStateId id) {
return apiUsageStateService.findApiUsageStateById(tenantId, id);
}
private interface StateChecker {
boolean check(long threshold, long warnThreshold, long value);
}

4
application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java

@ -16,7 +16,7 @@
package org.thingsboard.server.service.apiusage;
import org.springframework.context.ApplicationListener;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.rule.engine.api.RuleEngineApiUsageStateService;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.TenantProfileId;
@ -26,7 +26,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceM
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
public interface TbApiUsageStateService extends TbApiUsageStateClient, ApplicationListener<PartitionChangeEvent> {
public interface TbApiUsageStateService extends TbApiUsageStateClient, RuleEngineApiUsageStateService, ApplicationListener<PartitionChangeEvent> {
void process(TbProtoQueueMsg<ToUsageStatsServiceMsg> msg, TbCallback callback);

3
application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java

@ -40,6 +40,7 @@ import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.service.executors.DbCallbackExecutorService;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import org.thingsboard.server.service.telemetry.AlarmSubscriptionService;
import java.util.List;
import java.util.Optional;
@ -62,6 +63,8 @@ public abstract class AbstractTbEntityService {
@Autowired
protected AlarmService alarmService;
@Autowired
protected AlarmSubscriptionService alarmSubscriptionService;
@Autowired
protected CustomerService customerService;
@Autowired
protected TbClusterService tbClusterService;

9
application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java

@ -29,7 +29,6 @@ import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.alarm.AlarmOperationResult;
import org.thingsboard.server.service.entitiy.AbstractTbEntityService;
import java.util.List;
@ -43,7 +42,7 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb
ActionType actionType = alarm.getId() == null ? ActionType.ADDED : ActionType.UPDATED;
TenantId tenantId = alarm.getTenantId();
try {
Alarm savedAlarm = checkNotNull(alarmService.createOrUpdateAlarm(alarm).getAlarm());
Alarm savedAlarm = checkNotNull(alarmSubscriptionService.createOrUpdateAlarm(alarm));
notificationEntityService.notifyCreateOrUpdateAlarm(savedAlarm, actionType, user);
return savedAlarm;
} catch (Exception e) {
@ -55,7 +54,7 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb
@Override
public ListenableFuture<Void> ack(Alarm alarm, User user) {
long ackTs = System.currentTimeMillis();
ListenableFuture<AlarmOperationResult> future = alarmService.ackAlarm(alarm.getTenantId(), alarm.getId(), ackTs);
ListenableFuture<Boolean> future = alarmSubscriptionService.ackAlarm(alarm.getTenantId(), alarm.getId(), ackTs);
return Futures.transform(future, result -> {
alarm.setAckTs(ackTs);
alarm.setStatus(alarm.getStatus().isCleared() ? AlarmStatus.CLEARED_ACK : AlarmStatus.ACTIVE_ACK);
@ -67,7 +66,7 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb
@Override
public ListenableFuture<Void> clear(Alarm alarm, User user) {
long clearTs = System.currentTimeMillis();
ListenableFuture<AlarmOperationResult> future = alarmService.clearAlarm(alarm.getTenantId(), alarm.getId(), null, clearTs);
ListenableFuture<Boolean> future = alarmSubscriptionService.clearAlarm(alarm.getTenantId(), alarm.getId(), null, clearTs);
return Futures.transform(future, result -> {
alarm.setClearTs(clearTs);
alarm.setStatus(alarm.getStatus().isAck() ? AlarmStatus.CLEARED_ACK : AlarmStatus.CLEARED_UNACK);
@ -82,6 +81,6 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb
List<EdgeId> relatedEdgeIds = edgeService.findAllRelatedEdgeIds(tenantId, alarm.getOriginator());
notificationEntityService.notifyDeleteAlarm(tenantId, alarm, alarm.getOriginator(), alarm.getCustomerId(),
relatedEdgeIds, user, JacksonUtil.toString(alarm));
return alarmService.deleteAlarm(tenantId, alarm.getId()).isSuccessful();
return alarmSubscriptionService.deleteAlarm(tenantId, alarm.getId());
}
}

14
application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java

@ -22,8 +22,10 @@ import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.asset.AssetProfile;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.CustomerId;
@ -31,20 +33,32 @@ import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.service.entitiy.AbstractTbEntityService;
import org.thingsboard.server.service.profile.TbAssetProfileCache;
import java.util.List;
import static org.thingsboard.server.dao.asset.BaseAssetService.TB_SERVICE_QUEUE;
@Service
@AllArgsConstructor
public class DefaultTbAssetService extends AbstractTbEntityService implements TbAssetService {
private final AssetService assetService;
private final TbAssetProfileCache assetProfileCache;
@Override
public Asset save(Asset asset, User user) throws Exception {
ActionType actionType = asset.getId() == null ? ActionType.ADDED : ActionType.UPDATED;
TenantId tenantId = asset.getTenantId();
try {
if (TB_SERVICE_QUEUE.equals(asset.getType())) {
throw new ThingsboardException("Unable to save asset with type " + TB_SERVICE_QUEUE, ThingsboardErrorCode.BAD_REQUEST_PARAMS);
} else if (asset.getAssetProfileId() != null) {
AssetProfile assetProfile = assetProfileCache.get(tenantId, asset.getAssetProfileId());
if (assetProfile != null && TB_SERVICE_QUEUE.equals(assetProfile.getName())) {
throw new ThingsboardException("Unable to save asset with profile " + TB_SERVICE_QUEUE, ThingsboardErrorCode.BAD_REQUEST_PARAMS);
}
}
Asset savedAsset = checkNotNull(assetService.saveAsset(asset));
autoCommit(user, savedAsset.getId());
notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedAsset.getId(), savedAsset,

37
application/src/main/java/org/thingsboard/server/service/install/CassandraKeyspaceService.java

@ -0,0 +1,37 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.install;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.NoSqlAnyDaoNonCloud;
/*
* Create keyspace for Cassandra NoSQL database for non-cloud deployment.
* For cloud service like Astra DBaas admin have to create keyspace manually on cloud UI.
* Then create tokens with database admin role and put it on Thingsboard parameters.
* Without this service cloud DB will end up with exception like
* UnauthorizedException: Missing correct permission on thingsboard
* */
@Service
@NoSqlAnyDaoNonCloud
@Profile("install")
public class CassandraKeyspaceService extends CassandraAbstractDatabaseSchemaService
implements NoSqlKeyspaceService {
public CassandraKeyspaceService() {
super("schema-keyspace.cql");
}
}

26
application/src/main/java/org/thingsboard/server/service/install/DbUpgradeExecutorService.java

@ -0,0 +1,26 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.install;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Component;
import org.thingsboard.server.service.executors.DbCallbackExecutorService;
@Component
@Profile("install")
public class DbUpgradeExecutorService extends DbCallbackExecutorService {
}

19
application/src/main/java/org/thingsboard/server/service/install/NoSqlKeyspaceService.java

@ -0,0 +1,19 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.install;
public interface NoSqlKeyspaceService extends DatabaseSchemaService {
}

68
application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java

@ -15,6 +15,8 @@
*/
package org.thingsboard.server.service.install;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
@ -32,14 +34,14 @@ import org.thingsboard.server.common.data.queue.ProcessingStrategyType;
import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.data.queue.SubmitStrategy;
import org.thingsboard.server.common.data.queue.SubmitStrategyType;
import org.thingsboard.server.dao.asset.AssetDao;
import org.thingsboard.server.dao.asset.AssetProfileService;
import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.dao.dashboard.DashboardService;
import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.common.data.util.TbPair;
import org.thingsboard.server.dao.queue.QueueService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.tenant.TenantProfileService;
import org.thingsboard.server.dao.sql.tenant.TenantRepository;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
@ -56,7 +58,9 @@ import java.sql.SQLException;
import java.sql.SQLSyntaxErrorException;
import java.sql.SQLWarning;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import static org.thingsboard.server.service.install.DatabaseHelper.ADDITIONAL_INFO;
@ -106,11 +110,14 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
@Autowired
private TenantService tenantService;
@Autowired
private TenantRepository tenantRepository;
@Autowired
private DeviceService deviceService;
@Autowired
private AssetService assetService;
private AssetDao assetDao;
@Autowired
private DeviceProfileService deviceProfileService;
@ -129,10 +136,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
private TbRuleEngineQueueConfigService queueConfig;
@Autowired
private RuleChainService ruleChainService;
@Autowired
private TenantProfileService tenantProfileService;
private DbUpgradeExecutorService dbUpgradeExecutor;
@Override
public void upgradeDatabase(String fromVersion) throws Exception {
@ -620,26 +624,45 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.4.1", "schema_update_before.sql");
loadSql(schemaUpdateFile, conn);
conn.createStatement().execute("DELETE FROM asset a WHERE NOT exists(SELECT id FROM tenant WHERE id = a.tenant_id);");
log.info("Creating default asset profiles...");
PageLink pageLink = new PageLink(100);
PageData<Tenant> pageData;
PageLink pageLink = new PageLink(1000);
PageData<TenantId> tenantIds;
do {
pageData = tenantService.findTenants(pageLink);
for (Tenant tenant : pageData.getData()) {
List<EntitySubtype> assetTypes = assetService.findAssetTypesByTenantId(tenant.getId()).get();
try {
assetProfileService.createDefaultAssetProfile(tenant.getId());
} catch (Exception e) {
}
for (EntitySubtype assetType : assetTypes) {
List<ListenableFuture<?>> futures = new ArrayList<>();
tenantIds = tenantService.findTenantsIds(pageLink);
for (TenantId tenantId : tenantIds.getData()) {
futures.add(dbUpgradeExecutor.submit(() -> {
try {
assetProfileService.findOrCreateAssetProfile(tenant.getId(), assetType.getType());
} catch (Exception e) {
}
assetProfileService.createDefaultAssetProfile(tenantId);
} catch (Exception e) {}
}));
}
Futures.allAsList(futures).get();
pageLink = pageLink.nextPageLink();
} while (tenantIds.hasNext());
pageLink = new PageLink(1000);
PageData<TbPair<UUID, String>> pairs;
do {
List<ListenableFuture<?>> futures = new ArrayList<>();
pairs = assetDao.getAllAssetTypes(pageLink);
for (TbPair<UUID, String> pair : pairs.getData()) {
TenantId tenantId = new TenantId(pair.getFirst());
String assetType = pair.getSecond();
if (!"default".equals(assetType)) {
futures.add(dbUpgradeExecutor.submit(() -> {
try {
assetProfileService.findOrCreateAssetProfile(tenantId, assetType);
} catch (Exception e) {}
}));
}
}
Futures.allAsList(futures).get();
pageLink = pageLink.nextPageLink();
} while (pageData.hasNext());
} while (pairs.hasNext());
log.info("Updating asset profiles...");
conn.createStatement().execute("call update_asset_profiles()");
@ -728,5 +751,4 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
return queue;
}
}

5
application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java

@ -73,6 +73,11 @@ public class DefaultCacheCleanupService implements CacheCleanupService {
log.info("Clear cache to upgrade from version 3.3.4 to 3.4.0 ...");
clearAll();
break;
case "3.4.1":
log.info("Clear cache to upgrade from version 3.4.1 to 3.4.2 ...");
clearCacheByName("assets");
clearCacheByName("repositorySettings");
break;
default:
//Do nothing, since cache cleanup is optional.
}

13
application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java

@ -64,6 +64,7 @@ import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfi
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.alarm.AlarmDao;
import org.thingsboard.server.dao.audit.AuditLogDao;
import org.thingsboard.server.dao.edge.EdgeEventDao;
import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.event.EventService;
@ -142,6 +143,9 @@ public class DefaultDataUpdateService implements DataUpdateService {
@Autowired
private AuditLogDao auditLogDao;
@Autowired
private EdgeEventDao edgeEventDao;
@Override
public void updateData(String fromVersion) throws Exception {
switch (fromVersion) {
@ -181,14 +185,21 @@ public class DefaultDataUpdateService implements DataUpdateService {
}
break;
case "3.4.1":
log.info("Updating data from version 3.4.1 to 3.4.2 ...");
boolean skipAuditLogsMigration = getEnv("TB_SKIP_AUDIT_LOGS_MIGRATION", false);
if (!skipAuditLogsMigration) {
log.info("Updating data from version 3.4.1 to 3.4.2 ...");
log.info("Starting audit logs migration. Can be skipped with TB_SKIP_AUDIT_LOGS_MIGRATION env variable set to true");
auditLogDao.migrateAuditLogs();
} else {
log.info("Skipping audit logs migration");
}
boolean skipEdgeEventsMigration = getEnv("TB_SKIP_EDGE_EVENTS_MIGRATION", false);
if (!skipEdgeEventsMigration) {
log.info("Starting edge events migration. Can be skipped with TB_SKIP_EDGE_EVENTS_MIGRATION env variable set to true");
edgeEventDao.migrateEdgeEvents();
} else {
log.info("Skipping edge events migration");
}
break;
default:
throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion);

14
application/src/main/java/org/thingsboard/server/service/rpc/DefaultTbRuleEngineRpcService.java

@ -19,6 +19,9 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.id.RpcId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.rpc.Rpc;
import org.thingsboard.server.common.data.rpc.RpcError;
import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcRequest;
import org.thingsboard.rule.engine.api.RuleEngineDeviceRpcResponse;
@ -27,6 +30,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest;
import org.thingsboard.server.dao.rpc.RpcService;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
@ -52,6 +56,7 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi
private final PartitionService partitionService;
private final TbClusterService clusterService;
private final TbServiceInfoProvider serviceInfoProvider;
private final RpcService rpcService;
private final ConcurrentMap<UUID, Consumer<FromDeviceRpcResponse>> toDeviceRpcRequests = new ConcurrentHashMap<>();
@ -61,10 +66,12 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi
public DefaultTbRuleEngineRpcService(PartitionService partitionService,
TbClusterService clusterService,
TbServiceInfoProvider serviceInfoProvider) {
TbServiceInfoProvider serviceInfoProvider,
RpcService rpcService) {
this.partitionService = partitionService;
this.clusterService = clusterService;
this.serviceInfoProvider = serviceInfoProvider;
this.rpcService = rpcService;
}
@Autowired(required = false)
@ -119,6 +126,11 @@ public class DefaultTbRuleEngineRpcService implements TbRuleEngineDeviceRpcServi
});
}
@Override
public Rpc findRpcById(TenantId tenantId, RpcId id) {
return rpcService.findById(tenantId, id);
}
@Override
public void processRpcResponseFromDevice(FromDeviceRpcResponse response) {
log.trace("[{}] Received response to server-side RPC request from Core RPC Service", response.getId());

71
application/src/main/java/org/thingsboard/server/service/security/auth/DefaultTokenOutdatingService.java

@ -0,0 +1,71 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.security.auth;
import io.jsonwebtoken.Claims;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cache.TbTransactionalCache;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.security.event.UserAuthDataChangedEvent;
import org.thingsboard.server.common.data.security.model.JwtToken;
import org.thingsboard.server.service.security.model.token.JwtTokenFactory;
import java.util.Optional;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
@Service
public class DefaultTokenOutdatingService implements TokenOutdatingService {
private final TbTransactionalCache<String, Long> cache;
private final JwtTokenFactory tokenFactory;
public DefaultTokenOutdatingService(@Qualifier("UsersSessionInvalidation") TbTransactionalCache<String, Long> cache, JwtTokenFactory tokenFactory) {
this.cache = cache;
this.tokenFactory = tokenFactory;
}
@EventListener(classes = UserAuthDataChangedEvent.class)
public void onUserAuthDataChanged(UserAuthDataChangedEvent event) {
if (StringUtils.hasText(event.getId())) {
cache.put(event.getId(), event.getTs());
}
}
@Override
public boolean isOutdated(JwtToken token, UserId userId) {
Claims claims = tokenFactory.parseTokenClaims(token).getBody();
long issueTime = claims.getIssuedAt().getTime();
String sessionId = claims.get("sessionId", String.class);
if (isTokenOutdated(issueTime, userId.toString())){
return true;
} else {
return sessionId != null && isTokenOutdated(issueTime, sessionId);
}
}
private Boolean isTokenOutdated(long issueTime, String sessionId) {
return Optional.ofNullable(cache.get(sessionId)).map(outdatageTime -> isTokenOutdated(issueTime, outdatageTime.get())).orElse(false);
}
private boolean isTokenOutdated(long issueTime, Long outdatageTime) {
return MILLISECONDS.toSeconds(issueTime) < MILLISECONDS.toSeconds(outdatageTime);
}
}

60
application/src/main/java/org/thingsboard/server/service/security/auth/TokenOutdatingService.java

@ -15,67 +15,11 @@
*/
package org.thingsboard.server.service.security.auth;
import io.jsonwebtoken.Claims;
import lombok.RequiredArgsConstructor;
import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.CacheConstants;
import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.security.event.UserAuthDataChangedEvent;
import org.thingsboard.server.common.data.security.model.JwtToken;
import org.thingsboard.server.config.JwtSettings;
import org.thingsboard.server.service.security.model.token.JwtTokenFactory;
import javax.annotation.PostConstruct;
import java.util.Optional;
public interface TokenOutdatingService {
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
boolean isOutdated(JwtToken token, UserId userId);
@Service
@RequiredArgsConstructor
public class TokenOutdatingService {
private final CacheManager cacheManager;
private final JwtTokenFactory tokenFactory;
private final JwtSettings jwtSettings;
private Cache usersUpdateTimeCache;
@PostConstruct
protected void initCache() {
usersUpdateTimeCache = cacheManager.getCache(CacheConstants.USERS_UPDATE_TIME_CACHE);
}
@EventListener(classes = UserAuthDataChangedEvent.class)
public void onUserAuthDataChanged(UserAuthDataChangedEvent event) {
usersUpdateTimeCache.put(toKey(event.getUserId()), event.getTs());
}
public boolean isOutdated(JwtToken token, UserId userId) {
Claims claims = tokenFactory.parseTokenClaims(token).getBody();
long issueTime = claims.getIssuedAt().getTime();
return Optional.ofNullable(usersUpdateTimeCache.get(toKey(userId), Long.class))
.map(outdatageTime -> {
if (System.currentTimeMillis() - outdatageTime <= SECONDS.toMillis(jwtSettings.getRefreshTokenExpTime())) {
return MILLISECONDS.toSeconds(issueTime) < MILLISECONDS.toSeconds(outdatageTime);
} else {
/*
* Means that since the outdating has passed more than
* the lifetime of refresh token (the longest lived)
* and there is no need to store outdatage time anymore
* as all the tokens issued before the outdatage time
* are now expired by themselves
* */
usersUpdateTimeCache.evict(toKey(userId));
return false;
}
})
.orElse(false);
}
private String toKey(UserId userId) {
return userId.getId().toString();
}
}

2
application/src/main/java/org/thingsboard/server/service/security/auth/jwt/JwtAuthenticationProvider.java

@ -20,8 +20,8 @@ import org.springframework.security.authentication.AuthenticationProvider;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.AuthenticationException;
import org.springframework.stereotype.Component;
import org.thingsboard.server.service.security.auth.TokenOutdatingService;
import org.thingsboard.server.service.security.auth.JwtAuthenticationToken;
import org.thingsboard.server.service.security.auth.TokenOutdatingService;
import org.thingsboard.server.service.security.exception.JwtExpiredTokenException;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.model.token.JwtTokenFactory;

4
application/src/main/java/org/thingsboard/server/service/security/auth/jwt/RefreshTokenAuthenticationProvider.java

@ -33,11 +33,11 @@ import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.service.security.auth.TokenOutdatingService;
import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.service.security.auth.RefreshAuthenticationToken;
import org.thingsboard.server.service.security.auth.TokenOutdatingService;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.model.UserPrincipal;
import org.thingsboard.server.service.security.model.token.JwtTokenFactory;
@ -66,7 +66,7 @@ public class RefreshTokenAuthenticationProvider implements AuthenticationProvide
} else {
securityUser = authenticateByPublicId(principal.getValue());
}
securityUser.setSessionId(unsafeUser.getSessionId());
if (tokenOutdatingService.isOutdated(rawAccessToken, securityUser.getId())) {
throw new CredentialsExpiredException("Token is outdated");
}

6
application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/AbstractOAuth2ClientMapper.java

@ -47,6 +47,7 @@ import org.thingsboard.server.dao.oauth2.OAuth2User;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.service.entitiy.user.TbUserService;
import org.thingsboard.server.service.install.InstallScripts;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.model.UserPrincipal;
@ -81,6 +82,9 @@ public abstract class AbstractOAuth2ClientMapper {
@Autowired
private InstallScripts installScripts;
@Autowired
private TbUserService tbUserService;
@Autowired
protected TbTenantProfileCache tenantProfileCache;
@ -146,7 +150,7 @@ public abstract class AbstractOAuth2ClientMapper {
user.setAdditionalInfo(additionalInfo);
user = userService.saveUser(user);
user = tbUserService.save(tenantId, customerId, user, false, null, null);
if (config.isActivateUser()) {
UserCredentials userCredentials = userService.findUserCredentialsByUserId(user.getTenantId(), user.getId());
userService.activateUserCredentials(user.getTenantId(), userCredentials.getActivateToken(), passwordEncoder.encode(""));

14
application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java

@ -25,14 +25,15 @@ import org.springframework.security.oauth2.core.endpoint.OAuth2AuthorizationRequ
import org.springframework.security.web.authentication.SimpleUrlAuthenticationSuccessHandler;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.oauth2.OAuth2Registration;
import org.thingsboard.server.common.data.security.model.JwtToken;
import org.thingsboard.server.dao.oauth2.OAuth2Service;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository;
import org.thingsboard.server.service.security.model.JwtTokenPair;
import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.model.token.JwtTokenFactory;
import org.thingsboard.server.service.security.system.SystemSecurityService;
@ -54,7 +55,6 @@ import static org.thingsboard.server.service.security.auth.oauth2.HttpCookieOAut
public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationSuccessHandler {
private final JwtTokenFactory tokenFactory;
private final RefreshTokenRepository refreshTokenRepository;
private final OAuth2ClientMapperProvider oauth2ClientMapperProvider;
private final OAuth2Service oAuth2Service;
private final OAuth2AuthorizedClientService oAuth2AuthorizedClientService;
@ -63,14 +63,12 @@ public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationS
@Autowired
public Oauth2AuthenticationSuccessHandler(final JwtTokenFactory tokenFactory,
final RefreshTokenRepository refreshTokenRepository,
final OAuth2ClientMapperProvider oauth2ClientMapperProvider,
final OAuth2Service oAuth2Service,
final OAuth2AuthorizedClientService oAuth2AuthorizedClientService,
final HttpCookieOAuth2AuthorizationRequestRepository httpCookieOAuth2AuthorizationRequestRepository,
final SystemSecurityService systemSecurityService) {
this.tokenFactory = tokenFactory;
this.refreshTokenRepository = refreshTokenRepository;
this.oauth2ClientMapperProvider = oauth2ClientMapperProvider;
this.oAuth2Service = oAuth2Service;
this.oAuth2AuthorizedClientService = oAuth2AuthorizedClientService;
@ -106,11 +104,11 @@ public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationS
SecurityUser securityUser = mapper.getOrCreateUserByClientPrincipal(request, token, oAuth2AuthorizedClient.getAccessToken().getTokenValue(),
registration);
JwtToken accessToken = tokenFactory.createAccessJwtToken(securityUser);
JwtToken refreshToken = refreshTokenRepository.requestRefreshToken(securityUser);
JwtTokenPair tokenPair = tokenFactory.createTokenPair(securityUser);
clearAuthenticationAttributes(request, response);
getRedirectStrategy().sendRedirect(request, response, baseUrl + "/?accessToken=" + accessToken.getToken() + "&refreshToken=" + refreshToken.getToken());
getRedirectStrategy().sendRedirect(request, response, baseUrl + "/?accessToken=" + tokenPair.getToken() + "&refreshToken=" + tokenPair.getRefreshToken());
systemSecurityService.logLoginAction(securityUser, new RestAuthenticationDetails(request), ActionType.LOGIN, registration.getName(), null);
} catch (Exception e) {
log.debug("Error occurred during processing authentication success result. " +
"request [{}], response [{}], authentication [{}]", request, response, authentication, e);

5
application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java

@ -25,7 +25,6 @@ import org.springframework.security.web.authentication.AuthenticationSuccessHand
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.service.security.auth.MfaAuthenticationToken;
import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository;
import org.thingsboard.server.service.security.auth.mfa.config.TwoFaConfigManager;
import org.thingsboard.server.service.security.model.JwtTokenPair;
import org.thingsboard.server.service.security.model.SecurityUser;
@ -45,7 +44,6 @@ public class RestAwareAuthenticationSuccessHandler implements AuthenticationSucc
private final ObjectMapper mapper;
private final JwtTokenFactory tokenFactory;
private final TwoFaConfigManager twoFaConfigManager;
private final RefreshTokenRepository refreshTokenRepository;
@Override
public void onAuthenticationSuccess(HttpServletRequest request, HttpServletResponse response,
@ -62,8 +60,7 @@ public class RestAwareAuthenticationSuccessHandler implements AuthenticationSucc
tokenPair.setRefreshToken(null);
tokenPair.setScope(Authority.PRE_VERIFICATION_TOKEN);
} else {
tokenPair.setToken(tokenFactory.createAccessJwtToken(securityUser).getToken());
tokenPair.setRefreshToken(refreshTokenRepository.requestRefreshToken(securityUser).getToken());
tokenPair = tokenFactory.createTokenPair(securityUser);
}
response.setStatus(HttpStatus.OK.value());

10
application/src/main/java/org/thingsboard/server/service/security/model/SecurityUser.java

@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.id.UserId;
import java.util.Collection;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -31,6 +32,7 @@ public class SecurityUser extends User {
private Collection<GrantedAuthority> authorities;
private boolean enabled;
private UserPrincipal userPrincipal;
private String sessionId;
public SecurityUser() {
super();
@ -44,6 +46,7 @@ public class SecurityUser extends User {
super(user);
this.enabled = enabled;
this.userPrincipal = userPrincipal;
this.sessionId = UUID.randomUUID().toString();
}
public Collection<GrantedAuthority> getAuthorities() {
@ -71,4 +74,11 @@ public class SecurityUser extends User {
this.userPrincipal = userPrincipal;
}
public String getSessionId() {
return sessionId;
}
public void setSessionId(String sessionId) {
this.sessionId = sessionId;
}
}

10
application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java

@ -60,6 +60,7 @@ public class JwtTokenFactory {
private static final String IS_PUBLIC = "isPublic";
private static final String TENANT_ID = "tenantId";
private static final String CUSTOMER_ID = "customerId";
private static final String SESSION_ID = "sessionId";
private final JwtSettings settings;
@ -119,6 +120,9 @@ public class JwtTokenFactory {
if (customerId != null) {
securityUser.setCustomerId(new CustomerId(UUID.fromString(customerId)));
}
if (claims.get(SESSION_ID, String.class) != null) {
securityUser.setSessionId(claims.get(SESSION_ID, String.class));
}
UserPrincipal principal;
if (securityUser.getAuthority() != Authority.PRE_VERIFICATION_TOKEN) {
@ -161,6 +165,9 @@ public class JwtTokenFactory {
UserPrincipal principal = new UserPrincipal(isPublic ? UserPrincipal.Type.PUBLIC_ID : UserPrincipal.Type.USER_NAME, subject);
SecurityUser securityUser = new SecurityUser(new UserId(UUID.fromString(claims.get(USER_ID, String.class))));
securityUser.setUserPrincipal(principal);
if (claims.get(SESSION_ID, String.class) != null) {
securityUser.setSessionId(claims.get(SESSION_ID, String.class));
}
return securityUser;
}
@ -183,6 +190,9 @@ public class JwtTokenFactory {
Claims claims = Jwts.claims().setSubject(principal.getValue());
claims.put(USER_ID, securityUser.getId().getId().toString());
claims.put(SCOPES, scopes);
if (securityUser.getSessionId() != null) {
claims.put(SESSION_ID, securityUser.getSessionId());
}
ZonedDateTime currentTime = ZonedDateTime.now();

41
application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java

@ -263,6 +263,11 @@ public class DefaultSystemSecurityService implements SystemSecurityService {
@Override
public void logLoginAction(User user, Object authenticationDetails, ActionType actionType, Exception e) {
logLoginAction(user, authenticationDetails, actionType, null, e);
}
@Override
public void logLoginAction(User user, Object authenticationDetails, ActionType actionType, String provider, Exception e) {
String clientAddress = "Unknown";
String browser = "Unknown";
String os = "Unknown";
@ -278,38 +283,38 @@ public class DefaultSystemSecurityService implements SystemSecurityService {
browser += " " + userAgent.userAgent.major;
if (userAgent.userAgent.minor != null) {
browser += "." + userAgent.userAgent.minor;
if (userAgent.userAgent.patch != null) {
browser += "." + userAgent.userAgent.patch;
}
if (userAgent.userAgent.patch != null) {
browser += "." + userAgent.userAgent.patch;
}
}
}
if (userAgent.os != null) {
os = userAgent.os.family;
if (userAgent.os.major != null) {
os += " " + userAgent.os.major;
if (userAgent.os.minor != null) {
os += "." + userAgent.os.minor;
if (userAgent.os.patch != null) {
os += "." + userAgent.os.patch;
if (userAgent.os.patchMinor != null) {
os += "." + userAgent.os.patchMinor;
}
}
if (userAgent.os != null) {
os = userAgent.os.family;
if (userAgent.os.major != null) {
os += " " + userAgent.os.major;
if (userAgent.os.minor != null) {
os += "." + userAgent.os.minor;
if (userAgent.os.patch != null) {
os += "." + userAgent.os.patch;
if (userAgent.os.patchMinor != null) {
os += "." + userAgent.os.patchMinor;
}
}
}
}
if (userAgent.device != null) {
device = userAgent.device.family;
}
}
if (userAgent.device != null) {
device = userAgent.device.family;
}
}
}
if (actionType == ActionType.LOGIN && e == null) {
userService.setLastLoginTs(user.getTenantId(), user.getId());
}
auditLogService.logEntityAction(
user.getTenantId(), user.getCustomerId(), user.getId(),
user.getName(), user.getId(), null, actionType, e, clientAddress, browser, os, device);
user.getName(), user.getId(), null, actionType, e, clientAddress, browser, os, device, provider);
}
private static boolean isPositiveInteger(Integer val) {

1
application/src/main/java/org/thingsboard/server/service/security/system/SystemSecurityService.java

@ -44,4 +44,5 @@ public interface SystemSecurityService {
void logLoginAction(User user, Object authenticationDetails, ActionType actionType, Exception e);
void logLoginAction(User user, Object authenticationDetails, ActionType actionType, String provider, Exception e);
}

24
application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java

@ -24,6 +24,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
@ -121,7 +122,8 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
new EntityKey(EntityKeyType.TIME_SERIES, INACTIVITY_TIMEOUT),
new EntityKey(EntityKeyType.TIME_SERIES, ACTIVITY_STATE),
new EntityKey(EntityKeyType.TIME_SERIES, LAST_CONNECT_TIME),
new EntityKey(EntityKeyType.TIME_SERIES, LAST_DISCONNECT_TIME));
new EntityKey(EntityKeyType.TIME_SERIES, LAST_DISCONNECT_TIME),
new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, INACTIVITY_TIMEOUT));
private static final List<EntityKey> PERSISTENT_ATTRIBUTE_KEYS = Arrays.asList(
new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, LAST_ACTIVITY_TIME),
@ -152,14 +154,21 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
@Value("${state.defaultInactivityTimeoutInSec}")
@Getter
@Setter
private long defaultInactivityTimeoutInSec;
@Value("#{${state.defaultInactivityTimeoutInSec} * 1000}")
@Getter
@Setter
private long defaultInactivityTimeoutMs;
@Value("${state.defaultStateCheckIntervalInSec}")
@Getter
private int defaultStateCheckIntervalInSec;
@Value("${state.persistToTelemetry:false}")
@Getter
@Setter
private boolean persistToTelemetry;
@Value("${state.initFetchPackSize:50000}")
@ -540,7 +549,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
private ListenableFuture<DeviceStateData> transformInactivityTimeout(ListenableFuture<DeviceStateData> future) {
return Futures.transformAsync(future, deviceStateData -> {
if (!persistToTelemetry || deviceStateData.getState().getInactivityTimeout() != TimeUnit.SECONDS.toMillis(defaultInactivityTimeoutInSec)) {
if (!persistToTelemetry || deviceStateData.getState().getInactivityTimeout() != defaultInactivityTimeoutMs) {
return future; //fail fast
}
var attributesFuture = attributesService.find(TenantId.SYS_TENANT_ID, deviceStateData.getDeviceId(), SERVER_SCOPE, INACTIVITY_TIMEOUT);
@ -563,7 +572,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
try {
long lastActivityTime = getEntryValue(data, LAST_ACTIVITY_TIME, 0L);
long inactivityAlarmTime = getEntryValue(data, INACTIVITY_ALARM_TIME, 0L);
long inactivityTimeout = getEntryValue(data, INACTIVITY_TIMEOUT, TimeUnit.SECONDS.toMillis(defaultInactivityTimeoutInSec));
long inactivityTimeout = getEntryValue(data, INACTIVITY_TIMEOUT, defaultInactivityTimeoutMs);
//Actual active state by wall-clock will updated outside this method. This method is only for fetch persistent state
final boolean active = getEntryValue(data, ACTIVITY_STATE, false);
DeviceState deviceState = DeviceState.builder()
@ -634,10 +643,15 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
}
private DeviceStateData toDeviceStateData(EntityData ed, DeviceIdInfo deviceIdInfo) {
DeviceStateData toDeviceStateData(EntityData ed, DeviceIdInfo deviceIdInfo) {
long lastActivityTime = getEntryValue(ed, getKeyType(), LAST_ACTIVITY_TIME, 0L);
long inactivityAlarmTime = getEntryValue(ed, getKeyType(), INACTIVITY_ALARM_TIME, 0L);
long inactivityTimeout = getEntryValue(ed, getKeyType(), INACTIVITY_TIMEOUT, TimeUnit.SECONDS.toMillis(defaultInactivityTimeoutInSec));
long inactivityTimeout = getEntryValue(ed, getKeyType(), INACTIVITY_TIMEOUT, defaultInactivityTimeoutMs);
if (persistToTelemetry && inactivityTimeout == defaultInactivityTimeoutMs) {
log.trace("[{}] default value for inactivity timeout fetched {}, going to fetch inactivity timeout from attributes",
deviceIdInfo.getDeviceId(), inactivityTimeout);
inactivityTimeout = getEntryValue(ed, EntityKeyType.SERVER_ATTRIBUTE, INACTIVITY_TIMEOUT, defaultInactivityTimeoutMs);
}
//Actual active state by wall-clock will updated outside this method. This method is only for fetch persistent state
final boolean active = getEntryValue(ed, getKeyType(), ACTIVITY_STATE, false);
DeviceState deviceState = DeviceState.builder()

5
application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java

@ -129,6 +129,11 @@ public class DefaultAlarmSubscriptionService extends AbstractSubscriptionService
return alarmService.findAlarmByIdAsync(tenantId, alarmId);
}
@Override
public Alarm findAlarmById(TenantId tenantId, AlarmId alarmId) {
return alarmService.findAlarmById(tenantId, alarmId);
}
@Override
public ListenableFuture<AlarmInfo> findAlarmInfoByIdAsync(TenantId tenantId, AlarmId alarmId) {
return alarmService.findAlarmInfoByIdAsync(tenantId, alarmId);

22
application/src/main/java/org/thingsboard/server/service/ttl/EdgeEventsCleanUpService.java

@ -17,15 +17,22 @@ package org.thingsboard.server.service.ttl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.edge.EdgeEventService;
import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import java.util.concurrent.TimeUnit;
import static org.thingsboard.server.dao.model.ModelConstants.EDGE_EVENT_COLUMN_FAMILY_NAME;
@TbCoreComponent
@Slf4j
@Service
@ConditionalOnExpression("${sql.ttl.edge_events.enabled:true} && ${sql.ttl.edge_events.edge_event_ttl:0} > 0")
public class EdgeEventsCleanUpService extends AbstractCleanUpService {
public static final String RANDOM_DELAY_INTERVAL_MS_EXPRESSION =
@ -34,20 +41,29 @@ public class EdgeEventsCleanUpService extends AbstractCleanUpService {
@Value("${sql.ttl.edge_events.edge_events_ttl}")
private long ttl;
@Value("${sql.ttl.edge_events.enabled}")
@Value("${sql.edge_events.partition_size:168}")
private int partitionSizeInHours;
@Value("${sql.ttl.edge_events.enabled:true}")
private boolean ttlTaskExecutionEnabled;
private final EdgeEventService edgeEventService;
public EdgeEventsCleanUpService(PartitionService partitionService, EdgeEventService edgeEventService) {
private final SqlPartitioningRepository partitioningRepository;
public EdgeEventsCleanUpService(PartitionService partitionService, EdgeEventService edgeEventService, SqlPartitioningRepository partitioningRepository) {
super(partitionService);
this.edgeEventService = edgeEventService;
this.partitioningRepository = partitioningRepository;
}
@Scheduled(initialDelayString = RANDOM_DELAY_INTERVAL_MS_EXPRESSION, fixedDelayString = "${sql.ttl.edge_events.execution_interval_ms}")
public void cleanUp() {
long edgeEventsExpTime = System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(ttl);
if (ttlTaskExecutionEnabled && isSystemTenantPartitionMine()) {
edgeEventService.cleanupEvents(ttl);
edgeEventService.cleanupEvents(edgeEventsExpTime);
} else {
partitioningRepository.cleanupPartitionsCache(EDGE_EVENT_COLUMN_FAMILY_NAME, edgeEventsExpTime, TimeUnit.HOURS.toMillis(partitionSizeInHours));
}
}

21
application/src/main/resources/thingsboard.yml

@ -109,7 +109,7 @@ security:
# JWT Token parameters
jwt:
tokenExpirationTime: "${JWT_TOKEN_EXPIRATION_TIME:9000}" # Number of seconds (2.5 hours)
refreshTokenExpTime: "${JWT_REFRESH_TOKEN_EXPIRATION_TIME:604800}" # Number of seconds (1 week)
refreshTokenExpTime: "${JWT_REFRESH_TOKEN_EXPIRATION_TIME:604800}" # Number of seconds (1 week).
tokenIssuer: "${JWT_TOKEN_ISSUER:thingsboard.io}"
tokenSigningKey: "${JWT_TOKEN_SIGNING_KEY:thingsboardDefaultSigningKey}"
# Enable/disable access to Tenant Administrators JWT token by System Administrator or Customer Users JWT token by Tenant Administrator
@ -200,6 +200,14 @@ cassandra:
username: "${CASSANDRA_USERNAME:}"
# Specify your password
password: "${CASSANDRA_PASSWORD:}"
# Astra DB connect https://astra.datastax.com/
cloud:
# /etc/thingsboard/astra/secure-connect-thingsboard.zip
secure_connect_bundle_path: "${CASSANDRA_CLOUD_SECURE_BUNDLE_PATH:}"
# DucitQPHMzPCBOZqFYexAfKk
client_id: "${CASSANDRA_CLOUD_CLIENT_ID:}"
# ZnF7FpuHp43FP5BzM+KY8wGmSb4Ql6BhT4Z7sOU13ze+gXQ-n7OkFpNuB,oACUIQObQnK0g4bSPoZhK5ejkcF9F.j6f64j71Sr.tiRe0Fsq2hPS1ZCGSfAaIgg63IydG
client_secret: "${CASSANDRA_CLOUD_CLIENT_SECRET:}"
# Cassandra cluster connection socket parameters #
socket:
@ -265,6 +273,7 @@ sql:
batch_size: "${SQL_EDGE_EVENTS_BATCH_SIZE:1000}"
batch_max_delay: "${SQL_EDGE_EVENTS_BATCH_MAX_DELAY_MS:100}"
stats_print_interval_ms: "${SQL_EDGE_EVENTS_BATCH_STATS_PRINT_MS:10000}"
partition_size: "${SQL_EDGE_EVENTS_PARTITION_SIZE_HOURS:168}" # Number of hours to partition the events. The current value corresponds to one week.
audit_logs:
partition_size: "${SQL_AUDIT_LOGS_PARTITION_SIZE_HOURS:168}" # Default value - 1 week
# Specify whether to sort entities before batch update. Should be enabled for cluster mode to avoid deadlocks
@ -310,7 +319,9 @@ sql:
ttl: "${SQL_TTL_AUDIT_LOGS_SECS:0}" # Disabled by default. Accuracy of the cleanup depends on the sql.audit_logs.partition_size
checking_interval_ms: "${SQL_TTL_AUDIT_LOGS_CHECKING_INTERVAL_MS:86400000}" # Default value - 1 day
relations:
max_level: "${SQL_RELATIONS_MAX_LEVEL:50}" # //This value has to be reasonable small to prevent infinite recursion as early as possible
max_level: "${SQL_RELATIONS_MAX_LEVEL:50}" # This value has to be reasonable small to prevent infinite recursion as early as possible
pool_size: "${SQL_RELATIONS_POOL_SIZE:4}" # This value has to be reasonable small to prevent relation query blocking all other DB calls
query_timeout: "${SQL_RELATIONS_QUERY_TIMEOUT_SEC:20}" # This value has to be reasonable small to prevent relation query blocking all other DB calls
# Actor system parameters
actors:
@ -418,8 +429,9 @@ cache:
attributes:
timeToLiveInMinutes: "${CACHE_SPECS_ATTRIBUTES_TTL:1440}"
maxSize: "${CACHE_SPECS_ATTRIBUTES_MAX_SIZE:100000}"
usersUpdateTime:
timeToLiveInMinutes: "${CACHE_SPECS_USERS_UPDATE_TIME_TTL:20000}"
userSessionsInvalidation:
# The value of this TTL is ignored and replaced by JWT refresh token expiration time
timeToLiveInMinutes: "0"
maxSize: "${CACHE_SPECS_USERS_UPDATE_TIME_MAX_SIZE:10000}"
otaPackages:
timeToLiveInMinutes: "${CACHE_SPECS_OTA_PACKAGES_TTL:60}"
@ -531,6 +543,7 @@ spring.servlet.multipart.max-file-size: "50MB"
spring.servlet.multipart.max-request-size: "50MB"
spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation: "true"
# Note: as for current Spring JPA version, custom NullHandling for the Sort.Order is ignored and this parameter is used
spring.jpa.properties.hibernate.order_by.default_null_ordering: "${SPRING_JPA_PROPERTIES_HIBERNATE_ORDER_BY_DEFAULT_NULL_ORDERING:last}"
# SQL DAO Configuration

14
application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java

@ -234,7 +234,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
customerUser = createUserAndLogin(customerUser, CUSTOMER_USER_PASSWORD);
customerUserId = customerUser.getId();
logout();
resetTokens();
log.info("Executed web test setup");
}
@ -336,7 +336,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
Assert.assertNotNull(savedDifferentCustomer);
differentCustomerId = savedDifferentCustomer.getId();
logout();
resetTokens();
}
protected void deleteDifferentTenant() throws Exception {
@ -350,7 +350,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
protected User createUserAndLogin(User user, String password) throws Exception {
User savedUser = doPost("/api/user", user, User.class);
logout();
resetTokens();
JsonNode activateRequest = getActivateRequest(password);
JsonNode tokenInfo = readResponse(doPost("/api/noauth/activate", activateRequest).andExpect(status().isOk()), JsonNode.class);
validateAndSetJwtToken(tokenInfo, user.getEmail());
@ -375,7 +375,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
}
protected void login(String username, String password) throws Exception {
logout();
resetTokens();
JsonNode tokenInfo = readResponse(doPost("/api/auth/login", new LoginRequest(username, password)).andExpect(status().isOk()), JsonNode.class);
validateAndSetJwtToken(tokenInfo, username);
}
@ -411,12 +411,16 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
Assert.assertEquals(username, subject);
}
protected void logout() throws Exception {
protected void resetTokens() throws Exception {
this.token = null;
this.refreshToken = null;
this.username = null;
}
protected void logout() throws Exception {
doPost("/api/auth/logout").andExpect(status().isOk());
}
protected void setJwtToken(MockHttpServletRequestBuilder request) {
if (this.token != null) {
request.header(ThingsboardSecurityConfiguration.JWT_TOKEN_HEADER_PARAM, "Bearer " + this.token);

4
application/src/test/java/org/thingsboard/server/controller/BaseAlarmControllerTest.java

@ -77,7 +77,7 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest {
device.setCustomerId(customerId);
customerDevice = doPost("/api/device", device, Device.class);
logout();
resetTokens();
}
@After
@ -423,7 +423,7 @@ public abstract class BaseAlarmControllerTest extends AbstractControllerTest {
testNotifyEntityNeverMsgToEdgeServiceOneTime(alarm, alarm.getId(), tenantId, ActionType.ADDED);
logout();
resetTokens();
JsonNode publicLoginRequest = JacksonUtil.toJsonNode("{\"publicId\": \"" + publicId + "\"}");
JsonNode tokens = doPost("/api/auth/login/public", publicLoginRequest, JsonNode.class);

12
application/src/test/java/org/thingsboard/server/controller/BaseAuthControllerTest.java

@ -15,13 +15,15 @@
*/
package org.thingsboard.server.controller;
import org.junit.Test;
import org.thingsboard.server.common.data.security.Authority;
import java.util.concurrent.TimeUnit;
import static org.hamcrest.Matchers.is;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import org.thingsboard.server.common.data.security.Authority;
import org.junit.Test;
public abstract class BaseAuthControllerTest extends AbstractControllerTest {
@Test
@ -57,9 +59,13 @@ public abstract class BaseAuthControllerTest extends AbstractControllerTest {
.andExpect(jsonPath("$.authority",is(Authority.SYS_ADMIN.name())))
.andExpect(jsonPath("$.email",is(SYS_ADMIN_EMAIL)));
TimeUnit.SECONDS.sleep(1); //We need to make sure that event for invalidating token was successfully processed
logout();
doGet("/api/auth/user")
.andExpect(status().isUnauthorized());
resetTokens();
}
@Test

160
application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java

@ -16,12 +16,6 @@
package org.thingsboard.server.controller;
import com.fasterxml.jackson.core.type.TypeReference;
import com.github.os72.protobuf.dynamic.DynamicSchema;
import com.google.protobuf.Descriptors;
import com.google.protobuf.DynamicMessage;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.util.JsonFormat;
import com.squareup.wire.schema.internal.parser.ProtoFileElement;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@ -46,11 +40,9 @@ import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.JsonTransportPayloadConfiguration;
import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration;
import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
@ -62,13 +54,9 @@ import org.thingsboard.server.dao.exception.DataValidationException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import static org.hamcrest.Matchers.containsString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE;
import static org.thingsboard.server.common.data.ota.OtaPackageType.SOFTWARE;
@ -749,137 +737,6 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController
"}", "[Transport Configuration] invalid attributes proto schema provided! OneOf definition groups don't support!");
}
@Test
public void testSaveProtoDeviceProfileWithMessageNestedTypes() throws Exception {
String schema = "syntax = \"proto3\";\n" +
"\n" +
"package testnested;\n" +
"\n" +
"message Outer {\n" +
" message MiddleAA {\n" +
" message Inner {\n" +
" optional int64 ival = 1;\n" +
" optional bool booly = 2;\n" +
" }\n" +
" Inner inner = 1;\n" +
" }\n" +
" message MiddleBB {\n" +
" message Inner {\n" +
" optional int32 ival = 1;\n" +
" optional bool booly = 2;\n" +
" }\n" +
" Inner inner = 1;\n" +
" }\n" +
" MiddleAA middleAA = 1;\n" +
" MiddleBB middleBB = 2;\n" +
"}";
DynamicSchema dynamicSchema = getDynamicSchema(schema);
assertNotNull(dynamicSchema);
Set<String> messageTypes = dynamicSchema.getMessageTypes();
assertEquals(5, messageTypes.size());
assertTrue(messageTypes.contains("testnested.Outer"));
assertTrue(messageTypes.contains("testnested.Outer.MiddleAA"));
assertTrue(messageTypes.contains("testnested.Outer.MiddleAA.Inner"));
assertTrue(messageTypes.contains("testnested.Outer.MiddleBB"));
assertTrue(messageTypes.contains("testnested.Outer.MiddleBB.Inner"));
DynamicMessage.Builder middleAAInnerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA.Inner");
Descriptors.Descriptor middleAAInnerMsgDescriptor = middleAAInnerMsgBuilder.getDescriptorForType();
DynamicMessage middleAAInnerMsg = middleAAInnerMsgBuilder
.setField(middleAAInnerMsgDescriptor.findFieldByName("ival"), 1L)
.setField(middleAAInnerMsgDescriptor.findFieldByName("booly"), true)
.build();
DynamicMessage.Builder middleAAMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA");
Descriptors.Descriptor middleAAMsgDescriptor = middleAAMsgBuilder.getDescriptorForType();
DynamicMessage middleAAMsg = middleAAMsgBuilder
.setField(middleAAMsgDescriptor.findFieldByName("inner"), middleAAInnerMsg)
.build();
DynamicMessage.Builder middleBBInnerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA.Inner");
Descriptors.Descriptor middleBBInnerMsgDescriptor = middleBBInnerMsgBuilder.getDescriptorForType();
DynamicMessage middleBBInnerMsg = middleBBInnerMsgBuilder
.setField(middleBBInnerMsgDescriptor.findFieldByName("ival"), 0L)
.setField(middleBBInnerMsgDescriptor.findFieldByName("booly"), false)
.build();
DynamicMessage.Builder middleBBMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleBB");
Descriptors.Descriptor middleBBMsgDescriptor = middleBBMsgBuilder.getDescriptorForType();
DynamicMessage middleBBMsg = middleBBMsgBuilder
.setField(middleBBMsgDescriptor.findFieldByName("inner"), middleBBInnerMsg)
.build();
DynamicMessage.Builder outerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer");
Descriptors.Descriptor outerMsgBuilderDescriptor = outerMsgBuilder.getDescriptorForType();
DynamicMessage outerMsg = outerMsgBuilder
.setField(outerMsgBuilderDescriptor.findFieldByName("middleAA"), middleAAMsg)
.setField(outerMsgBuilderDescriptor.findFieldByName("middleBB"), middleBBMsg)
.build();
assertEquals("{\n" +
" \"middleAA\": {\n" +
" \"inner\": {\n" +
" \"ival\": \"1\",\n" +
" \"booly\": true\n" +
" }\n" +
" },\n" +
" \"middleBB\": {\n" +
" \"inner\": {\n" +
" \"ival\": 0,\n" +
" \"booly\": false\n" +
" }\n" +
" }\n" +
"}", dynamicMsgToJson(outerMsgBuilderDescriptor, outerMsg.toByteArray()));
}
@Test
public void testSaveProtoDeviceProfileWithMessageOneOfs() throws Exception {
String schema = "syntax = \"proto3\";\n" +
"\n" +
"package testoneofs;\n" +
"\n" +
"message SubMessage {\n" +
" repeated string name = 1;\n" +
"}\n" +
"\n" +
"message SampleMessage {\n" +
" optional int32 id = 1;\n" +
" oneof testOneOf {\n" +
" string name = 4;\n" +
" SubMessage subMessage = 9;\n" +
" }\n" +
"}";
DynamicSchema dynamicSchema = getDynamicSchema(schema);
assertNotNull(dynamicSchema);
Set<String> messageTypes = dynamicSchema.getMessageTypes();
assertEquals(2, messageTypes.size());
assertTrue(messageTypes.contains("testoneofs.SubMessage"));
assertTrue(messageTypes.contains("testoneofs.SampleMessage"));
DynamicMessage.Builder sampleMsgBuilder = dynamicSchema.newMessageBuilder("testoneofs.SampleMessage");
Descriptors.Descriptor sampleMsgDescriptor = sampleMsgBuilder.getDescriptorForType();
assertNotNull(sampleMsgDescriptor);
List<Descriptors.FieldDescriptor> fields = sampleMsgDescriptor.getFields();
assertEquals(3, fields.size());
DynamicMessage sampleMsg = sampleMsgBuilder
.setField(sampleMsgDescriptor.findFieldByName("name"), "Bob")
.build();
assertEquals("{\n" + " \"name\": \"Bob\"\n" + "}", dynamicMsgToJson(sampleMsgDescriptor, sampleMsg.toByteArray()));
DynamicMessage.Builder subMsgBuilder = dynamicSchema.newMessageBuilder("testoneofs.SubMessage");
Descriptors.Descriptor subMsgDescriptor = subMsgBuilder.getDescriptorForType();
DynamicMessage subMsg = subMsgBuilder
.addRepeatedField(subMsgDescriptor.findFieldByName("name"), "Alice")
.addRepeatedField(subMsgDescriptor.findFieldByName("name"), "John")
.build();
DynamicMessage sampleMsgWithOneOfSubMessage = sampleMsgBuilder.setField(sampleMsgDescriptor.findFieldByName("subMessage"), subMsg).build();
assertEquals("{\n" + " \"subMessage\": {\n" + " \"name\": [\"Alice\", \"John\"]\n" + " }\n" + "}",
dynamicMsgToJson(sampleMsgDescriptor, sampleMsgWithOneOfSubMessage.toByteArray()));
}
@Test
public void testSaveProtoDeviceProfileWithInvalidTelemetrySchemaTsField() throws Exception {
testSaveDeviceProfileWithInvalidProtoSchema("syntax =\"proto3\";\n" +
@ -1127,23 +984,6 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController
tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(errorMsg));
}
private DynamicSchema getDynamicSchema(String schema) throws Exception {
DeviceProfile deviceProfile = testSaveDeviceProfileWithProtoPayloadType(schema);
DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration();
assertTrue(transportConfiguration instanceof MqttDeviceProfileTransportConfiguration);
MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = (MqttDeviceProfileTransportConfiguration) transportConfiguration;
TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttDeviceProfileTransportConfiguration.getTransportPayloadTypeConfiguration();
assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration);
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration;
ProtoFileElement protoFile = protoTransportPayloadConfiguration.getTransportProtoSchema(schema);
return protoTransportPayloadConfiguration.getDynamicSchema(protoFile, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA);
}
private String dynamicMsgToJson(Descriptors.Descriptor descriptor, byte[] payload) throws InvalidProtocolBufferException {
DynamicMessage dynamicMessage = DynamicMessage.parseFrom(descriptor, payload);
return JsonFormat.printer().includingDefaultValueFields().print(dynamicMessage);
}
@Test
public void testDeleteDeviceProfileWithDeleteRelationsOk() throws Exception {
DeviceProfileId deviceProfileId = savedDeviceProfile("DeviceProfile for Test WithRelationsOk").getId();

69
application/src/test/java/org/thingsboard/server/controller/BaseEdgeEventControllerTest.java

@ -22,6 +22,9 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.test.mock.mockito.SpyBean;
import org.springframework.test.context.TestPropertySource;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.Tenant;
@ -29,16 +32,27 @@ import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.edge.EdgeEvent;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.TimePageLink;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.edge.EdgeEventDao;
import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository;
import org.thingsboard.server.service.ttl.EdgeEventsCleanUpService;
import java.time.LocalDate;
import java.time.ZoneOffset;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.verify;
import static org.assertj.core.api.Assertions.assertThat;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@TestPropertySource(properties = {
@ -50,6 +64,18 @@ public abstract class BaseEdgeEventControllerTest extends AbstractControllerTest
private Tenant savedTenant;
private User tenantAdmin;
@Autowired
private EdgeEventDao edgeEventDao;
@SpyBean
private SqlPartitioningRepository partitioningRepository;
@Autowired
private EdgeEventsCleanUpService edgeEventsCleanUpService;
@Value("#{${sql.edge_events.partition_size} * 60 * 60 * 1000}")
private long partitionDurationInMs;
@Value("${sql.ttl.edge_events.edge_event_ttl}")
private long edgeEventTtlInSec;
@Before
public void beforeTest() throws Exception {
loginSysAdmin();
@ -114,6 +140,34 @@ public abstract class BaseEdgeEventControllerTest extends AbstractControllerTest
Assert.assertTrue(edgeEvents.stream().anyMatch(ee -> EdgeEventType.RELATION.equals(ee.getType())));
}
@Test
public void saveEdgeEvent_thenCreatePartitionIfNotExist() {
reset(partitioningRepository);
EdgeEvent edgeEvent = createEdgeEvent();
verify(partitioningRepository).createPartitionIfNotExists(eq("edge_event"), eq(edgeEvent.getCreatedTime()), eq(partitionDurationInMs));
List<Long> partitions = partitioningRepository.fetchPartitions("edge_event");
assertThat(partitions).singleElement().satisfies(partitionStartTs -> {
assertThat(partitionStartTs).isEqualTo(partitioningRepository.calculatePartitionStartTime(edgeEvent.getCreatedTime(), partitionDurationInMs));
});
}
@Test
public void cleanUpEdgeEventByTtl_dropOldPartitions() {
long oldEdgeEventTs = LocalDate.of(2020, 10, 1).atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli();
long partitionStartTs = partitioningRepository.calculatePartitionStartTime(oldEdgeEventTs, partitionDurationInMs);
partitioningRepository.createPartitionIfNotExists("edge_event", oldEdgeEventTs, partitionDurationInMs);
List<Long> partitions = partitioningRepository.fetchPartitions("edge_event");
assertThat(partitions).contains(partitionStartTs);
edgeEventsCleanUpService.cleanUp();
partitions = partitioningRepository.fetchPartitions("edge_event");
assertThat(partitions).doesNotContain(partitionStartTs);
assertThat(partitions).allSatisfy(partitionsStart -> {
long partitionEndTs = partitionsStart + partitionDurationInMs;
assertThat(partitionEndTs).isGreaterThan(System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(edgeEventTtlInSec));
});
}
private List<EdgeEvent> findEdgeEvents(EdgeId edgeId) throws Exception {
return doGetTypedWithTimePageLink("/api/edge/" + edgeId.toString() + "/events?",
new TypeReference<PageData<EdgeEvent>>() {
@ -134,4 +188,19 @@ public abstract class BaseEdgeEventControllerTest extends AbstractControllerTest
return asset;
}
private EdgeEvent createEdgeEvent() {
EdgeEvent edgeEvent = new EdgeEvent();
edgeEvent.setCreatedTime(System.currentTimeMillis());
edgeEvent.setTenantId(tenantId);
edgeEvent.setAction(EdgeEventActionType.ADDED);
edgeEvent.setEntityId(tenantAdmin.getUuidId());
edgeEvent.setType(EdgeEventType.ALARM);
try {
edgeEventDao.saveAsync(edgeEvent).get();
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}
return edgeEvent;
}
}

8
application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java

@ -106,7 +106,7 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest {
ActionType.ADDED, ActionType.ADDED, 1, 1, 1);
Mockito.reset(tbClusterService, auditLogService);
logout();
resetTokens();
doGet("/api/noauth/activate?activateToken={activateToken}", TestMailService.currentActivateToken)
.andExpect(status().isSeeOther())
.andExpect(header().string(HttpHeaders.LOCATION, "/login/createPassword?activateToken=" + TestMailService.currentActivateToken));
@ -123,7 +123,7 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest {
.andExpect(jsonPath("$.authority", is(Authority.TENANT_ADMIN.name())))
.andExpect(jsonPath("$.email", is(email)));
logout();
resetTokens();
login(email, "testPassword");
@ -218,7 +218,7 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest {
user.setLastName("Downs");
User savedUser = createUserAndLogin(user, "testPassword1");
logout();
resetTokens();
JsonNode resetPasswordByEmailRequest = new ObjectMapper().createObjectNode()
.put("email", email);
@ -244,7 +244,7 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest {
.andExpect(jsonPath("$.authority", is(Authority.TENANT_ADMIN.name())))
.andExpect(jsonPath("$.email", is(email)));
logout();
resetTokens();
login(email, "testPassword2");
doGet("/api/auth/user")

3
application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java

@ -15,7 +15,6 @@
*/
package org.thingsboard.server.queue.discovery;
import com.datastax.driver.core.utils.UUIDs;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@ -136,7 +135,7 @@ public class HashPartitionServiceTest {
Random random = new Random();
long ts = new SimpleDateFormat("dd-MM-yyyy").parse("06-12-2016").getTime() - TimeUnit.DAYS.toMillis(tenantCount);
for (int tenantIndex = 0; tenantIndex < tenantCount; tenantIndex++) {
TenantId tenantId = new TenantId(UUIDs.startOf(ts));
TenantId tenantId = new TenantId(Uuids.startOf(ts));
ts += TimeUnit.DAYS.toMillis(1) + random.nextInt(1000);
for (int queueIndex = 0; queueIndex < queueCount; queueIndex++) {
QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, "queue" + queueIndex, tenantId);

115
application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java

@ -0,0 +1,115 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.entitiy.alarm;
import com.google.common.util.concurrent.Futures;
import lombok.extern.slf4j.Slf4j;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.boot.test.mock.mockito.SpyBean;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit4.SpringRunner;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.alarm.Alarm;
import org.thingsboard.server.common.data.alarm.AlarmStatus;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.dao.alarm.AlarmService;
import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.service.entitiy.TbNotificationEntityService;
import org.thingsboard.server.service.executors.DbCallbackExecutorService;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import org.thingsboard.server.service.telemetry.AlarmSubscriptionService;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@Slf4j
@RunWith(SpringRunner.class)
@ContextConfiguration(classes = DefaultTbAlarmService.class)
@TestPropertySource(properties = {
"server.log_controller_error_stack_trace=false"
})
public class DefaultTbAlarmServiceTest {
@MockBean
protected DbCallbackExecutorService dbExecutor;
@MockBean
protected TbNotificationEntityService notificationEntityService;
@MockBean
protected EdgeService edgeService;
@MockBean
protected AlarmService alarmService;
@MockBean
protected AlarmSubscriptionService alarmSubscriptionService;
@MockBean
protected CustomerService customerService;
@MockBean
protected TbClusterService tbClusterService;
@MockBean
private EntitiesVersionControlService vcService;
@SpyBean
DefaultTbAlarmService service;
@Test
public void testSave() throws ThingsboardException {
var alarm = new Alarm();
when(alarmSubscriptionService.createOrUpdateAlarm(alarm)).thenReturn(alarm);
service.save(alarm, new User());
verify(notificationEntityService, times(1)).notifyCreateOrUpdateAlarm(any(), any(), any());
verify(alarmSubscriptionService, times(1)).createOrUpdateAlarm(eq(alarm));
}
@Test
public void testAck() {
var alarm = new Alarm();
alarm.setStatus(AlarmStatus.ACTIVE_UNACK);
when(alarmSubscriptionService.ackAlarm(any(), any(), anyLong())).thenReturn(Futures.immediateFuture(true));
service.ack(alarm, new User());
verify(notificationEntityService, times(1)).notifyCreateOrUpdateAlarm(any(), any(), any());
verify(alarmSubscriptionService, times(1)).ackAlarm(any(), any(), anyLong());
}
@Test
public void testClear() {
var alarm = new Alarm();
alarm.setStatus(AlarmStatus.ACTIVE_ACK);
when(alarmSubscriptionService.clearAlarm(any(), any(), any(), anyLong())).thenReturn(Futures.immediateFuture(true));
service.clear(alarm, new User());
verify(notificationEntityService, times(1)).notifyCreateOrUpdateAlarm(any(), any(), any());
verify(alarmSubscriptionService, times(1)).clearAlarm(any(), any(), any(), anyLong());
}
@Test
public void testDelete() {
service.delete(new Alarm(), new User());
verify(notificationEntityService, times(1)).notifyDeleteAlarm(any(), any(), any(), any(), any(), any(), anyString());
verify(alarmSubscriptionService, times(1)).deleteAlarm(any(), any());
}
}

167
application/src/test/java/org/thingsboard/server/service/security/auth/TokenOutdatingTest.java

@ -15,19 +15,29 @@
*/
package org.thingsboard.server.service.security.auth;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.cache.concurrent.ConcurrentMapCacheManager;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootContextLoader;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.security.authentication.CredentialsExpiredException;
import org.thingsboard.server.common.data.CacheConstants;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit4.SpringRunner;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.security.event.UserAuthDataChangedEvent;
import org.thingsboard.server.common.data.security.event.UserCredentialsInvalidationEvent;
import org.thingsboard.server.common.data.security.event.UserSessionInvalidationEvent;
import org.thingsboard.server.common.data.security.model.JwtToken;
import org.thingsboard.server.config.JwtSettings;
import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.service.security.auth.jwt.JwtAuthenticationProvider;
import org.thingsboard.server.service.security.auth.jwt.RefreshTokenAuthenticationProvider;
@ -39,13 +49,9 @@ import org.thingsboard.server.service.security.model.token.RawAccessJwtToken;
import java.util.UUID;
import static java.util.concurrent.TimeUnit.DAYS;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
@ -53,31 +59,37 @@ import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@ActiveProfiles("test")
@RunWith(SpringRunner.class)
@ContextConfiguration(classes = TokenOutdatingTest.class, loader = SpringBootContextLoader.class)
@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
@ComponentScan({"org.thingsboard.server"})
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@DaoSqlTest
@TestPropertySource(properties = {
"security.jwt.tokenIssuer=test.io",
"security.jwt.tokenSigningKey=secret",
"security.jwt.tokenExpirationTime=600",
"security.jwt.refreshTokenExpTime=15",
// explicitly set the wrong value to check that it is NOT used.
"cache.specs.userSessionsInvalidation.timeToLiveInMinutes=2"
})
public class TokenOutdatingTest {
private JwtAuthenticationProvider accessTokenAuthenticationProvider;
private RefreshTokenAuthenticationProvider refreshTokenAuthenticationProvider;
@Autowired
private TokenOutdatingService tokenOutdatingService;
private ConcurrentMapCacheManager cacheManager;
@Autowired
private ApplicationEventPublisher eventPublisher;
@Autowired
private JwtTokenFactory tokenFactory;
private JwtSettings jwtSettings;
private SecurityUser securityUser;
private UserId userId;
@BeforeEach
@Before
public void setUp() {
jwtSettings = new JwtSettings();
jwtSettings.setTokenIssuer("test.io");
jwtSettings.setTokenExpirationTime((int) MINUTES.toSeconds(10));
jwtSettings.setRefreshTokenExpTime((int) DAYS.toSeconds(7));
jwtSettings.setTokenSigningKey("secret");
tokenFactory = new JwtTokenFactory(jwtSettings);
cacheManager = new ConcurrentMapCacheManager();
tokenOutdatingService = new TokenOutdatingService(cacheManager, tokenFactory, jwtSettings);
tokenOutdatingService.initCache();
userId = new UserId(UUID.randomUUID());
UserId userId = new UserId(UUID.randomUUID());
securityUser = createMockSecurityUser(userId);
UserService userService = mock(UserService.class);
@ -97,28 +109,29 @@ public class TokenOutdatingTest {
@Test
public void testOutdateOldUserTokens() throws Exception {
JwtToken jwtToken = createAccessJwtToken(userId);
JwtToken jwtToken = tokenFactory.createAccessJwtToken(securityUser);
SECONDS.sleep(1); // need to wait before outdating so that outdatage time is strictly after token issue time
tokenOutdatingService.onUserAuthDataChanged(new UserAuthDataChangedEvent(userId));
assertTrue(tokenOutdatingService.isOutdated(jwtToken, userId));
// Token outdatage time is rounded to 1 sec. Need to wait before outdating so that outdatage time is strictly after token issue time
SECONDS.sleep(1);
eventPublisher.publishEvent(new UserCredentialsInvalidationEvent(securityUser.getId()));
assertTrue(tokenOutdatingService.isOutdated(jwtToken, securityUser.getId()));
SECONDS.sleep(1);
JwtToken newJwtToken = tokenFactory.createAccessJwtToken(createMockSecurityUser(userId));
assertFalse(tokenOutdatingService.isOutdated(newJwtToken, userId));
JwtToken newJwtToken = tokenFactory.createAccessJwtToken(securityUser);
assertFalse(tokenOutdatingService.isOutdated(newJwtToken, securityUser.getId()));
}
@Test
public void testAuthenticateWithOutdatedAccessToken() throws InterruptedException {
RawAccessJwtToken accessJwtToken = getRawJwtToken(createAccessJwtToken(userId));
RawAccessJwtToken accessJwtToken = getRawJwtToken(tokenFactory.createAccessJwtToken(securityUser));
assertDoesNotThrow(() -> {
accessTokenAuthenticationProvider.authenticate(new JwtAuthenticationToken(accessJwtToken));
});
SECONDS.sleep(1);
tokenOutdatingService.onUserAuthDataChanged(new UserAuthDataChangedEvent(userId));
eventPublisher.publishEvent(new UserCredentialsInvalidationEvent(securityUser.getId()));
assertThrows(JwtExpiredTokenException.class, () -> {
accessTokenAuthenticationProvider.authenticate(new JwtAuthenticationToken(accessJwtToken));
@ -127,49 +140,94 @@ public class TokenOutdatingTest {
@Test
public void testAuthenticateWithOutdatedRefreshToken() throws InterruptedException {
RawAccessJwtToken refreshJwtToken = getRawJwtToken(createRefreshJwtToken(userId));
RawAccessJwtToken refreshJwtToken = getRawJwtToken(tokenFactory.createRefreshToken(securityUser));
assertDoesNotThrow(() -> {
refreshTokenAuthenticationProvider.authenticate(new RefreshAuthenticationToken(refreshJwtToken));
});
SECONDS.sleep(1);
tokenOutdatingService.onUserAuthDataChanged(new UserAuthDataChangedEvent(userId));
eventPublisher.publishEvent(new UserCredentialsInvalidationEvent(securityUser.getId()));
assertThrows(CredentialsExpiredException.class, () -> {
refreshTokenAuthenticationProvider.authenticate(new RefreshAuthenticationToken(refreshJwtToken));
});
}
// This test takes too long to run and is basically testing the cache logic
// @Test
// public void testTokensOutdatageTimeRemovalFromCache() throws Exception {
// JwtToken jwtToken = tokenFactory.createAccessJwtToken(securityUser);
//
// SECONDS.sleep(1);
// eventPublisher.publishEvent(new UserCredentialsInvalidationEvent(securityUser.getId()));
//
// SECONDS.sleep(1);
//
// assertTrue(tokenOutdatingService.isOutdated(jwtToken, securityUser.getId()));
//
// SECONDS.sleep(30); // refreshTokenExpTime/2
//
// assertTrue(tokenOutdatingService.isOutdated(jwtToken, securityUser.getId()));
//
// SECONDS.sleep(30 + 1); // refreshTokenExpTime/2 + 1
//
// assertFalse(tokenOutdatingService.isOutdated(jwtToken, securityUser.getId()));
// }
@Test
public void testTokensOutdatageTimeRemovalFromCache() throws Exception {
JwtToken jwtToken = createAccessJwtToken(userId);
public void testOnlyOneTokenExpired() throws InterruptedException {
JwtToken jwtToken = tokenFactory.createAccessJwtToken(securityUser);
SECONDS.sleep(1);
tokenOutdatingService.onUserAuthDataChanged(new UserAuthDataChangedEvent(userId));
SecurityUser anotherSecurityUser = new SecurityUser(securityUser, securityUser.isEnabled(), securityUser.getUserPrincipal());
JwtToken anotherJwtToken = tokenFactory.createAccessJwtToken(anotherSecurityUser);
int refreshTokenExpirationTime = 3;
jwtSettings.setRefreshTokenExpTime(refreshTokenExpirationTime);
assertDoesNotThrow(() -> {
accessTokenAuthenticationProvider.authenticate(new JwtAuthenticationToken(getRawJwtToken(jwtToken)));
});
SECONDS.sleep(refreshTokenExpirationTime - 2);
SECONDS.sleep(1);
assertTrue(tokenOutdatingService.isOutdated(jwtToken, userId));
assertNotNull(cacheManager.getCache(CacheConstants.USERS_UPDATE_TIME_CACHE).get(userId.getId().toString()));
eventPublisher.publishEvent(new UserSessionInvalidationEvent(securityUser.getSessionId()));
SECONDS.sleep(3);
assertThrows(JwtExpiredTokenException.class, () -> {
accessTokenAuthenticationProvider.authenticate(new JwtAuthenticationToken(getRawJwtToken(jwtToken)));
});
assertFalse(tokenOutdatingService.isOutdated(jwtToken, userId));
assertNull(cacheManager.getCache(CacheConstants.USERS_UPDATE_TIME_CACHE).get(userId.getId().toString()));
assertDoesNotThrow(() -> {
accessTokenAuthenticationProvider.authenticate(new JwtAuthenticationToken(getRawJwtToken(anotherJwtToken)));
});
}
private JwtToken createAccessJwtToken(UserId userId) {
return tokenFactory.createAccessJwtToken(createMockSecurityUser(userId));
}
@Test
public void testResetAllSessions() throws InterruptedException {
JwtToken jwtToken = tokenFactory.createAccessJwtToken(securityUser);
SecurityUser anotherSecurityUser = new SecurityUser(securityUser, securityUser.isEnabled(), securityUser.getUserPrincipal());
JwtToken anotherJwtToken = tokenFactory.createAccessJwtToken(anotherSecurityUser);
private JwtToken createRefreshJwtToken(UserId userId) {
return tokenFactory.createRefreshToken(createMockSecurityUser(userId));
assertDoesNotThrow(() -> {
accessTokenAuthenticationProvider.authenticate(new JwtAuthenticationToken(getRawJwtToken(jwtToken)));
});
assertDoesNotThrow(() -> {
accessTokenAuthenticationProvider.authenticate(new JwtAuthenticationToken(getRawJwtToken(anotherJwtToken)));
});
SECONDS.sleep(1);
eventPublisher.publishEvent(new UserCredentialsInvalidationEvent(securityUser.getId()));
assertThrows(JwtExpiredTokenException.class, () -> {
accessTokenAuthenticationProvider.authenticate(new JwtAuthenticationToken(getRawJwtToken(jwtToken)));
});
assertThrows(JwtExpiredTokenException.class, () -> {
accessTokenAuthenticationProvider.authenticate(new JwtAuthenticationToken(getRawJwtToken(anotherJwtToken)));
});
}
private RawAccessJwtToken getRawJwtToken(JwtToken token) {
return new RawAccessJwtToken(token.getToken());
}
@ -180,6 +238,7 @@ public class TokenOutdatingTest {
securityUser.setUserPrincipal(new UserPrincipal(UserPrincipal.Type.USER_NAME, securityUser.getEmail()));
securityUser.setAuthority(Authority.CUSTOMER_USER);
securityUser.setId(userId);
securityUser.setSessionId(UUID.randomUUID().toString());
return securityUser;
}
}

48
application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java

@ -15,27 +15,37 @@
*/
package org.thingsboard.server.service.state;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.DeviceIdInfo;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.TsValue;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import java.util.Map;
import java.util.UUID;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.mockito.BDDMockito.willReturn;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.thingsboard.server.service.state.DefaultDeviceStateService.INACTIVITY_TIMEOUT;
@RunWith(MockitoJUnitRunner.class)
public class DefaultDeviceStateServiceTest {
@ -83,4 +93,40 @@ public class DefaultDeviceStateServiceTest {
Mockito.verify(service, times(1)).fetchDeviceStateDataUsingEntityDataQuery(deviceId);
}
@Test
public void givenPersistToTelemetryAndDefaultInactivityTimeoutFetched_whenTransformingToDeviceStateData_thenTryGetInactivityFromAttribute() {
var defaultInactivityTimeoutInSec = 60L;
var latest =
Map.of(
EntityKeyType.TIME_SERIES, Map.of(INACTIVITY_TIMEOUT, new TsValue(0, Long.toString(defaultInactivityTimeoutInSec * 1000))),
EntityKeyType.SERVER_ATTRIBUTE, Map.of(INACTIVITY_TIMEOUT, new TsValue(0, Long.toString(5000L)))
);
process(latest, defaultInactivityTimeoutInSec);
}
@Test
public void givenPersistToTelemetryAndNoInactivityTimeoutFetchedFromTimeSeries_whenTransformingToDeviceStateData_thenTryGetInactivityFromAttribute() {
var defaultInactivityTimeoutInSec = 60L;
var latest =
Map.of(
EntityKeyType.SERVER_ATTRIBUTE, Map.of(INACTIVITY_TIMEOUT, new TsValue(0, Long.toString(5000L)))
);
process(latest, defaultInactivityTimeoutInSec);
}
private void process(Map<EntityKeyType, Map<String, TsValue>> latest, long defaultInactivityTimeoutInSec) {
service.setDefaultInactivityTimeoutInSec(defaultInactivityTimeoutInSec);
service.setDefaultInactivityTimeoutMs(defaultInactivityTimeoutInSec * 1000);
service.setPersistToTelemetry(true);
var deviceUuid = UUID.randomUUID();
var deviceId = new DeviceId(deviceUuid);
DeviceStateData deviceStateData = service.toDeviceStateData(new EntityData(deviceId, latest, Map.of()), new DeviceIdInfo(TenantId.SYS_TENANT_ID.getId(), UUID.randomUUID(), deviceUuid));
Assert.assertEquals(5000L, deviceStateData.getState().getInactivityTimeout());
}
}

12
application/src/test/java/org/thingsboard/server/system/BaseRestApiLimitsTest.java

@ -63,15 +63,15 @@ public abstract class BaseRestApiLimitsTest extends AbstractControllerTest {
public void before() throws Exception {
loginSysAdmin();
tenantProfile = getDefaultTenantProfile();
logout();
resetTokens();
}
@After
public void after() throws Exception {
logout();
resetTokens();
loginSysAdmin();
saveTenantProfileWitConfiguration(tenantProfile, new DefaultTenantProfileConfiguration());
logout();
resetTokens();
service.shutdown();
}
@ -85,7 +85,7 @@ public abstract class BaseRestApiLimitsTest extends AbstractControllerTest {
saveTenantProfileWitConfiguration(tenantProfile, configurationWithCustomerRestLimits);
logout();
resetTokens();
loginCustomerUser();
@ -105,7 +105,7 @@ public abstract class BaseRestApiLimitsTest extends AbstractControllerTest {
saveTenantProfileWitConfiguration(tenantProfile, configurationWithTenantRestLimits);
logout();
resetTokens();
loginCustomerUser();
@ -125,7 +125,7 @@ public abstract class BaseRestApiLimitsTest extends AbstractControllerTest {
saveTenantProfileWitConfiguration(tenantProfile, configurationWithTenantRestLimits);
logout();
resetTokens();
loginTenantAdmin();

1
application/src/test/java/org/thingsboard/server/transport/TransportNoSqlTestSuite.java

@ -35,6 +35,7 @@ public class TransportNoSqlTestSuite {
public static CustomCassandraCQLUnit cassandraUnit =
new CustomCassandraCQLUnit(
Arrays.asList(
new ClassPathCQLDataSet("cassandra/schema-keyspace.cql", false, false),
new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false),
new ClassPathCQLDataSet("cassandra/schema-ts-latest.cql", false, false)
),

6
application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java

@ -27,6 +27,7 @@ import org.eclipse.californium.core.CoapResponse;
import org.eclipse.californium.core.coap.CoAP;
import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.DynamicProtoUtils;
import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration;
import org.thingsboard.server.common.data.device.profile.DefaultCoapDeviceTypeConfiguration;
@ -124,7 +125,6 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap
}
private byte[] getAttributesProtoPayloadBytes() {
DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration();
assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration);
CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration;
@ -134,8 +134,8 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap
TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration();
assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration);
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration;
ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(ATTRIBUTES_SCHEMA_STR);
DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA);
ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceAttributesProtoSchema());
DynamicSchema attributesSchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA);
DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject");
Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType();

9
application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java

@ -28,6 +28,7 @@ import org.eclipse.californium.core.CoapResponse;
import org.eclipse.californium.core.coap.CoAP;
import org.eclipse.californium.core.coap.MediaTypeRegistry;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.DynamicProtoUtils;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration;
@ -158,8 +159,8 @@ public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractC
protected void processOnLoadProtoResponse(CoapResponse response, CoapTestClient client, Integer observe, CountDownLatch latch) {
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = getProtoTransportPayloadConfiguration();
ProtoFileElement rpcRequestProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(RPC_REQUEST_PROTO_SCHEMA);
DynamicSchema rpcRequestProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcRequestProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA);
ProtoFileElement rpcRequestProtoFileElement = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceRpcRequestProtoSchema());
DynamicSchema rpcRequestProtoSchema = DynamicProtoUtils.getDynamicSchema(rpcRequestProtoFileElement, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA);
byte[] requestPayload = response.getPayload();
DynamicMessage.Builder rpcRequestMsg = rpcRequestProtoSchema.newMessageBuilder("RpcRequestMsg");
@ -168,8 +169,8 @@ public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractC
DynamicMessage dynamicMessage = DynamicMessage.parseFrom(rpcRequestMsgDescriptor, requestPayload);
Descriptors.FieldDescriptor requestIdDescriptor = rpcRequestMsgDescriptor.findFieldByName("requestId");
int requestId = (int) dynamicMessage.getField(requestIdDescriptor);
ProtoFileElement rpcResponseProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_RPC_RESPONSE_PROTO_SCHEMA);
DynamicSchema rpcResponseProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcResponseProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA);
ProtoFileElement rpcResponseProtoSchemaFile = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceRpcResponseProtoSchema());
DynamicSchema rpcResponseProtoSchema = DynamicProtoUtils.getDynamicSchema(rpcResponseProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA);
DynamicMessage.Builder rpcResponseBuilder = rpcResponseProtoSchema.newMessageBuilder("RpcResponseMsg");
Descriptors.Descriptor rpcResponseMsgDescriptor = rpcResponseBuilder.getDescriptorForType();
DynamicMessage rpcResponseMsg = rpcResponseBuilder

42
application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesProtoIntegrationTest.java

@ -23,6 +23,7 @@ import lombok.extern.slf4j.Slf4j;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.server.common.data.CoapDeviceType;
import org.thingsboard.server.common.data.DynamicProtoUtils;
import org.thingsboard.server.common.data.TransportPayloadType;
import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration;
@ -55,18 +56,7 @@ public class CoapAttributesProtoIntegrationTest extends CoapAttributesIntegratio
@Test
public void testPushAttributes() throws Exception {
DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration();
assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration);
CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration;
CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration();
assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration);
DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration;
TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration();
assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration);
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration;
ProtoFileElement transportProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_ATTRIBUTES_PROTO_SCHEMA);
DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchemaFile, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA);
DynamicSchema attributesSchema = getDynamicSchema();
DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject");
Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType();
assertNotNull(nestedJsonObjectBuilderDescriptor);
@ -98,18 +88,7 @@ public class CoapAttributesProtoIntegrationTest extends CoapAttributesIntegratio
@Test
public void testPushAttributesWithExplicitPresenceProtoKeys() throws Exception {
DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration();
assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration);
CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration;
CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration();
assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration);
DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration;
TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration();
assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration);
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration;
ProtoFileElement transportProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_ATTRIBUTES_PROTO_SCHEMA);
DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchemaFile, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA);
DynamicSchema attributesSchema = getDynamicSchema();
DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject");
Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType();
assertNotNull(nestedJsonObjectBuilderDescriptor);
@ -135,4 +114,19 @@ public class CoapAttributesProtoIntegrationTest extends CoapAttributesIntegratio
processAttributesTest(Arrays.asList("key1", "key5"), postAttributesMsg.toByteArray(), true);
}
private DynamicSchema getDynamicSchema() {
DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration();
assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration);
CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration;
CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration();
assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration);
DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration;
TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration();
assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration);
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration;
String deviceAttributesProtoSchema = protoTransportPayloadConfiguration.getDeviceAttributesProtoSchema();
ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceAttributesProtoSchema);
return DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA);
}
}

22
application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java

@ -23,6 +23,8 @@ import lombok.extern.slf4j.Slf4j;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.server.common.data.CoapDeviceType;
import org.thingsboard.server.common.data.DeviceProfileProvisionType;
import org.thingsboard.server.common.data.DynamicProtoUtils;
import org.thingsboard.server.common.data.TransportPayloadType;
import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration;
@ -63,8 +65,9 @@ public abstract class AbstractCoapTimeseriesProtoIntegrationTest extends Abstrac
TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration();
assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration);
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration;
ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_TELEMETRY_PROTO_SCHEMA);
DynamicSchema telemetrySchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema");
String deviceTelemetryProtoSchema = protoTransportPayloadConfiguration.getDeviceTelemetryProtoSchema();
ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceTelemetryProtoSchema);
DynamicSchema telemetrySchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.TELEMETRY_PROTO_SCHEMA);
DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject");
Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType();
@ -138,8 +141,9 @@ public abstract class AbstractCoapTimeseriesProtoIntegrationTest extends Abstrac
TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration();
assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration);
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration;
ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(schemaStr);
DynamicSchema telemetrySchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema");
String deviceTelemetryProtoSchema = protoTransportPayloadConfiguration.getDeviceTelemetryProtoSchema();
ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceTelemetryProtoSchema);
DynamicSchema telemetrySchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.TELEMETRY_PROTO_SCHEMA);
DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject");
Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType();
@ -198,8 +202,9 @@ public abstract class AbstractCoapTimeseriesProtoIntegrationTest extends Abstrac
TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration();
assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration);
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration;
ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_TELEMETRY_PROTO_SCHEMA);
DynamicSchema telemetrySchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema");
String deviceTelemetryProtoSchema = protoTransportPayloadConfiguration.getDeviceTelemetryProtoSchema();
ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceTelemetryProtoSchema);
DynamicSchema telemetrySchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.TELEMETRY_PROTO_SCHEMA);
DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject");
Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType();
@ -271,8 +276,9 @@ public abstract class AbstractCoapTimeseriesProtoIntegrationTest extends Abstrac
TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration();
assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration);
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration;
ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(schemaStr);
DynamicSchema telemetrySchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema");
String deviceTelemetryProtoSchema = protoTransportPayloadConfiguration.getDeviceTelemetryProtoSchema();
ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceTelemetryProtoSchema);
DynamicSchema telemetrySchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.TELEMETRY_PROTO_SCHEMA);
DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject");
Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType();

5
application/src/test/java/org/thingsboard/server/transport/mqtt/attributes/AbstractMqttAttributesIntegrationTest.java

@ -25,6 +25,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.test.context.TestPropertySource;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DynamicProtoUtils;
import org.thingsboard.server.common.data.TransportPayloadType;
import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration;
@ -494,8 +495,8 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt
TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttTransportConfiguration.getTransportPayloadTypeConfiguration();
assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration);
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration;
ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(ATTRIBUTES_SCHEMA_STR);
DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA);
ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceAttributesProtoSchema());
DynamicSchema attributesSchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA);
DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject");
Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType();

9
application/src/test/java/org/thingsboard/server/transport/mqtt/rpc/AbstractMqttServerSideRpcIntegrationTest.java

@ -29,6 +29,7 @@ import org.eclipse.paho.client.mqttv3.MqttException;
import org.eclipse.paho.client.mqttv3.MqttMessage;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DynamicProtoUtils;
import org.thingsboard.server.common.data.DeviceTransportType;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.TransportPayloadType;
@ -382,8 +383,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractM
protected byte[] processProtoMessageArrived(String requestTopic, MqttMessage mqttMessage) throws MqttException, InvalidProtocolBufferException {
if (requestTopic.startsWith(BASE_DEVICE_API_TOPIC) || requestTopic.startsWith(BASE_DEVICE_API_TOPIC_V2)) {
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = getProtoTransportPayloadConfiguration();
ProtoFileElement rpcRequestProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(RPC_REQUEST_PROTO_SCHEMA);
DynamicSchema rpcRequestProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcRequestProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA);
ProtoFileElement rpcRequestProtoFileElement = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceRpcRequestProtoSchema());
DynamicSchema rpcRequestProtoSchema = DynamicProtoUtils.getDynamicSchema(rpcRequestProtoFileElement, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA);
byte[] requestPayload = mqttMessage.getPayload();
DynamicMessage.Builder rpcRequestMsg = rpcRequestProtoSchema.newMessageBuilder("RpcRequestMsg");
@ -395,8 +396,8 @@ public abstract class AbstractMqttServerSideRpcIntegrationTest extends AbstractM
for (Descriptors.FieldDescriptor fieldDescriptor: fields) {
assertTrue(dynamicMessage.hasField(fieldDescriptor));
}
ProtoFileElement transportProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_RPC_RESPONSE_PROTO_SCHEMA);
DynamicSchema rpcResponseProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA);
ProtoFileElement rpcResponseProtoFileElement = DynamicProtoUtils.getProtoFileElement(protoTransportPayloadConfiguration.getDeviceRpcResponseProtoSchema());
DynamicSchema rpcResponseProtoSchema = DynamicProtoUtils.getDynamicSchema(rpcResponseProtoFileElement, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA);
DynamicMessage.Builder rpcResponseBuilder = rpcResponseProtoSchema.newMessageBuilder("RpcResponseMsg");
Descriptors.Descriptor rpcResponseMsgDescriptor = rpcResponseBuilder.getDescriptorForType();

6
application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/attributes/MqttAttributesProtoIntegrationTest.java

@ -22,6 +22,7 @@ import com.squareup.wire.schema.internal.parser.ProtoFileElement;
import lombok.extern.slf4j.Slf4j;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.server.common.data.DynamicProtoUtils;
import org.thingsboard.server.common.data.TransportPayloadType;
import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration;
@ -176,8 +177,9 @@ public class MqttAttributesProtoIntegrationTest extends MqttAttributesIntegratio
TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttTransportConfiguration.getTransportPayloadTypeConfiguration();
assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration);
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration;
ProtoFileElement transportProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_ATTRIBUTES_PROTO_SCHEMA);
return protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchemaFile, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA);
String deviceAttributesProtoSchema = protoTransportPayloadConfiguration.getDeviceAttributesProtoSchema();
ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceAttributesProtoSchema);
return DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA);
}
private DynamicMessage getDefaultDynamicMessage() {

16
application/src/test/java/org/thingsboard/server/transport/mqtt/telemetry/timeseries/AbstractMqttTimeseriesProtoIntegrationTest.java

@ -23,6 +23,7 @@ import lombok.extern.slf4j.Slf4j;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DynamicProtoUtils;
import org.thingsboard.server.common.data.TransportPayloadType;
import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration;
@ -117,7 +118,7 @@ public abstract class AbstractMqttTimeseriesProtoIntegrationTest extends Abstrac
.telemetryProtoSchema(schemaStr)
.build();
processBeforeTest(configProperties);
DynamicSchema telemetrySchema = getDynamicSchema(schemaStr);
DynamicSchema telemetrySchema = getDynamicSchema();
DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject");
Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType();
@ -167,7 +168,7 @@ public abstract class AbstractMqttTimeseriesProtoIntegrationTest extends Abstrac
.telemetryTopicFilter(POST_DATA_TELEMETRY_TOPIC)
.build();
processBeforeTest(configProperties);
DynamicSchema telemetrySchema = getDynamicSchema(DEVICE_TELEMETRY_PROTO_SCHEMA);
DynamicSchema telemetrySchema = getDynamicSchema();
DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject");
Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType();
@ -230,7 +231,7 @@ public abstract class AbstractMqttTimeseriesProtoIntegrationTest extends Abstrac
.telemetryProtoSchema(schemaStr)
.build();
processBeforeTest(configProperties);
DynamicSchema telemetrySchema = getDynamicSchema(schemaStr);
DynamicSchema telemetrySchema = getDynamicSchema();
DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject");
Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType();
@ -458,19 +459,20 @@ public abstract class AbstractMqttTimeseriesProtoIntegrationTest extends Abstrac
assertFalse(callback.isPubAckReceived());
}
private DynamicSchema getDynamicSchema(String deviceTelemetryProtoSchema) {
private DynamicSchema getDynamicSchema() {
DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration();
assertTrue(transportConfiguration instanceof MqttDeviceProfileTransportConfiguration);
MqttDeviceProfileTransportConfiguration mqttTransportConfiguration = (MqttDeviceProfileTransportConfiguration) transportConfiguration;
TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = mqttTransportConfiguration.getTransportPayloadTypeConfiguration();
assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration);
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration;
ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(deviceTelemetryProtoSchema);
return protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, "telemetrySchema");
String deviceTelemetryProtoSchema = protoTransportPayloadConfiguration.getDeviceTelemetryProtoSchema();
ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(deviceTelemetryProtoSchema);
return DynamicProtoUtils.getDynamicSchema(protoFileElement, ProtoTransportPayloadConfiguration.TELEMETRY_PROTO_SCHEMA);
}
private DynamicMessage getDefaultDynamicMessage() {
DynamicSchema telemetrySchema = getDynamicSchema(DEVICE_TELEMETRY_PROTO_SCHEMA);
DynamicSchema telemetrySchema = getDynamicSchema();
DynamicMessage.Builder nestedJsonObjectBuilder = telemetrySchema.newMessageBuilder("PostTelemetry.JsonObject.NestedJsonObject");
Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType();

5
application/src/test/resources/application-test.properties

@ -59,4 +59,7 @@ queue.rule-engine.queues[2].processing-strategy.max-pause-between-retries=0
usage.stats.report.enabled=false
sql.audit_logs.partition_size=24
sql.ttl.audit_logs.ttl=2592000
sql.ttl.audit_logs.ttl=2592000
sql.edge_events.partition_size=168
sql.ttl.edge_events.edge_event_ttl=2592000

16
common/cache/src/main/java/org/thingsboard/server/cache/CacheSpecsMap.java

@ -17,9 +17,12 @@ package org.thingsboard.server.cache;
import lombok.Data;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.thingsboard.server.common.data.CacheConstants;
import javax.annotation.PostConstruct;
import java.util.Map;
@Configuration
@ -27,7 +30,20 @@ import java.util.Map;
@Data
public class CacheSpecsMap {
@Value("${security.jwt.refreshTokenExpTime:604800}")
private int refreshTokenExpTime;
@Getter
private Map<String, CacheSpecs> specs;
@PostConstruct
public void replaceTheJWTTokenRefreshExpTime() {
if (specs != null) {
var cacheSpecs = specs.get(CacheConstants.USERS_SESSION_INVALIDATION_CACHE);
if (cacheSpecs != null) {
cacheSpecs.setTimeToLiveInMinutes((refreshTokenExpTime / 60) + 1);
}
}
}
}

2
common/cache/src/main/java/org/thingsboard/server/cache/TbCaffeineCacheConfiguration.java

@ -20,6 +20,7 @@ import com.github.benmanes.caffeine.cache.RemovalCause;
import com.github.benmanes.caffeine.cache.Ticker;
import com.github.benmanes.caffeine.cache.Weigher;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.cache.CacheManager;
import org.springframework.cache.annotation.EnableCaching;
@ -71,6 +72,7 @@ public class TbCaffeineCacheConfiguration {
}
private CaffeineCache buildCache(String name, CacheSpecs cacheSpec) {
final Caffeine<Object, Object> caffeineBuilder
= Caffeine.newBuilder()
.weigher(collectionSafeWeigher())

34
common/cache/src/main/java/org/thingsboard/server/cache/usersUpdateTime/UsersSessionInvalidationCaffeineCache.java

@ -0,0 +1,34 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.usersUpdateTime;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.cache.CacheManager;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cache.CaffeineTbTransactionalCache;
import org.thingsboard.server.common.data.CacheConstants;
@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true)
@Service("UsersSessionInvalidation")
public class UsersSessionInvalidationCaffeineCache extends CaffeineTbTransactionalCache<String, Long> {
@Autowired
public UsersSessionInvalidationCaffeineCache(CacheManager cacheManager) {
super(cacheManager, CacheConstants.USERS_SESSION_INVALIDATION_CACHE);
}
}

36
common/cache/src/main/java/org/thingsboard/server/cache/usersUpdateTime/UsersSessionInvalidationRedisCache.java

@ -0,0 +1,36 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.usersUpdateTime;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cache.CacheSpecsMap;
import org.thingsboard.server.cache.RedisTbTransactionalCache;
import org.thingsboard.server.cache.TBRedisCacheConfiguration;
import org.thingsboard.server.cache.TbFSTRedisSerializer;
import org.thingsboard.server.common.data.CacheConstants;
@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis")
@Service("UsersSessionInvalidation")
public class UsersSessionInvalidationRedisCache extends RedisTbTransactionalCache<String, Long> {
@Autowired
public UsersSessionInvalidationRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) {
super(CacheConstants.USERS_SESSION_INVALIDATION_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbFSTRedisSerializer<>());
}
}

2
common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java

@ -48,6 +48,8 @@ public interface AlarmService {
ListenableFuture<AlarmOperationResult> clearAlarm(TenantId tenantId, AlarmId alarmId, JsonNode details, long clearTs);
Alarm findAlarmById(TenantId tenantId, AlarmId alarmId);
ListenableFuture<Alarm> findAlarmByIdAsync(TenantId tenantId, AlarmId alarmId);
ListenableFuture<AlarmInfo> findAlarmInfoByIdAsync(TenantId tenantId, AlarmId alarmId);

16
common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/AbstractCassandraCluster.java

@ -24,11 +24,13 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.env.Environment;
import org.springframework.core.env.Profiles;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.dao.cassandra.guava.GuavaSession;
import org.thingsboard.server.dao.cassandra.guava.GuavaSessionBuilder;
import org.thingsboard.server.dao.cassandra.guava.GuavaSessionUtils;
import javax.annotation.PreDestroy;
import java.nio.file.Paths;
@Slf4j
public abstract class AbstractCassandraCluster {
@ -40,6 +42,13 @@ public abstract class AbstractCassandraCluster {
@Value("${cassandra.local_datacenter:datacenter1}")
private String localDatacenter;
@Value("${cassandra.cloud.secure_connect_bundle_path:}")
private String cloudSecureConnectBundlePath;
@Value("${cassandra.cloud.client_id:}")
private String cloudClientId;
@Value("${cassandra.cloud.client_secret:}")
private String cloudClientSecret;
@Autowired
private CassandraDriverOptions driverOptions;
@ -86,7 +95,14 @@ public abstract class AbstractCassandraCluster {
this.sessionBuilder.withKeyspace(this.keyspaceName);
}
this.sessionBuilder.withLocalDatacenter(localDatacenter);
if (StringUtils.isNotBlank(cloudSecureConnectBundlePath)) {
this.sessionBuilder.withCloudSecureConnectBundle(Paths.get(cloudSecureConnectBundlePath));
this.sessionBuilder.withAuthCredentials(cloudClientId, cloudClientSecret);
}
session = sessionBuilder.build();
if (this.metrics && this.jmx) {
MetricRegistry registry =
session.getMetrics().orElseThrow(

22
common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/guava/GuavaDriverContext.java

@ -40,26 +40,8 @@ import java.util.function.Predicate;
*/
public class GuavaDriverContext extends DefaultDriverContext {
public GuavaDriverContext(
DriverConfigLoader configLoader,
List<TypeCodec<?>> typeCodecs,
NodeStateListener nodeStateListener,
SchemaChangeListener schemaChangeListener,
RequestTracker requestTracker,
Map<String, String> localDatacenters,
Map<String, Predicate<Node>> nodeFilters,
ClassLoader classLoader) {
super(
configLoader,
ProgrammaticArguments.builder()
.addTypeCodecs(typeCodecs.toArray(new TypeCodec<?>[0]))
.withNodeStateListener(nodeStateListener)
.withSchemaChangeListener(schemaChangeListener)
.withRequestTracker(requestTracker)
.withLocalDatacenters(localDatacenters)
.withNodeFilters(nodeFilters)
.withClassLoader(classLoader)
.build());
public GuavaDriverContext(DriverConfigLoader configLoader, ProgrammaticArguments programmaticArguments) {
super(configLoader, programmaticArguments);
}
@Override

14
common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/guava/GuavaSessionBuilder.java

@ -25,18 +25,8 @@ import edu.umd.cs.findbugs.annotations.NonNull;
public class GuavaSessionBuilder extends SessionBuilder<GuavaSessionBuilder, GuavaSession> {
@Override
protected DriverContext buildContext(
DriverConfigLoader configLoader,
ProgrammaticArguments programmaticArguments) {
return new GuavaDriverContext(
configLoader,
programmaticArguments.getTypeCodecs(),
programmaticArguments.getNodeStateListener(),
programmaticArguments.getSchemaChangeListener(),
programmaticArguments.getRequestTracker(),
programmaticArguments.getLocalDatacenters(),
programmaticArguments.getNodeFilters(),
programmaticArguments.getClassLoader());
protected DriverContext buildContext(DriverConfigLoader configLoader, ProgrammaticArguments programmaticArguments) {
return new GuavaDriverContext(configLoader, programmaticArguments);
}
@Override

27
common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDaoNonCloud.java

@ -0,0 +1,27 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.util;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@Retention(RetentionPolicy.RUNTIME)
@ConditionalOnExpression("('${database.ts.type}'=='cassandra' || '${database.ts_latest.type}'=='cassandra') " +
"&& ('${cassandra.cloud.secure_connect_bundle_path}' == null || '${cassandra.cloud.secure_connect_bundle_path}'.isBlank() )")
public @interface NoSqlAnyDaoNonCloud {
}

4
common/data/pom.xml

@ -108,6 +108,10 @@
<groupId>de.ruedigermoeller</groupId>
<artifactId>fst</artifactId>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
</dependency>
</dependencies>
<build>

2
common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java

@ -32,7 +32,7 @@ public class CacheConstants {
public static final String ASSET_PROFILE_CACHE = "assetProfiles";
public static final String ATTRIBUTES_CACHE = "attributes";
public static final String USERS_UPDATE_TIME_CACHE = "usersUpdateTime";
public static final String USERS_SESSION_INVALIDATION_CACHE = "userSessionsInvalidation";
public static final String OTA_PACKAGE_CACHE = "otaPackages";
public static final String OTA_PACKAGE_DATA_CACHE = "otaPackagesData";
public static final String REPOSITORY_SETTINGS_CACHE = "repositorySettings";

301
common/data/src/main/java/org/thingsboard/server/common/data/DynamicProtoUtils.java

@ -0,0 +1,301 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data;
import com.github.os72.protobuf.dynamic.DynamicSchema;
import com.github.os72.protobuf.dynamic.EnumDefinition;
import com.github.os72.protobuf.dynamic.MessageDefinition;
import com.google.protobuf.Descriptors;
import com.google.protobuf.DynamicMessage;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.util.JsonFormat;
import com.squareup.wire.Syntax;
import com.squareup.wire.schema.Field;
import com.squareup.wire.schema.Location;
import com.squareup.wire.schema.internal.parser.EnumConstantElement;
import com.squareup.wire.schema.internal.parser.EnumElement;
import com.squareup.wire.schema.internal.parser.FieldElement;
import com.squareup.wire.schema.internal.parser.MessageElement;
import com.squareup.wire.schema.internal.parser.OneOfElement;
import com.squareup.wire.schema.internal.parser.ProtoFileElement;
import com.squareup.wire.schema.internal.parser.ProtoParser;
import com.squareup.wire.schema.internal.parser.TypeElement;
import lombok.extern.slf4j.Slf4j;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
@Slf4j
public class DynamicProtoUtils {
public static final Location LOCATION = new Location("", "", -1, -1);
public static final String PROTO_3_SYNTAX = "proto3";
public static Descriptors.Descriptor getDescriptor(String protoSchema, String schemaName) {
try {
DynamicMessage.Builder builder = getDynamicMessageBuilder(protoSchema, schemaName);
return builder.getDescriptorForType();
} catch (Exception e) {
log.warn("Failed to get Message Descriptor due to {}", e.getMessage());
return null;
}
}
public static DynamicMessage.Builder getDynamicMessageBuilder(String protoSchema, String schemaName) {
ProtoFileElement protoFileElement = getProtoFileElement(protoSchema);
DynamicSchema dynamicSchema = getDynamicSchema(protoFileElement, schemaName);
String lastMsgName = getMessageTypes(protoFileElement.getTypes()).stream()
.map(MessageElement::getName).reduce((previous, last) -> last).get();
return dynamicSchema.newMessageBuilder(lastMsgName);
}
public static DynamicSchema getDynamicSchema(ProtoFileElement protoFileElement, String schemaName) {
DynamicSchema.Builder schemaBuilder = DynamicSchema.newBuilder();
schemaBuilder.setName(schemaName);
schemaBuilder.setSyntax(PROTO_3_SYNTAX);
schemaBuilder.setPackage(StringUtils.isNotEmpty(protoFileElement.getPackageName()) ?
protoFileElement.getPackageName() : schemaName.toLowerCase());
List<TypeElement> types = protoFileElement.getTypes();
List<MessageElement> messageTypes = getMessageTypes(types);
if (!messageTypes.isEmpty()) {
List<EnumElement> enumTypes = getEnumElements(types);
if (!enumTypes.isEmpty()) {
enumTypes.forEach(enumElement -> {
EnumDefinition enumDefinition = getEnumDefinition(enumElement);
schemaBuilder.addEnumDefinition(enumDefinition);
});
}
List<MessageDefinition> messageDefinitions = getMessageDefinitions(messageTypes);
messageDefinitions.forEach(schemaBuilder::addMessageDefinition);
try {
return schemaBuilder.build();
} catch (Descriptors.DescriptorValidationException e) {
throw new RuntimeException("Failed to create dynamic schema due to: " + e.getMessage());
}
} else {
throw new RuntimeException("Failed to get Dynamic Schema! Message types is empty for schema:" + schemaName);
}
}
public static ProtoFileElement getProtoFileElement(String protoSchema) {
return new ProtoParser(LOCATION, protoSchema.toCharArray()).readProtoFile();
}
public static String dynamicMsgToJson(Descriptors.Descriptor descriptor, byte[] payload) throws InvalidProtocolBufferException {
DynamicMessage dynamicMessage = DynamicMessage.parseFrom(descriptor, payload);
return JsonFormat.printer().includingDefaultValueFields().print(dynamicMessage);
}
public static DynamicMessage jsonToDynamicMessage(DynamicMessage.Builder builder, String payload) throws InvalidProtocolBufferException {
JsonFormat.parser().ignoringUnknownFields().merge(payload, builder);
return builder.build();
}
private static List<MessageElement> getMessageTypes(List<TypeElement> types) {
return types.stream()
.filter(typeElement -> typeElement instanceof MessageElement)
.map(typeElement -> (MessageElement) typeElement)
.collect(Collectors.toList());
}
private static List<EnumElement> getEnumElements(List<TypeElement> types) {
return types.stream()
.filter(typeElement -> typeElement instanceof EnumElement)
.map(typeElement -> (EnumElement) typeElement)
.collect(Collectors.toList());
}
private static List<MessageDefinition> getMessageDefinitions(List<MessageElement> messageElementsList) {
if (!messageElementsList.isEmpty()) {
List<MessageDefinition> messageDefinitions = new ArrayList<>();
messageElementsList.forEach(messageElement -> {
MessageDefinition.Builder messageDefinitionBuilder = MessageDefinition.newBuilder(messageElement.getName());
List<TypeElement> nestedTypes = messageElement.getNestedTypes();
if (!nestedTypes.isEmpty()) {
List<EnumElement> nestedEnumTypes = getEnumElements(nestedTypes);
if (!nestedEnumTypes.isEmpty()) {
nestedEnumTypes.forEach(enumElement -> {
EnumDefinition nestedEnumDefinition = getEnumDefinition(enumElement);
messageDefinitionBuilder.addEnumDefinition(nestedEnumDefinition);
});
}
List<MessageElement> nestedMessageTypes = getMessageTypes(nestedTypes);
List<MessageDefinition> nestedMessageDefinitions = getMessageDefinitions(nestedMessageTypes);
nestedMessageDefinitions.forEach(messageDefinitionBuilder::addMessageDefinition);
}
List<FieldElement> messageElementFields = messageElement.getFields();
List<OneOfElement> oneOfs = messageElement.getOneOfs();
if (!oneOfs.isEmpty()) {
for (OneOfElement oneOfelement : oneOfs) {
MessageDefinition.OneofBuilder oneofBuilder = messageDefinitionBuilder.addOneof(oneOfelement.getName());
addMessageFieldsToTheOneOfDefinition(oneOfelement.getFields(), oneofBuilder);
}
}
if (!messageElementFields.isEmpty()) {
addMessageFieldsToTheMessageDefinition(messageElementFields, messageDefinitionBuilder);
}
messageDefinitions.add(messageDefinitionBuilder.build());
});
return messageDefinitions;
} else {
return Collections.emptyList();
}
}
private static EnumDefinition getEnumDefinition(EnumElement enumElement) {
List<EnumConstantElement> enumElementTypeConstants = enumElement.getConstants();
EnumDefinition.Builder enumDefinitionBuilder = EnumDefinition.newBuilder(enumElement.getName());
if (!enumElementTypeConstants.isEmpty()) {
enumElementTypeConstants.forEach(constantElement -> enumDefinitionBuilder.addValue(constantElement.getName(), constantElement.getTag()));
}
return enumDefinitionBuilder.build();
}
private static void addMessageFieldsToTheMessageDefinition(List<FieldElement> messageElementFields, MessageDefinition.Builder messageDefinitionBuilder) {
messageElementFields.forEach(fieldElement -> {
String labelStr = null;
if (fieldElement.getLabel() != null) {
labelStr = fieldElement.getLabel().name().toLowerCase();
}
messageDefinitionBuilder.addField(
labelStr,
fieldElement.getType(),
fieldElement.getName(),
fieldElement.getTag());
});
}
private static void addMessageFieldsToTheOneOfDefinition(List<FieldElement> oneOfsElementFields, MessageDefinition.OneofBuilder oneofBuilder) {
oneOfsElementFields.forEach(fieldElement -> oneofBuilder.addField(
fieldElement.getType(),
fieldElement.getName(),
fieldElement.getTag()));
oneofBuilder.msgDefBuilder();
}
// validation
public static void validateProtoSchema(String schema, String schemaName, String exceptionPrefix) throws IllegalArgumentException {
ProtoParser schemaParser = new ProtoParser(LOCATION, schema.toCharArray());
ProtoFileElement protoFileElement;
try {
protoFileElement = schemaParser.readProtoFile();
} catch (Exception e) {
throw new IllegalArgumentException(exceptionPrefix + " failed to parse " + schemaName + " due to: " + e.getMessage());
}
checkProtoFileSyntax(schemaName, protoFileElement);
checkProtoFileCommonSettings(schemaName, protoFileElement.getOptions().isEmpty(), " Schema options don't support!", exceptionPrefix);
checkProtoFileCommonSettings(schemaName, protoFileElement.getPublicImports().isEmpty(), " Schema public imports don't support!", exceptionPrefix);
checkProtoFileCommonSettings(schemaName, protoFileElement.getImports().isEmpty(), " Schema imports don't support!", exceptionPrefix);
checkProtoFileCommonSettings(schemaName, protoFileElement.getExtendDeclarations().isEmpty(), " Schema extend declarations don't support!", exceptionPrefix);
checkTypeElements(schemaName, protoFileElement, exceptionPrefix);
}
private static void checkProtoFileSyntax(String schemaName, ProtoFileElement protoFileElement) {
if (protoFileElement.getSyntax() == null || !protoFileElement.getSyntax().equals(Syntax.PROTO_3)) {
throw new IllegalArgumentException("[Transport Configuration] invalid schema syntax: " + protoFileElement.getSyntax() +
" for " + schemaName + " provided! Only " + Syntax.PROTO_3 + " allowed!");
}
}
private static void checkProtoFileCommonSettings(String schemaName, boolean isEmptySettings, String invalidSettingsMessage, String exceptionPrefix) {
if (!isEmptySettings) {
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + invalidSettingsMessage);
}
}
private static void checkTypeElements(String schemaName, ProtoFileElement protoFileElement, String exceptionPrefix) {
List<TypeElement> types = protoFileElement.getTypes();
if (!types.isEmpty()) {
if (types.stream().noneMatch(typeElement -> typeElement instanceof MessageElement)) {
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " At least one Message definition should exists!");
} else {
checkEnumElements(schemaName, getEnumElements(types), exceptionPrefix);
checkMessageElements(schemaName, getMessageTypes(types), exceptionPrefix);
}
} else {
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Type elements is empty!");
}
}
private static void checkFieldElements(String schemaName, List<FieldElement> fieldElements, String exceptionPrefix) {
if (!fieldElements.isEmpty()) {
boolean hasRequiredLabel = fieldElements.stream().anyMatch(fieldElement -> {
Field.Label label = fieldElement.getLabel();
return label != null && label.equals(Field.Label.REQUIRED);
});
if (hasRequiredLabel) {
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Required labels are not supported!");
}
boolean hasDefaultValue = fieldElements.stream().anyMatch(fieldElement -> fieldElement.getDefaultValue() != null);
if (hasDefaultValue) {
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Default values are not supported!");
}
}
}
private static void checkEnumElements(String schemaName, List<EnumElement> enumTypes, String exceptionPrefix) {
if (enumTypes.stream().anyMatch(enumElement -> !enumElement.getNestedTypes().isEmpty())) {
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Nested types in Enum definitions are not supported!");
}
if (enumTypes.stream().anyMatch(enumElement -> !enumElement.getOptions().isEmpty())) {
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName, exceptionPrefix) + " Enum definitions options are not supported!");
}
}
private static void checkMessageElements(String schemaName, List<MessageElement> messageElementsList, String exceptionPrefix) {
if (!messageElementsList.isEmpty()) {
messageElementsList.forEach(messageElement -> {
checkProtoFileCommonSettings(schemaName, messageElement.getGroups().isEmpty(),
" Message definition groups don't support!", exceptionPrefix);
checkProtoFileCommonSettings(schemaName, messageElement.getOptions().isEmpty(),
" Message definition options don't support!", exceptionPrefix);
checkProtoFileCommonSettings(schemaName, messageElement.getExtensions().isEmpty(),
" Message definition extensions don't support!", exceptionPrefix);
checkProtoFileCommonSettings(schemaName, messageElement.getReserveds().isEmpty(),
" Message definition reserved elements don't support!", exceptionPrefix);
checkFieldElements(schemaName, messageElement.getFields(), exceptionPrefix);
List<OneOfElement> oneOfs = messageElement.getOneOfs();
if (!oneOfs.isEmpty()) {
oneOfs.forEach(oneOfElement -> {
checkProtoFileCommonSettings(schemaName, oneOfElement.getGroups().isEmpty(),
" OneOf definition groups don't support!", exceptionPrefix);
checkFieldElements(schemaName, oneOfElement.getFields(), exceptionPrefix);
});
}
List<TypeElement> nestedTypes = messageElement.getNestedTypes();
if (!nestedTypes.isEmpty()) {
List<EnumElement> nestedEnumTypes = getEnumElements(nestedTypes);
if (!nestedEnumTypes.isEmpty()) {
checkEnumElements(schemaName, nestedEnumTypes, exceptionPrefix);
}
List<MessageElement> nestedMessageTypes = getMessageTypes(nestedTypes);
checkMessageElements(schemaName, nestedMessageTypes, exceptionPrefix);
}
});
}
}
public static String invalidSchemaProvidedMessage(String schemaName, String exceptionPrefix) {
return exceptionPrefix + " invalid " + schemaName + " provided!";
}
}

4
common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java

@ -156,6 +156,10 @@ public class StringUtils {
return false;
}
public static boolean contains(final CharSequence seq, final CharSequence searchSeq) {
return org.apache.commons.lang3.StringUtils.contains(seq, searchSeq);
}
public static String randomNumeric(int length) {
return RandomStringUtils.randomNumeric(length);
}

169
common/data/src/main/java/org/thingsboard/server/common/data/device/profile/ProtoTransportPayloadConfiguration.java

@ -15,29 +15,15 @@
*/
package org.thingsboard.server.common.data.device.profile;
import com.github.os72.protobuf.dynamic.DynamicSchema;
import com.github.os72.protobuf.dynamic.EnumDefinition;
import com.github.os72.protobuf.dynamic.MessageDefinition;
import com.google.protobuf.Descriptors;
import com.google.protobuf.DynamicMessage;
import com.squareup.wire.schema.Location;
import com.squareup.wire.schema.internal.parser.EnumConstantElement;
import com.squareup.wire.schema.internal.parser.EnumElement;
import com.squareup.wire.schema.internal.parser.FieldElement;
import com.squareup.wire.schema.internal.parser.MessageElement;
import com.squareup.wire.schema.internal.parser.OneOfElement;
import com.squareup.wire.schema.internal.parser.ProtoFileElement;
import com.squareup.wire.schema.internal.parser.ProtoParser;
import com.squareup.wire.schema.internal.parser.TypeElement;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.DynamicProtoUtils;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.TransportPayloadType;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
@Slf4j
@Data
public class ProtoTransportPayloadConfiguration implements TransportPayloadTypeConfiguration {
@ -63,23 +49,23 @@ public class ProtoTransportPayloadConfiguration implements TransportPayloadTypeC
}
public Descriptors.Descriptor getTelemetryDynamicMessageDescriptor(String deviceTelemetryProtoSchema) {
return getDescriptor(deviceTelemetryProtoSchema, TELEMETRY_PROTO_SCHEMA);
return DynamicProtoUtils.getDescriptor(deviceTelemetryProtoSchema, TELEMETRY_PROTO_SCHEMA);
}
public Descriptors.Descriptor getAttributesDynamicMessageDescriptor(String deviceAttributesProtoSchema) {
return getDescriptor(deviceAttributesProtoSchema, ATTRIBUTES_PROTO_SCHEMA);
return DynamicProtoUtils.getDescriptor(deviceAttributesProtoSchema, ATTRIBUTES_PROTO_SCHEMA);
}
public Descriptors.Descriptor getRpcResponseDynamicMessageDescriptor(String deviceRpcResponseProtoSchema) {
return getDescriptor(deviceRpcResponseProtoSchema, RPC_RESPONSE_PROTO_SCHEMA);
return DynamicProtoUtils.getDescriptor(deviceRpcResponseProtoSchema, RPC_RESPONSE_PROTO_SCHEMA);
}
public DynamicMessage.Builder getRpcRequestDynamicMessageBuilder(String deviceRpcRequestProtoSchema) {
return getDynamicMessageBuilder(deviceRpcRequestProtoSchema, RPC_REQUEST_PROTO_SCHEMA);
return DynamicProtoUtils.getDynamicMessageBuilder(deviceRpcRequestProtoSchema, RPC_REQUEST_PROTO_SCHEMA);
}
public String getDeviceRpcResponseProtoSchema() {
if (!isEmptyStr(deviceRpcResponseProtoSchema)) {
if (StringUtils.isNotEmpty(deviceRpcResponseProtoSchema)) {
return deviceRpcResponseProtoSchema;
} else {
return "syntax =\"proto3\";\n" +
@ -92,7 +78,7 @@ public class ProtoTransportPayloadConfiguration implements TransportPayloadTypeC
}
public String getDeviceRpcRequestProtoSchema() {
if (!isEmptyStr(deviceRpcRequestProtoSchema)) {
if (StringUtils.isNotEmpty(deviceRpcRequestProtoSchema)) {
return deviceRpcRequestProtoSchema;
} else {
return "syntax =\"proto3\";\n" +
@ -106,143 +92,4 @@ public class ProtoTransportPayloadConfiguration implements TransportPayloadTypeC
}
}
private Descriptors.Descriptor getDescriptor(String protoSchema, String schemaName) {
try {
DynamicMessage.Builder builder = getDynamicMessageBuilder(protoSchema, schemaName);
return builder.getDescriptorForType();
} catch (Exception e) {
log.warn("Failed to get Message Descriptor due to {}", e.getMessage());
return null;
}
}
public DynamicMessage.Builder getDynamicMessageBuilder(String protoSchema, String schemaName) {
ProtoFileElement protoFileElement = getTransportProtoSchema(protoSchema);
DynamicSchema dynamicSchema = getDynamicSchema(protoFileElement, schemaName);
String lastMsgName = getMessageTypes(protoFileElement.getTypes()).stream()
.map(MessageElement::getName).reduce((previous, last) -> last).get();
return dynamicSchema.newMessageBuilder(lastMsgName);
}
public DynamicSchema getDynamicSchema(ProtoFileElement protoFileElement, String schemaName) {
DynamicSchema.Builder schemaBuilder = DynamicSchema.newBuilder();
schemaBuilder.setName(schemaName);
schemaBuilder.setSyntax(PROTO_3_SYNTAX);
schemaBuilder.setPackage(!isEmptyStr(protoFileElement.getPackageName()) ?
protoFileElement.getPackageName() : schemaName.toLowerCase());
List<TypeElement> types = protoFileElement.getTypes();
List<MessageElement> messageTypes = getMessageTypes(types);
if (!messageTypes.isEmpty()) {
List<EnumElement> enumTypes = getEnumElements(types);
if (!enumTypes.isEmpty()) {
enumTypes.forEach(enumElement -> {
EnumDefinition enumDefinition = getEnumDefinition(enumElement);
schemaBuilder.addEnumDefinition(enumDefinition);
});
}
List<MessageDefinition> messageDefinitions = getMessageDefinitions(messageTypes);
messageDefinitions.forEach(schemaBuilder::addMessageDefinition);
try {
return schemaBuilder.build();
} catch (Descriptors.DescriptorValidationException e) {
throw new RuntimeException("Failed to create dynamic schema due to: " + e.getMessage());
}
} else {
throw new RuntimeException("Failed to get Dynamic Schema! Message types is empty for schema:" + schemaName);
}
}
public ProtoFileElement getTransportProtoSchema(String protoSchema) {
return new ProtoParser(LOCATION, protoSchema.toCharArray()).readProtoFile();
}
private List<MessageElement> getMessageTypes(List<TypeElement> types) {
return types.stream()
.filter(typeElement -> typeElement instanceof MessageElement)
.map(typeElement -> (MessageElement) typeElement)
.collect(Collectors.toList());
}
private List<EnumElement> getEnumElements(List<TypeElement> types) {
return types.stream()
.filter(typeElement -> typeElement instanceof EnumElement)
.map(typeElement -> (EnumElement) typeElement)
.collect(Collectors.toList());
}
private List<MessageDefinition> getMessageDefinitions(List<MessageElement> messageElementsList) {
if (!messageElementsList.isEmpty()) {
List<MessageDefinition> messageDefinitions = new ArrayList<>();
messageElementsList.forEach(messageElement -> {
MessageDefinition.Builder messageDefinitionBuilder = MessageDefinition.newBuilder(messageElement.getName());
List<TypeElement> nestedTypes = messageElement.getNestedTypes();
if (!nestedTypes.isEmpty()) {
List<EnumElement> nestedEnumTypes = getEnumElements(nestedTypes);
if (!nestedEnumTypes.isEmpty()) {
nestedEnumTypes.forEach(enumElement -> {
EnumDefinition nestedEnumDefinition = getEnumDefinition(enumElement);
messageDefinitionBuilder.addEnumDefinition(nestedEnumDefinition);
});
}
List<MessageElement> nestedMessageTypes = getMessageTypes(nestedTypes);
List<MessageDefinition> nestedMessageDefinitions = getMessageDefinitions(nestedMessageTypes);
nestedMessageDefinitions.forEach(messageDefinitionBuilder::addMessageDefinition);
}
List<FieldElement> messageElementFields = messageElement.getFields();
List<OneOfElement> oneOfs = messageElement.getOneOfs();
if (!oneOfs.isEmpty()) {
for (OneOfElement oneOfelement : oneOfs) {
MessageDefinition.OneofBuilder oneofBuilder = messageDefinitionBuilder.addOneof(oneOfelement.getName());
addMessageFieldsToTheOneOfDefinition(oneOfelement.getFields(), oneofBuilder);
}
}
if (!messageElementFields.isEmpty()) {
addMessageFieldsToTheMessageDefinition(messageElementFields, messageDefinitionBuilder);
}
messageDefinitions.add(messageDefinitionBuilder.build());
});
return messageDefinitions;
} else {
return Collections.emptyList();
}
}
private EnumDefinition getEnumDefinition(EnumElement enumElement) {
List<EnumConstantElement> enumElementTypeConstants = enumElement.getConstants();
EnumDefinition.Builder enumDefinitionBuilder = EnumDefinition.newBuilder(enumElement.getName());
if (!enumElementTypeConstants.isEmpty()) {
enumElementTypeConstants.forEach(constantElement -> enumDefinitionBuilder.addValue(constantElement.getName(), constantElement.getTag()));
}
return enumDefinitionBuilder.build();
}
private void addMessageFieldsToTheMessageDefinition(List<FieldElement> messageElementFields, MessageDefinition.Builder messageDefinitionBuilder) {
messageElementFields.forEach(fieldElement -> {
String labelStr = null;
if (fieldElement.getLabel() != null) {
labelStr = fieldElement.getLabel().name().toLowerCase();
}
messageDefinitionBuilder.addField(
labelStr,
fieldElement.getType(),
fieldElement.getName(),
fieldElement.getTag());
});
}
private void addMessageFieldsToTheOneOfDefinition(List<FieldElement> oneOfsElementFields, MessageDefinition.OneofBuilder oneofBuilder) {
oneOfsElementFields.forEach(fieldElement -> oneofBuilder.addField(
fieldElement.getType(),
fieldElement.getName(),
fieldElement.getTag()));
oneofBuilder.msgDefBuilder();
}
private boolean isEmptyStr(String str) {
return str == null || "".equals(str);
}
}

16
common/data/src/main/java/org/thingsboard/server/common/data/security/event/UserAuthDataChangedEvent.java

@ -15,17 +15,9 @@
*/
package org.thingsboard.server.common.data.security.event;
import lombok.Data;
import org.thingsboard.server.common.data.id.UserId;
@Data
public class UserAuthDataChangedEvent {
private final UserId userId;
private final long ts;
public UserAuthDataChangedEvent(UserId userId) {
this.userId = userId;
this.ts = System.currentTimeMillis();
}
import java.io.Serializable;
public abstract class UserAuthDataChangedEvent implements Serializable {
public abstract String getId();
public abstract long getTs();
}

40
common/data/src/main/java/org/thingsboard/server/common/data/security/event/UserCredentialsInvalidationEvent.java

@ -0,0 +1,40 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.security.event;
import lombok.EqualsAndHashCode;
import org.thingsboard.server.common.data.id.UserId;
@EqualsAndHashCode(callSuper = true)
public class UserCredentialsInvalidationEvent extends UserAuthDataChangedEvent {
private final UserId userId;
private final long ts;
public UserCredentialsInvalidationEvent(UserId userId) {
this.userId = userId;
this.ts = System.currentTimeMillis();
}
@Override
public String getId() {
return userId.toString();
}
@Override
public long getTs() {
return ts;
}
}

39
common/data/src/main/java/org/thingsboard/server/common/data/security/event/UserSessionInvalidationEvent.java

@ -0,0 +1,39 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.security.event;
import lombok.EqualsAndHashCode;
@EqualsAndHashCode(callSuper = true)
public class UserSessionInvalidationEvent extends UserAuthDataChangedEvent {
private final String sessionId;
private final long ts;
public UserSessionInvalidationEvent(String sessionId) {
this.sessionId = sessionId;
this.ts = System.currentTimeMillis();
}
@Override
public String getId() {
return sessionId;
}
@Override
public long getTs() {
return ts;
}
}

26
common/data/src/main/java/org/thingsboard/server/common/data/util/TbPair.java

@ -0,0 +1,26 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.util;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class TbPair<S, T> {
private S first;
private T second;
}

169
common/data/src/test/java/org/thingsboard/server/common/data/DynamicProtoUtilsTest.java

@ -0,0 +1,169 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data;
import com.github.os72.protobuf.dynamic.DynamicSchema;
import com.google.protobuf.Descriptors;
import com.google.protobuf.DynamicMessage;
import com.squareup.wire.schema.internal.parser.ProtoFileElement;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.junit.MockitoJUnitRunner;
import java.util.List;
import java.util.Set;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@RunWith(MockitoJUnitRunner.class)
public class DynamicProtoUtilsTest {
@Test
public void testProtoSchemaWithMessageNestedTypes() throws Exception {
String schema = "syntax = \"proto3\";\n" +
"\n" +
"package testnested;\n" +
"\n" +
"message Outer {\n" +
" message MiddleAA {\n" +
" message Inner {\n" +
" optional int64 ival = 1;\n" +
" optional bool booly = 2;\n" +
" }\n" +
" Inner inner = 1;\n" +
" }\n" +
" message MiddleBB {\n" +
" message Inner {\n" +
" optional int32 ival = 1;\n" +
" optional bool booly = 2;\n" +
" }\n" +
" Inner inner = 1;\n" +
" }\n" +
" MiddleAA middleAA = 1;\n" +
" MiddleBB middleBB = 2;\n" +
"}";
ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(schema);
DynamicSchema dynamicSchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, "test schema with nested types");
assertNotNull(dynamicSchema);
Set<String> messageTypes = dynamicSchema.getMessageTypes();
assertEquals(5, messageTypes.size());
assertTrue(messageTypes.contains("testnested.Outer"));
assertTrue(messageTypes.contains("testnested.Outer.MiddleAA"));
assertTrue(messageTypes.contains("testnested.Outer.MiddleAA.Inner"));
assertTrue(messageTypes.contains("testnested.Outer.MiddleBB"));
assertTrue(messageTypes.contains("testnested.Outer.MiddleBB.Inner"));
DynamicMessage.Builder middleAAInnerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA.Inner");
Descriptors.Descriptor middleAAInnerMsgDescriptor = middleAAInnerMsgBuilder.getDescriptorForType();
DynamicMessage middleAAInnerMsg = middleAAInnerMsgBuilder
.setField(middleAAInnerMsgDescriptor.findFieldByName("ival"), 1L)
.setField(middleAAInnerMsgDescriptor.findFieldByName("booly"), true)
.build();
DynamicMessage.Builder middleAAMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA");
Descriptors.Descriptor middleAAMsgDescriptor = middleAAMsgBuilder.getDescriptorForType();
DynamicMessage middleAAMsg = middleAAMsgBuilder
.setField(middleAAMsgDescriptor.findFieldByName("inner"), middleAAInnerMsg)
.build();
DynamicMessage.Builder middleBBInnerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleAA.Inner");
Descriptors.Descriptor middleBBInnerMsgDescriptor = middleBBInnerMsgBuilder.getDescriptorForType();
DynamicMessage middleBBInnerMsg = middleBBInnerMsgBuilder
.setField(middleBBInnerMsgDescriptor.findFieldByName("ival"), 0L)
.setField(middleBBInnerMsgDescriptor.findFieldByName("booly"), false)
.build();
DynamicMessage.Builder middleBBMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer.MiddleBB");
Descriptors.Descriptor middleBBMsgDescriptor = middleBBMsgBuilder.getDescriptorForType();
DynamicMessage middleBBMsg = middleBBMsgBuilder
.setField(middleBBMsgDescriptor.findFieldByName("inner"), middleBBInnerMsg)
.build();
DynamicMessage.Builder outerMsgBuilder = dynamicSchema.newMessageBuilder("testnested.Outer");
Descriptors.Descriptor outerMsgBuilderDescriptor = outerMsgBuilder.getDescriptorForType();
DynamicMessage outerMsg = outerMsgBuilder
.setField(outerMsgBuilderDescriptor.findFieldByName("middleAA"), middleAAMsg)
.setField(outerMsgBuilderDescriptor.findFieldByName("middleBB"), middleBBMsg)
.build();
assertEquals("{\n" +
" \"middleAA\": {\n" +
" \"inner\": {\n" +
" \"ival\": \"1\",\n" +
" \"booly\": true\n" +
" }\n" +
" },\n" +
" \"middleBB\": {\n" +
" \"inner\": {\n" +
" \"ival\": 0,\n" +
" \"booly\": false\n" +
" }\n" +
" }\n" +
"}", DynamicProtoUtils.dynamicMsgToJson(outerMsgBuilderDescriptor, outerMsg.toByteArray()));
}
@Test
public void testProtoSchemaWithMessageOneOfs() throws Exception {
String schema = "syntax = \"proto3\";\n" +
"\n" +
"package testoneofs;\n" +
"\n" +
"message SubMessage {\n" +
" repeated string name = 1;\n" +
"}\n" +
"\n" +
"message SampleMessage {\n" +
" optional int32 id = 1;\n" +
" oneof testOneOf {\n" +
" string name = 4;\n" +
" SubMessage subMessage = 9;\n" +
" }\n" +
"}";
ProtoFileElement protoFileElement = DynamicProtoUtils.getProtoFileElement(schema);
DynamicSchema dynamicSchema = DynamicProtoUtils.getDynamicSchema(protoFileElement, "test schema with message oneOfs");
assertNotNull(dynamicSchema);
Set<String> messageTypes = dynamicSchema.getMessageTypes();
assertEquals(2, messageTypes.size());
assertTrue(messageTypes.contains("testoneofs.SubMessage"));
assertTrue(messageTypes.contains("testoneofs.SampleMessage"));
DynamicMessage.Builder sampleMsgBuilder = dynamicSchema.newMessageBuilder("testoneofs.SampleMessage");
Descriptors.Descriptor sampleMsgDescriptor = sampleMsgBuilder.getDescriptorForType();
assertNotNull(sampleMsgDescriptor);
List<Descriptors.FieldDescriptor> fields = sampleMsgDescriptor.getFields();
assertEquals(3, fields.size());
DynamicMessage sampleMsg = sampleMsgBuilder
.setField(sampleMsgDescriptor.findFieldByName("name"), "Bob")
.build();
assertEquals("{\n" + " \"name\": \"Bob\"\n" + "}", DynamicProtoUtils.dynamicMsgToJson(sampleMsgDescriptor, sampleMsg.toByteArray()));
DynamicMessage.Builder subMsgBuilder = dynamicSchema.newMessageBuilder("testoneofs.SubMessage");
Descriptors.Descriptor subMsgDescriptor = subMsgBuilder.getDescriptorForType();
DynamicMessage subMsg = subMsgBuilder
.addRepeatedField(subMsgDescriptor.findFieldByName("name"), "Alice")
.addRepeatedField(subMsgDescriptor.findFieldByName("name"), "John")
.build();
DynamicMessage sampleMsgWithOneOfSubMessage = sampleMsgBuilder.setField(sampleMsgDescriptor.findFieldByName("subMessage"), subMsg).build();
assertEquals("{\n" + " \"subMessage\": {\n" + " \"name\": [\"Alice\", \"John\"]\n" + " }\n" + "}",
DynamicProtoUtils.dynamicMsgToJson(sampleMsgDescriptor, sampleMsgWithOneOfSubMessage.toByteArray()));
}
}

3
common/message/src/main/java/org/thingsboard/server/common/msg/TbMsgProcessingStackItem.java

@ -20,10 +20,11 @@ import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.msg.gen.MsgProtos;
import java.io.Serializable;
import java.util.UUID;
@Data
public class TbMsgProcessingStackItem {
public class TbMsgProcessingStackItem implements Serializable {
private final RuleChainId ruleChainId;
private final RuleNodeId ruleNodeId;

37
common/message/src/test/java/org/thingsboard/server/common/msg/TbMsgProcessingStackItemTest.java

@ -0,0 +1,37 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.msg;
import org.junit.jupiter.api.Test;
import org.thingsboard.server.common.data.FSTUtils;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId;
import java.util.UUID;
import static org.assertj.core.api.Assertions.assertThat;
class TbMsgProcessingStackItemTest {
@Test
void testSerialization() {
TbMsgProcessingStackItem item = new TbMsgProcessingStackItem(new RuleChainId(UUID.randomUUID()), new RuleNodeId(UUID.randomUUID()));
byte[] bytes = FSTUtils.encode(item);
TbMsgProcessingStackItem itemDecoded = FSTUtils.decode(bytes);
assertThat(item).isEqualTo(itemDecoded);
}
}

4
common/queue/pom.xml

@ -116,10 +116,6 @@
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>

18
common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java

@ -15,7 +15,6 @@
*/
package org.thingsboard.script.api.mvel;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
@ -24,7 +23,6 @@ import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.mvel2.ExecutionContext;
import org.mvel2.MVEL;
import org.mvel2.ParserContext;
import org.mvel2.SandboxedParserConfiguration;
import org.mvel2.SandboxedParserContext;
import org.mvel2.ScriptMemoryOverflowException;
@ -44,6 +42,7 @@ import org.thingsboard.server.common.stats.TbApiUsageStateClient;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.io.Serializable;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
@ -111,8 +110,16 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem
OptimizerFactory.setDefaultOptimizer(OptimizerFactory.SAFE_REFLECTIVE);
parserConfig = new SandboxedParserConfiguration();
parserConfig.addImport("JSON", TbJson.class);
parserConfig.registerDataType("Date", TbDate.class, date -> 8L);
TbUtils.register(parserConfig);
executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, "mvel-executor"));
try {
// Special command to warm up MVEL engine
Serializable script = MVEL.compileExpression("var warmUp = {}; warmUp", new SandboxedParserContext(parserConfig));
MVEL.executeTbExpression(script, new ExecutionContext(parserConfig), Collections.emptyMap());
} catch (Exception e) {
// do nothing
}
}
@PreDestroy
@ -139,11 +146,6 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem
@Override
protected ListenableFuture<UUID> doEvalScript(TenantId tenantId, ScriptType scriptType, String scriptBody, UUID scriptId, String[] argNames) {
if (NEW_KEYWORD_PATTERN.matcher(scriptBody).matches()) {
//TODO: output line number and char pos.
return Futures.immediateFailedFuture(new TbScriptException(scriptId, TbScriptException.ErrorCode.COMPILATION, scriptBody,
new IllegalArgumentException("Keyword 'new' is forbidden!")));
}
return executor.submit(() -> {
try {
Serializable compiledScript = MVEL.compileExpression(scriptBody, new SandboxedParserContext(parserConfig));
@ -158,7 +160,7 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem
@Override
protected MvelScriptExecutionTask doInvokeFunction(UUID scriptId, Object[] args) {
ExecutionContext executionContext = new ExecutionContext(maxMemoryLimitMb * 1024 * 1024);
ExecutionContext executionContext = new ExecutionContext(this.parserConfig, maxMemoryLimitMb * 1024 * 1024);
return new MvelScriptExecutionTask(executionContext, executor.submit(() -> {
MvelScript script = scriptMap.get(scriptId);
if (script == null) {

95
common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/TbDate.java

@ -0,0 +1,95 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.script.api.mvel;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.Locale;
import java.util.TimeZone;
public class TbDate extends Date {
public TbDate() {
super();
}
public TbDate(String s) {
super(parse(s));
}
public TbDate(long date) {
super(date);
}
public TbDate(int year, int month, int date) {
this(year, month, date, 0, 0, 0);
}
public TbDate(int year, int month, int date, int hrs, int min) {
this(year, month, date, hrs, min, 0);
}
public TbDate(int year, int month, int date,
int hrs, int min, int second) {
super(new GregorianCalendar(year, month, date, hrs, min, second).getTimeInMillis());
}
public String toDateString() {
DateFormat formatter = DateFormat.getDateInstance();
return formatter.format(this);
}
public String toTimeString() {
DateFormat formatter = DateFormat.getTimeInstance(DateFormat.LONG);
return formatter.format(this);
}
public String toISOString() {
DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.sssZ");
return formatter.format(this);
}
public String toLocaleString(String locale) {
DateFormat formatter = DateFormat.getDateTimeInstance(DateFormat.MEDIUM, DateFormat.MEDIUM, Locale.forLanguageTag(locale));
return formatter.format(this);
}
public String toLocaleString(String locale, String tz) {
DateFormat formatter = DateFormat.getDateTimeInstance(DateFormat.MEDIUM, DateFormat.MEDIUM, Locale.forLanguageTag(locale));
formatter.setTimeZone(TimeZone.getTimeZone(tz));
return formatter.format(this);
}
public static long now() {
return System.currentTimeMillis();
}
public static long parse(String value) {
try {
return Date.parse(value);
} catch (IllegalArgumentException e) {
return -1;
}
}
public static long UTC(int year, int month, int date,
int hrs, int min, int sec) {
return Date.UTC(year - 1900, month, date, hrs, min, sec);
}
}

80
common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/TbUtils.java

@ -39,6 +39,18 @@ public class TbUtils {
ExecutionContext.class, String.class)));
parserConfig.addImport("stringToBytes", new MethodStub(TbUtils.class.getMethod("stringToBytes",
ExecutionContext.class, String.class, String.class)));
parserConfig.addImport("parseInt", new MethodStub(TbUtils.class.getMethod("parseInt",
String.class)));
parserConfig.addImport("parseInt", new MethodStub(TbUtils.class.getMethod("parseInt",
String.class, int.class)));
parserConfig.addImport("parseFloat", new MethodStub(TbUtils.class.getMethod("parseFloat",
String.class)));
parserConfig.addImport("parseDouble", new MethodStub(TbUtils.class.getMethod("parseDouble",
String.class)));
}
public static void main(String[] args) {
System.out.println(Integer.class == int.class);
}
public static String btoa(String input) {
@ -84,4 +96,72 @@ public class TbUtils {
}
return list;
}
public static Integer parseInt(String value) {
if (value != null) {
try {
int radix = 10;
if (isHexadecimal(value)) {
radix = 16;
}
return Integer.parseInt(prepareNumberString(value), radix);
} catch (NumberFormatException e) {
Float f = parseFloat(value);
if (f != null) {
return f.intValue();
}
}
}
return null;
}
public static Integer parseInt(String value, int radix) {
if (value != null) {
try {
return Integer.parseInt(prepareNumberString(value), radix);
} catch (NumberFormatException e) {
Float f = parseFloat(value);
if (f != null) {
return f.intValue();
}
}
}
return null;
}
public static Float parseFloat(String value) {
if (value != null) {
try {
return Float.parseFloat(prepareNumberString(value));
} catch (NumberFormatException e) {
}
}
return null;
}
public static Double parseDouble(String value) {
if (value != null) {
try {
return Double.parseDouble(prepareNumberString(value));
} catch (NumberFormatException e) {
}
}
return null;
}
private static boolean isHexadecimal(String value) {
return value != null && (value.contains("0x") || value.contains("0X"));
}
private static String prepareNumberString(String value) {
if (value != null) {
value = value.trim();
if (isHexadecimal(value)) {
value = value.replace("0x", "");
value = value.replace("0X", "");
}
value = value.replace(",", ".");
}
return value;
}
}

17
common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java

@ -883,7 +883,22 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
log.error("[{}] Unexpected Exception", sessionId, cause);
if (cause instanceof IOException) {
if (log.isDebugEnabled()) {
log.debug("[{}][{}][{}] IOException: {}", sessionId,
Optional.ofNullable(this.deviceSessionCtx.getDeviceInfo()).map(TransportDeviceInfo::getDeviceId).orElse(null),
Optional.ofNullable(this.deviceSessionCtx.getDeviceInfo()).map(TransportDeviceInfo::getDeviceName).orElse(""),
cause);
} else if (log.isInfoEnabled()) {
log.info("[{}][{}][{}] IOException: {}", sessionId,
Optional.ofNullable(this.deviceSessionCtx.getDeviceInfo()).map(TransportDeviceInfo::getDeviceId).orElse(null),
Optional.ofNullable(this.deviceSessionCtx.getDeviceInfo()).map(TransportDeviceInfo::getDeviceName).orElse(""),
cause.getMessage());
}
} else {
log.error("[{}] Unexpected Exception", sessionId, cause);
}
ctx.close();
if (cause instanceof OutOfMemoryError) {
log.error("Received critical error. Going to shutdown the service.");

7
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java

@ -27,6 +27,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.DynamicProtoUtils;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.gen.transport.TransportApiProtos;
import org.thingsboard.server.gen.transport.TransportProtos;
@ -184,8 +185,7 @@ public class ProtoConverter {
try {
JsonElement paramsElement = JSON_PARSER.parse(params);
rpcRequestJson.add("params", paramsElement);
JsonFormat.parser().ignoringUnknownFields().merge(GSON.toJson(rpcRequestJson), rpcRequestDynamicMessageBuilder);
DynamicMessage dynamicRpcRequest = rpcRequestDynamicMessageBuilder.build();
DynamicMessage dynamicRpcRequest = DynamicProtoUtils.jsonToDynamicMessage(rpcRequestDynamicMessageBuilder, GSON.toJson(rpcRequestJson));
return dynamicRpcRequest.toByteArray();
} catch (Exception e) {
throw new AdaptorException("Failed to convert ToDeviceRpcRequestMsg to Dynamic Rpc request message due to: ", e);
@ -200,8 +200,7 @@ public class ProtoConverter {
}
public static String dynamicMsgToJson(byte[] bytes, Descriptors.Descriptor descriptor) throws InvalidProtocolBufferException {
DynamicMessage dynamicMessage = DynamicMessage.parseFrom(descriptor, bytes);
return JsonFormat.printer().includingDefaultValueFields().print(dynamicMessage);
return DynamicProtoUtils.dynamicMsgToJson(descriptor, bytes);
}
}

37
common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java

@ -15,7 +15,9 @@
*/
package org.thingsboard.common.util;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.json.JsonWriteFeature;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.MapperFeature;
@ -46,6 +48,10 @@ public class JacksonUtil {
.configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true)
.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true)
.build();
public static ObjectMapper ALLOW_UNQUOTED_FIELD_NAMES_MAPPER = JsonMapper.builder()
.configure(JsonWriteFeature.QUOTE_FIELD_NAMES.mappedFeature(), false)
.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true)
.build();
public static <T> T convertValue(Object fromValue, Class<T> toValueType) {
try {
@ -127,18 +133,26 @@ public class JacksonUtil {
}
public static JsonNode toJsonNode(String value) {
return toJsonNode(value, OBJECT_MAPPER);
}
public static JsonNode toJsonNode(String value, ObjectMapper mapper) {
if (value == null || value.isEmpty()) {
return null;
}
try {
return OBJECT_MAPPER.readTree(value);
return mapper.readTree(value);
} catch (IOException e) {
throw new IllegalArgumentException(e);
}
}
public static ObjectNode newObjectNode() {
return OBJECT_MAPPER.createObjectNode();
return newObjectNode(OBJECT_MAPPER);
}
public static ObjectNode newObjectNode(ObjectMapper mapper) {
return mapper.createObjectNode();
}
public static <T> T clone(T value) {
@ -216,18 +230,27 @@ public class JacksonUtil {
}
public static void addKvEntry(ObjectNode entityNode, KvEntry kvEntry) {
addKvEntry(entityNode, kvEntry, kvEntry.getKey());
}
public static void addKvEntry(ObjectNode entityNode, KvEntry kvEntry, String key) {
addKvEntry(entityNode, kvEntry, key, OBJECT_MAPPER);
}
public static void addKvEntry(ObjectNode entityNode, KvEntry kvEntry, String key, ObjectMapper mapper) {
if (kvEntry.getDataType() == DataType.BOOLEAN) {
kvEntry.getBooleanValue().ifPresent(value -> entityNode.put(kvEntry.getKey(), value));
kvEntry.getBooleanValue().ifPresent(value -> entityNode.put(key, value));
} else if (kvEntry.getDataType() == DataType.DOUBLE) {
kvEntry.getDoubleValue().ifPresent(value -> entityNode.put(kvEntry.getKey(), value));
kvEntry.getDoubleValue().ifPresent(value -> entityNode.put(key, value));
} else if (kvEntry.getDataType() == DataType.LONG) {
kvEntry.getLongValue().ifPresent(value -> entityNode.put(kvEntry.getKey(), value));
kvEntry.getLongValue().ifPresent(value -> entityNode.put(key, value));
} else if (kvEntry.getDataType() == DataType.JSON) {
if (kvEntry.getJsonValue().isPresent()) {
entityNode.set(kvEntry.getKey(), JacksonUtil.toJsonNode(kvEntry.getJsonValue().get()));
entityNode.set(key, toJsonNode(kvEntry.getJsonValue().get(), mapper));
}
} else {
entityNode.put(kvEntry.getKey(), kvEntry.getValueAsString());
entityNode.put(key, kvEntry.getValueAsString());
}
}
}

5
common/util/src/main/java/org/thingsboard/common/util/ThingsBoardExecutors.java

@ -15,6 +15,10 @@
*/
package org.thingsboard.common.util;
import com.google.common.util.concurrent.MoreExecutors;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ForkJoinPool;
@ -47,4 +51,5 @@ public class ThingsBoardExecutors {
public static ExecutorService newWorkStealingPool(int parallelism, Class clazz) {
return newWorkStealingPool(parallelism, clazz.getSimpleName());
}
}

35
common/util/src/test/java/org/thingsboard/common/util/JacksonUtilTest.java

@ -0,0 +1,35 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.common.util;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.Assert;
import org.junit.Test;
public class JacksonUtilTest {
@Test
public void allow_unquoted_field_mapper_test() {
String data = "{data: 123}";
JsonNode actualResult = JacksonUtil.toJsonNode(data, JacksonUtil.ALLOW_UNQUOTED_FIELD_NAMES_MAPPER); // should be: {"data": 123}
ObjectNode expectedResult = JacksonUtil.newObjectNode();
expectedResult.put("data", 123); // {"data": 123}
Assert.assertEquals(expectedResult, actualResult);
Assert.assertThrows(IllegalArgumentException.class, () -> JacksonUtil.toJsonNode(data)); // syntax exception due to missing quotes in the field name!
}
}

5
dao/pom.xml

@ -177,6 +177,11 @@
</exclusions>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.cassandra</groupId>
<artifactId>cassandra-thrift</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>

6
dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java

@ -42,7 +42,11 @@ import java.util.UUID;
*/
public interface AlarmDao extends Dao<Alarm> {
ListenableFuture<Alarm> findLatestByOriginatorAndType(TenantId tenantId, EntityId originator, String type);
Alarm findLatestByOriginatorAndType(TenantId tenantId, EntityId originator, String type);
ListenableFuture<Alarm> findLatestByOriginatorAndTypeAsync(TenantId tenantId, EntityId originator, String type);
Alarm findAlarmById(TenantId tenantId, UUID key);
ListenableFuture<Alarm> findAlarmByIdAsync(TenantId tenantId, UUID key);

62
dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java

@ -114,7 +114,7 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ
}
alarm.setCustomerId(entityService.fetchEntityCustomerId(alarm.getTenantId(), alarm.getOriginator()));
if (alarm.getId() == null) {
Alarm existing = alarmDao.findLatestByOriginatorAndType(alarm.getTenantId(), alarm.getOriginator(), alarm.getType()).get();
Alarm existing = alarmDao.findLatestByOriginatorAndType(alarm.getTenantId(), alarm.getOriginator(), alarm.getType());
if (existing == null || existing.getStatus().isCleared()) {
if (!alarmCreationEnabled) {
throw new ApiUsageLimitsExceededException("Alarms creation is disabled");
@ -124,7 +124,7 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ
return updateAlarm(existing, alarm);
}
} else {
return updateAlarm(alarm).get();
return updateAlarm(alarm);
}
} catch (ExecutionException | InterruptedException e) {
throw new RuntimeException(e);
@ -132,7 +132,7 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ
}
public ListenableFuture<Alarm> findLatestByOriginatorAndType(TenantId tenantId, EntityId originator, String type) {
return alarmDao.findLatestByOriginatorAndType(tenantId, originator, type);
return alarmDao.findLatestByOriginatorAndTypeAsync(tenantId, originator, type);
}
@Override
@ -145,19 +145,15 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ
@Override
@Transactional
public AlarmOperationResult deleteAlarm(TenantId tenantId, AlarmId alarmId) {
try {
log.debug("Deleting Alarm Id: {}", alarmId);
Alarm alarm = alarmDao.findAlarmByIdAsync(tenantId, alarmId.getId()).get();
if (alarm == null) {
return new AlarmOperationResult(alarm, false);
}
AlarmOperationResult result = new AlarmOperationResult(alarm, true, new ArrayList<>(getPropagationEntityIds(alarm)));
deleteEntityRelations(tenantId, alarm.getId());
alarmDao.removeById(tenantId, alarm.getUuidId());
return result;
} catch (ExecutionException | InterruptedException e) {
throw new RuntimeException(e);
log.debug("Deleting Alarm Id: {}", alarmId);
Alarm alarm = alarmDao.findAlarmById(tenantId, alarmId.getId());
if (alarm == null) {
return new AlarmOperationResult(alarm, false);
}
AlarmOperationResult result = new AlarmOperationResult(alarm, true, new ArrayList<>(getPropagationEntityIds(alarm)));
deleteEntityRelations(tenantId, alarm.getId());
alarmDao.removeById(tenantId, alarm.getUuidId());
return result;
}
private AlarmOperationResult createAlarm(Alarm alarm) throws InterruptedException, ExecutionException {
@ -197,19 +193,10 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ
return relations.map(EntityRelation::getFrom).collect(Collectors.toCollection(LinkedHashSet::new));
}
private ListenableFuture<AlarmOperationResult> updateAlarm(Alarm update) {
private AlarmOperationResult updateAlarm(Alarm update) {
alarmDataValidator.validate(update, Alarm::getTenantId);
return getAndUpdate(update.getTenantId(), update.getId(), new Function<Alarm, AlarmOperationResult>() {
@Nullable
@Override
public AlarmOperationResult apply(@Nullable Alarm alarm) {
if (alarm == null) {
return null;
} else {
return updateAlarm(alarm, update);
}
}
});
return getAndUpdate(update.getTenantId(), update.getId(),
(alarm) -> alarm == null ? null : updateAlarm(alarm, update));
}
private AlarmOperationResult updateAlarm(Alarm oldAlarm, Alarm newAlarm) {
@ -233,7 +220,7 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ
@Override
public ListenableFuture<AlarmOperationResult> ackAlarm(TenantId tenantId, AlarmId alarmId, long ackTime) {
return getAndUpdate(tenantId, alarmId, new Function<Alarm, AlarmOperationResult>() {
return getAndUpdateAsync(tenantId, alarmId, new Function<Alarm, AlarmOperationResult>() {
@Nullable
@Override
public AlarmOperationResult apply(@Nullable Alarm alarm) {
@ -253,7 +240,7 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ
@Override
public ListenableFuture<AlarmOperationResult> clearAlarm(TenantId tenantId, AlarmId alarmId, JsonNode details, long clearTime) {
return getAndUpdate(tenantId, alarmId, new Function<Alarm, AlarmOperationResult>() {
return getAndUpdateAsync(tenantId, alarmId, new Function<Alarm, AlarmOperationResult>() {
@Nullable
@Override
public AlarmOperationResult apply(@Nullable Alarm alarm) {
@ -275,9 +262,16 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ
}
@Override
public ListenableFuture<Alarm> findAlarmByIdAsync(TenantId tenantId, AlarmId alarmId) {
public Alarm findAlarmById(TenantId tenantId, AlarmId alarmId) {
log.trace("Executing findAlarmById [{}]", alarmId);
validateId(alarmId, "Incorrect alarmId " + alarmId);
return alarmDao.findAlarmById(tenantId, alarmId.getId());
}
@Override
public ListenableFuture<Alarm> findAlarmByIdAsync(TenantId tenantId, AlarmId alarmId) {
log.trace("Executing findAlarmByIdAsync [{}]", alarmId);
validateId(alarmId, "Incorrect alarmId " + alarmId);
return alarmDao.findAlarmByIdAsync(tenantId, alarmId.getId());
}
@ -404,9 +398,15 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ
}
}
private <T> ListenableFuture<T> getAndUpdate(TenantId tenantId, AlarmId alarmId, Function<Alarm, T> function) {
private <T> ListenableFuture<T> getAndUpdateAsync(TenantId tenantId, AlarmId alarmId, Function<Alarm, T> function) {
validateId(alarmId, "Alarm id should be specified!");
ListenableFuture<Alarm> entity = alarmDao.findAlarmByIdAsync(tenantId, alarmId.getId());
return Futures.transform(entity, function, readResultsProcessingExecutor);
}
private <T> T getAndUpdate(TenantId tenantId, AlarmId alarmId, Function<Alarm, T> function) {
validateId(alarmId, "Alarm id should be specified!");
Alarm entity = alarmDao.findAlarmById(tenantId, alarmId.getId());
return function.apply(entity);
}
}

21
dao/src/main/java/org/thingsboard/server/dao/aspect/SqlDaoCallsAspect.java

@ -72,7 +72,7 @@ public class SqlDaoCallsAspect {
try {
if (log.isTraceEnabled()) {
logTopNTenants(snapshots, Comparator.comparing(DbCallStatsSnapshot::getTotalTiming).reversed(), 0, snapshot -> {
logSnapshot(snapshot, 0, Comparator.comparing(MethodCallStatsSnapshot::getTiming).reversed(), log::trace);
logSnapshot(snapshot, 0, Comparator.comparing(MethodCallStatsSnapshot::getTiming).reversed(), "timing", log::trace);
});
Map<String, Map<TenantId, MethodCallStatsSnapshot>> byMethodStats = new HashMap<>();
@ -97,29 +97,30 @@ public class SqlDaoCallsAspect {
} else if (log.isDebugEnabled()) {
log.debug("Total calls statistics below:");
logTopNTenants(snapshots, Comparator.comparingInt(DbCallStatsSnapshot::getTotalCalls).reversed(), 10,
s -> logSnapshot(s, 10, Comparator.comparing(MethodCallStatsSnapshot::getExecutions).reversed(), log::debug));
s -> logSnapshot(s, 10, Comparator.comparing(MethodCallStatsSnapshot::getExecutions).reversed(), "executions", log::debug));
log.debug("Total timing statistics below:");
logTopNTenants(snapshots, Comparator.comparingLong(DbCallStatsSnapshot::getTotalTiming).reversed(),
10, s -> logSnapshot(s, 10, Comparator.comparing(MethodCallStatsSnapshot::getTiming).reversed(), log::debug));
logTopNTenants(snapshots, Comparator.comparingLong(DbCallStatsSnapshot::getTotalTiming).reversed(), 10,
s -> logSnapshot(s, 10, Comparator.comparing(MethodCallStatsSnapshot::getTiming).reversed(), "timing", log::debug));
log.debug("Total errors statistics below:");
logTopNTenants(snapshots, Comparator.comparingInt(DbCallStatsSnapshot::getTotalFailure).reversed(),
10, s -> logSnapshot(s, 10, Comparator.comparing(MethodCallStatsSnapshot::getFailures).reversed(), log::debug));
logTopNTenants(snapshots, Comparator.comparingInt(DbCallStatsSnapshot::getTotalFailure).reversed(), 10,
s -> logSnapshot(s, 10, Comparator.comparing(MethodCallStatsSnapshot::getFailures).reversed(), "failures", log::debug));
} else if (log.isInfoEnabled()) {
log.info("Total calls statistics below:");
logTopNTenants(snapshots, Comparator.comparingInt(DbCallStatsSnapshot::getTotalFailure).reversed(),
3, s -> logSnapshot(s, 3, Comparator.comparing(MethodCallStatsSnapshot::getFailures).reversed(), log::info));
log.info("Total timing statistics below:");
logTopNTenants(snapshots, Comparator.comparingLong(DbCallStatsSnapshot::getTotalTiming).reversed(), 3,
s -> logSnapshot(s, 3, Comparator.comparing(MethodCallStatsSnapshot::getTiming).reversed(), "timing", log::info));
}
} finally {
statsMap.clear();
}
}
private void logSnapshot(DbCallStatsSnapshot snapshot, int limit, Comparator<MethodCallStatsSnapshot> methodStatsComparator, Consumer<String> logger) {
private void logSnapshot(DbCallStatsSnapshot snapshot, int limit, Comparator<MethodCallStatsSnapshot> methodStatsComparator, String sortingKey, Consumer<String> logger) {
logger.accept(String.format("[%s]: calls: %s, failures: %s, exec time: %s ",
snapshot.getTenantId(), snapshot.getTotalCalls(), snapshot.getTotalFailure(), snapshot.getTotalTiming()));
var stream = snapshot.getMethodStats().entrySet().stream()
.sorted(Map.Entry.comparingByValue(methodStatsComparator));
if (limit > 0) {
logger.accept(String.format("[%s] Top %s methods by %s:", snapshot.getTenantId(), limit, sortingKey));
stream = stream.limit(limit);
}
stream.forEach(e -> {

3
dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java

@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.dao.Dao;
import org.thingsboard.server.dao.ExportableEntityDao;
import org.thingsboard.server.dao.TenantEntityDao;
import org.thingsboard.server.common.data.util.TbPair;
import java.util.List;
import java.util.Optional;
@ -222,4 +223,6 @@ public interface AssetDao extends Dao<Asset>, TenantEntityDao, ExportableEntityD
* @return the list of asset objects
*/
PageData<Asset> findAssetsByTenantIdAndEdgeIdAndType(UUID tenantId, UUID edgeId, String type, PageLink pageLink);
PageData<TbPair<UUID, String>> getAllAssetTypes(PageLink pageLink);
}

6
dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogServiceImpl.java

@ -23,13 +23,13 @@ import com.google.common.collect.Lists;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.HasName;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.audit.ActionStatus;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.audit.AuditLog;
@ -257,10 +257,14 @@ public class AuditLogServiceImpl implements AuditLogService {
String browser = extractParameter(String.class, 1, additionalInfo);
String os = extractParameter(String.class, 2, additionalInfo);
String device = extractParameter(String.class, 3, additionalInfo);
String provider = extractParameter(String.class, 4, additionalInfo);
actionData.put("clientAddress", clientAddress);
actionData.put("browser", browser);
actionData.put("os", os);
actionData.put("device", device);
if (StringUtils.hasText(provider)) {
actionData.put("provider", provider);
}
break;
case PROVISION_SUCCESS:
case PROVISION_FAILURE:

9
dao/src/main/java/org/thingsboard/server/dao/edge/BaseEdgeEventService.java

@ -16,8 +16,8 @@
package org.thingsboard.server.dao.edge;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.edge.EdgeEvent;
import org.thingsboard.server.common.data.id.EdgeId;
@ -28,13 +28,12 @@ import org.thingsboard.server.dao.service.DataValidator;
@Service
@Slf4j
@AllArgsConstructor
public class BaseEdgeEventService implements EdgeEventService {
@Autowired
private EdgeEventDao edgeEventDao;
private final EdgeEventDao edgeEventDao;
@Autowired
private DataValidator<EdgeEvent> edgeEventValidator;
private final DataValidator<EdgeEvent> edgeEventValidator;
@Override
public ListenableFuture<Void> saveAsync(EdgeEvent edgeEvent) {

2
dao/src/main/java/org/thingsboard/server/dao/edge/EdgeEventDao.java

@ -54,4 +54,6 @@ public interface EdgeEventDao extends Dao<EdgeEvent> {
*/
void cleanupEvents(long ttl);
void migrateEdgeEvents();
}

146
dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java

@ -20,7 +20,10 @@ import com.google.common.collect.Lists;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.SettableFuture;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.annotation.Lazy;
import org.springframework.dao.ConcurrencyFailureException;
@ -28,6 +31,7 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.event.TransactionalEventListener;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.cache.TbTransactionalCache;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.id.EntityId;
@ -44,13 +48,22 @@ import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.service.ConstraintValidator;
import org.thingsboard.server.dao.sql.JpaExecutorService;
import org.thingsboard.server.dao.sql.relation.JpaRelationQueryExecutorService;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
import static org.thingsboard.server.dao.service.Validator.validateId;
@ -67,15 +80,34 @@ public class BaseRelationService implements RelationService {
private final TbTransactionalCache<RelationCacheKey, RelationCacheValue> cache;
private final ApplicationEventPublisher eventPublisher;
private final JpaExecutorService executor;
private final JpaRelationQueryExecutorService relationsExecutor;
protected ScheduledExecutorService timeoutExecutorService;
@Value("${sql.relations.query_timeout:20}")
private Integer relationQueryTimeout;
public BaseRelationService(RelationDao relationDao, @Lazy EntityService entityService,
TbTransactionalCache<RelationCacheKey, RelationCacheValue> cache,
ApplicationEventPublisher eventPublisher, JpaExecutorService executor) {
ApplicationEventPublisher eventPublisher, JpaExecutorService executor,
JpaRelationQueryExecutorService relationsExecutor) {
this.relationDao = relationDao;
this.entityService = entityService;
this.cache = cache;
this.eventPublisher = eventPublisher;
this.executor = executor;
this.relationsExecutor = relationsExecutor;
}
@PostConstruct
public void init() {
timeoutExecutorService = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("relations-query-timeout"));
}
@PreDestroy
public void destroy() {
if (timeoutExecutorService != null) {
timeoutExecutorService.shutdownNow();
}
}
@TransactionalEventListener(classes = EntityRelationEvent.class)
@ -375,7 +407,6 @@ public class BaseRelationService implements RelationService {
@Override
public ListenableFuture<List<EntityRelation>> findByQuery(TenantId tenantId, EntityRelationsQuery query) {
//boolean fetchLastLevelOnly = true;
log.trace("Executing findByQuery [{}]", query);
RelationsSearchParameters params = query.getParameters();
final List<RelationEntityTypeFilter> filters = query.getFilters();
@ -386,7 +417,8 @@ public class BaseRelationService implements RelationService {
int maxLvl = params.getMaxLevel() > 0 ? params.getMaxLevel() : Integer.MAX_VALUE;
try {
ListenableFuture<Set<EntityRelation>> relationSet = findRelationsRecursively(tenantId, params.getEntityId(), params.getDirection(), params.getRelationTypeGroup(), maxLvl, params.isFetchLastLevelOnly(), new ConcurrentHashMap<>());
ListenableFuture<Set<EntityRelation>> relationSet = findRelationsRecursively(tenantId, params.getEntityId(), params.getDirection(),
params.getRelationTypeGroup(), maxLvl, params.isFetchLastLevelOnly(), new ConcurrentHashMap<>());
return Futures.transform(relationSet, input -> {
List<EntityRelation> relations = new ArrayList<>();
if (filters == null || filters.isEmpty()) {
@ -510,53 +542,89 @@ public class BaseRelationService implements RelationService {
}
}
private ListenableFuture<Set<EntityRelation>> findRelationsRecursively(final TenantId tenantId, final EntityId rootId, final EntitySearchDirection direction,
RelationTypeGroup relationTypeGroup, int lvl, boolean fetchLastLevelOnly,
final ConcurrentHashMap<EntityId, Boolean> uniqueMap) throws Exception {
if (lvl == 0) {
return Futures.immediateFuture(Collections.emptySet());
}
lvl--;
//TODO: try to remove this blocking operation
Set<EntityRelation> children = new HashSet<>(findRelations(tenantId, rootId, direction, relationTypeGroup).get());
Set<EntityId> childrenIds = new HashSet<>();
for (EntityRelation childRelation : children) {
log.trace("Found Relation: {}", childRelation);
EntityId childId;
if (direction == EntitySearchDirection.FROM) {
childId = childRelation.getTo();
} else {
childId = childRelation.getFrom();
@RequiredArgsConstructor
private static class RelationQueueCtx {
final SettableFuture<Set<EntityRelation>> future = SettableFuture.create();
final Set<EntityRelation> result = ConcurrentHashMap.newKeySet();
final Queue<RelationTask> tasks = new ConcurrentLinkedQueue<>();
final TenantId tenantId;
final EntitySearchDirection direction;
final RelationTypeGroup relationTypeGroup;
final boolean fetchLastLevelOnly;
final int maxLvl;
final ConcurrentHashMap<EntityId, Boolean> uniqueMap;
}
@RequiredArgsConstructor
private static class RelationTask {
private final int currentLvl;
private final EntityId root;
private final List<EntityRelation> prevRelations;
}
private void processQueue(RelationQueueCtx ctx) {
RelationTask task = ctx.tasks.poll();
while (task != null) {
List<EntityRelation> relations = findRelations(ctx.tenantId, task.root, ctx.direction, ctx.relationTypeGroup);
Map<EntityId, List<EntityRelation>> newChildrenRelations = new HashMap<>();
for (EntityRelation childRelation : relations) {
log.trace("Found Relation: {}", childRelation);
EntityId childId = ctx.direction == EntitySearchDirection.FROM ? childRelation.getTo() : childRelation.getFrom();
if (ctx.uniqueMap.putIfAbsent(childId, Boolean.TRUE) == null) {
log.trace("Adding Relation: {}", childId);
newChildrenRelations.put(childId, new ArrayList<>());
}
if (ctx.fetchLastLevelOnly) {
var list = newChildrenRelations.get(childId);
if (list != null) {
list.add(childRelation);
}
}
}
if (uniqueMap.putIfAbsent(childId, Boolean.TRUE) == null) {
log.trace("Adding Relation: {}", childId);
if (childrenIds.add(childId)) {
log.trace("Added Relation: {}", childId);
if (ctx.fetchLastLevelOnly) {
if (relations.isEmpty()) {
ctx.result.addAll(task.prevRelations);
} else if (task.currentLvl == ctx.maxLvl) {
ctx.result.addAll(relations);
}
} else {
ctx.result.addAll(relations);
}
var finalTask = task;
newChildrenRelations.forEach((child, childRelations) -> {
var newLvl = finalTask.currentLvl + 1;
if (newLvl <= ctx.maxLvl)
ctx.tasks.add(new RelationTask(newLvl, child, childRelations));
});
task = ctx.tasks.poll();
}
List<ListenableFuture<Set<EntityRelation>>> futures = new ArrayList<>();
for (EntityId entityId : childrenIds) {
futures.add(findRelationsRecursively(tenantId, entityId, direction, relationTypeGroup, lvl, fetchLastLevelOnly, uniqueMap));
}
//TODO: try to remove this blocking operation
List<Set<EntityRelation>> relations = Futures.successfulAsList(futures).get();
if (fetchLastLevelOnly && lvl > 0) {
children.clear();
ctx.future.set(ctx.result);
}
private ListenableFuture<Set<EntityRelation>> findRelationsRecursively(final TenantId tenantId, final EntityId rootId, final EntitySearchDirection direction,
RelationTypeGroup relationTypeGroup, int lvl, boolean fetchLastLevelOnly,
final ConcurrentHashMap<EntityId, Boolean> uniqueMap) {
if (lvl == 0) {
return Futures.immediateFuture(Collections.emptySet());
}
relations.forEach(r -> r.forEach(children::add));
return Futures.immediateFuture(children);
var relationQueueCtx = new RelationQueueCtx(tenantId, direction, relationTypeGroup, fetchLastLevelOnly, lvl, uniqueMap);
relationQueueCtx.tasks.add(new RelationTask(1, rootId, Collections.emptyList()));
relationsExecutor.submit(() -> processQueue(relationQueueCtx));
return Futures.withTimeout(relationQueueCtx.future, relationQueryTimeout, TimeUnit.SECONDS, timeoutExecutorService);
}
private ListenableFuture<List<EntityRelation>> findRelations(final TenantId tenantId, final EntityId rootId, final EntitySearchDirection direction, RelationTypeGroup relationTypeGroup) {
ListenableFuture<List<EntityRelation>> relations;
private List<EntityRelation> findRelations(final TenantId tenantId, final EntityId rootId, final EntitySearchDirection direction, RelationTypeGroup relationTypeGroup) {
List<EntityRelation> relations;
if (relationTypeGroup == null) {
relationTypeGroup = RelationTypeGroup.COMMON;
}
if (direction == EntitySearchDirection.FROM) {
relations = findByFromAsync(tenantId, rootId, relationTypeGroup);
relations = findByFrom(tenantId, rootId, relationTypeGroup);
} else {
relations = findByToAsync(tenantId, rootId, relationTypeGroup);
relations = findByTo(tenantId, rootId, relationTypeGroup);
}
return relations;
}

171
dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java

@ -17,16 +17,6 @@ package org.thingsboard.server.dao.service.validator;
import com.google.protobuf.Descriptors;
import com.google.protobuf.DynamicMessage;
import com.squareup.wire.Syntax;
import com.squareup.wire.schema.Field;
import com.squareup.wire.schema.Location;
import com.squareup.wire.schema.internal.parser.EnumElement;
import com.squareup.wire.schema.internal.parser.FieldElement;
import com.squareup.wire.schema.internal.parser.MessageElement;
import com.squareup.wire.schema.internal.parser.OneOfElement;
import com.squareup.wire.schema.internal.parser.ProtoFileElement;
import com.squareup.wire.schema.internal.parser.ProtoParser;
import com.squareup.wire.schema.internal.parser.TypeElement;
import org.eclipse.leshan.core.util.SecurityUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
@ -35,6 +25,7 @@ import org.springframework.util.CollectionUtils;
import org.thingsboard.server.common.data.DashboardInfo;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.DeviceProfileProvisionType;
import org.thingsboard.server.common.data.DynamicProtoUtils;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode;
import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration;
@ -67,16 +58,16 @@ import org.thingsboard.server.dao.tenant.TenantService;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@Component
public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator<DeviceProfile> {
private static final Location LOCATION = new Location("", "", -1, -1);
private static final String ATTRIBUTES_PROTO_SCHEMA = "attributes proto schema";
private static final String TELEMETRY_PROTO_SCHEMA = "telemetry proto schema";
private static final String RPC_REQUEST_PROTO_SCHEMA = "rpc request proto schema";
private static final String RPC_RESPONSE_PROTO_SCHEMA = "rpc response proto schema";
private static final String EXCEPTION_PREFIX = "[Transport Configuration]";
@Autowired
private DeviceProfileDao deviceProfileDao;
@Autowired
@ -94,10 +85,6 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator<D
@Autowired
private DashboardService dashboardService;
private static String invalidSchemaProvidedMessage(String schemaName) {
return "[Transport Configuration] invalid " + schemaName + " provided!";
}
@Override
protected void validateDataImpl(TenantId tenantId, DeviceProfile deviceProfile) {
if (StringUtils.isEmpty(deviceProfile.getName())) {
@ -229,151 +216,37 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator<D
private void validateProtoSchemas(ProtoTransportPayloadConfiguration protoTransportPayloadTypeConfiguration) {
try {
validateTransportProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceAttributesProtoSchema(), ATTRIBUTES_PROTO_SCHEMA);
validateTransportProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceTelemetryProtoSchema(), TELEMETRY_PROTO_SCHEMA);
validateTransportProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceRpcRequestProtoSchema(), RPC_REQUEST_PROTO_SCHEMA);
validateTransportProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceRpcResponseProtoSchema(), RPC_RESPONSE_PROTO_SCHEMA);
DynamicProtoUtils.validateProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceAttributesProtoSchema(), ATTRIBUTES_PROTO_SCHEMA, EXCEPTION_PREFIX);
DynamicProtoUtils.validateProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceTelemetryProtoSchema(), TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX);
DynamicProtoUtils.validateProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceRpcRequestProtoSchema(), RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX);
DynamicProtoUtils.validateProtoSchema(protoTransportPayloadTypeConfiguration.getDeviceRpcResponseProtoSchema(), RPC_RESPONSE_PROTO_SCHEMA, EXCEPTION_PREFIX);
} catch (Exception exception) {
throw new DataValidationException(exception.getMessage());
}
}
private void validateTransportProtoSchema(String schema, String schemaName) throws IllegalArgumentException {
ProtoParser schemaParser = new ProtoParser(LOCATION, schema.toCharArray());
ProtoFileElement protoFileElement;
try {
protoFileElement = schemaParser.readProtoFile();
} catch (Exception e) {
throw new IllegalArgumentException("[Transport Configuration] failed to parse " + schemaName + " due to: " + e.getMessage());
}
checkProtoFileSyntax(schemaName, protoFileElement);
checkProtoFileCommonSettings(schemaName, protoFileElement.getOptions().isEmpty(), " Schema options don't support!");
checkProtoFileCommonSettings(schemaName, protoFileElement.getPublicImports().isEmpty(), " Schema public imports don't support!");
checkProtoFileCommonSettings(schemaName, protoFileElement.getImports().isEmpty(), " Schema imports don't support!");
checkProtoFileCommonSettings(schemaName, protoFileElement.getExtendDeclarations().isEmpty(), " Schema extend declarations don't support!");
checkTypeElements(schemaName, protoFileElement);
}
private void checkProtoFileSyntax(String schemaName, ProtoFileElement protoFileElement) {
if (protoFileElement.getSyntax() == null || !protoFileElement.getSyntax().equals(Syntax.PROTO_3)) {
throw new IllegalArgumentException("[Transport Configuration] invalid schema syntax: " + protoFileElement.getSyntax() +
" for " + schemaName + " provided! Only " + Syntax.PROTO_3 + " allowed!");
}
}
private void checkProtoFileCommonSettings(String schemaName, boolean isEmptySettings, String invalidSettingsMessage) {
if (!isEmptySettings) {
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + invalidSettingsMessage);
}
}
private void checkTypeElements(String schemaName, ProtoFileElement protoFileElement) {
List<TypeElement> types = protoFileElement.getTypes();
if (!types.isEmpty()) {
if (types.stream().noneMatch(typeElement -> typeElement instanceof MessageElement)) {
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " At least one Message definition should exists!");
} else {
checkEnumElements(schemaName, getEnumElements(types));
checkMessageElements(schemaName, getMessageTypes(types));
}
} else {
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " Type elements is empty!");
}
}
private void checkFieldElements(String schemaName, List<FieldElement> fieldElements) {
if (!fieldElements.isEmpty()) {
boolean hasRequiredLabel = fieldElements.stream().anyMatch(fieldElement -> {
Field.Label label = fieldElement.getLabel();
return label != null && label.equals(Field.Label.REQUIRED);
});
if (hasRequiredLabel) {
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " Required labels are not supported!");
}
boolean hasDefaultValue = fieldElements.stream().anyMatch(fieldElement -> fieldElement.getDefaultValue() != null);
if (hasDefaultValue) {
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " Default values are not supported!");
}
}
}
private void checkEnumElements(String schemaName, List<EnumElement> enumTypes) {
if (enumTypes.stream().anyMatch(enumElement -> !enumElement.getNestedTypes().isEmpty())) {
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " Nested types in Enum definitions are not supported!");
}
if (enumTypes.stream().anyMatch(enumElement -> !enumElement.getOptions().isEmpty())) {
throw new IllegalArgumentException(invalidSchemaProvidedMessage(schemaName) + " Enum definitions options are not supported!");
}
}
private void checkMessageElements(String schemaName, List<MessageElement> messageElementsList) {
if (!messageElementsList.isEmpty()) {
messageElementsList.forEach(messageElement -> {
checkProtoFileCommonSettings(schemaName, messageElement.getGroups().isEmpty(),
" Message definition groups don't support!");
checkProtoFileCommonSettings(schemaName, messageElement.getOptions().isEmpty(),
" Message definition options don't support!");
checkProtoFileCommonSettings(schemaName, messageElement.getExtensions().isEmpty(),
" Message definition extensions don't support!");
checkProtoFileCommonSettings(schemaName, messageElement.getReserveds().isEmpty(),
" Message definition reserved elements don't support!");
checkFieldElements(schemaName, messageElement.getFields());
List<OneOfElement> oneOfs = messageElement.getOneOfs();
if (!oneOfs.isEmpty()) {
oneOfs.forEach(oneOfElement -> {
checkProtoFileCommonSettings(schemaName, oneOfElement.getGroups().isEmpty(),
" OneOf definition groups don't support!");
checkFieldElements(schemaName, oneOfElement.getFields());
});
}
List<TypeElement> nestedTypes = messageElement.getNestedTypes();
if (!nestedTypes.isEmpty()) {
List<EnumElement> nestedEnumTypes = getEnumElements(nestedTypes);
if (!nestedEnumTypes.isEmpty()) {
checkEnumElements(schemaName, nestedEnumTypes);
}
List<MessageElement> nestedMessageTypes = getMessageTypes(nestedTypes);
checkMessageElements(schemaName, nestedMessageTypes);
}
});
}
}
private List<MessageElement> getMessageTypes(List<TypeElement> types) {
return types.stream()
.filter(typeElement -> typeElement instanceof MessageElement)
.map(typeElement -> (MessageElement) typeElement)
.collect(Collectors.toList());
}
private List<EnumElement> getEnumElements(List<TypeElement> types) {
return types.stream()
.filter(typeElement -> typeElement instanceof EnumElement)
.map(typeElement -> (EnumElement) typeElement)
.collect(Collectors.toList());
}
private void validateTelemetryDynamicMessageFields(ProtoTransportPayloadConfiguration protoTransportPayloadTypeConfiguration) {
String deviceTelemetryProtoSchema = protoTransportPayloadTypeConfiguration.getDeviceTelemetryProtoSchema();
Descriptors.Descriptor telemetryDynamicMessageDescriptor = protoTransportPayloadTypeConfiguration.getTelemetryDynamicMessageDescriptor(deviceTelemetryProtoSchema);
if (telemetryDynamicMessageDescriptor == null) {
throw new DataValidationException(invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA) + " Failed to get telemetryDynamicMessageDescriptor!");
throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Failed to get telemetryDynamicMessageDescriptor!");
} else {
List<Descriptors.FieldDescriptor> fields = telemetryDynamicMessageDescriptor.getFields();
if (CollectionUtils.isEmpty(fields)) {
throw new DataValidationException(invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA) + " " + telemetryDynamicMessageDescriptor.getName() + " fields is empty!");
throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX) + " " + telemetryDynamicMessageDescriptor.getName() + " fields is empty!");
} else if (fields.size() == 2) {
Descriptors.FieldDescriptor tsFieldDescriptor = telemetryDynamicMessageDescriptor.findFieldByName("ts");
Descriptors.FieldDescriptor valuesFieldDescriptor = telemetryDynamicMessageDescriptor.findFieldByName("values");
if (tsFieldDescriptor != null && valuesFieldDescriptor != null) {
if (!Descriptors.FieldDescriptor.Type.MESSAGE.equals(valuesFieldDescriptor.getType())) {
throw new DataValidationException(invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA) + " Field 'values' has invalid data type. Only message type is supported!");
throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'values' has invalid data type. Only message type is supported!");
}
if (!Descriptors.FieldDescriptor.Type.INT64.equals(tsFieldDescriptor.getType())) {
throw new DataValidationException(invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA) + " Field 'ts' has invalid data type. Only int64 type is supported!");
throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'ts' has invalid data type. Only int64 type is supported!");
}
if (!tsFieldDescriptor.hasOptionalKeyword()) {
throw new DataValidationException(invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA) + " Field 'ts' has invalid label. Field 'ts' should have optional keyword!");
throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(TELEMETRY_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'ts' has invalid label. Field 'ts' should have optional keyword!");
}
}
}
@ -384,39 +257,39 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator<D
DynamicMessage.Builder rpcRequestDynamicMessageBuilder = protoTransportPayloadTypeConfiguration.getRpcRequestDynamicMessageBuilder(protoTransportPayloadTypeConfiguration.getDeviceRpcRequestProtoSchema());
Descriptors.Descriptor rpcRequestDynamicMessageDescriptor = rpcRequestDynamicMessageBuilder.getDescriptorForType();
if (rpcRequestDynamicMessageDescriptor == null) {
throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Failed to get rpcRequestDynamicMessageDescriptor!");
throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Failed to get rpcRequestDynamicMessageDescriptor!");
} else {
if (CollectionUtils.isEmpty(rpcRequestDynamicMessageDescriptor.getFields()) || rpcRequestDynamicMessageDescriptor.getFields().size() != 3) {
throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " " + rpcRequestDynamicMessageDescriptor.getName() + " message should always contains 3 fields: method, requestId and params!");
throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " " + rpcRequestDynamicMessageDescriptor.getName() + " message should always contains 3 fields: method, requestId and params!");
}
Descriptors.FieldDescriptor methodFieldDescriptor = rpcRequestDynamicMessageDescriptor.findFieldByName("method");
if (methodFieldDescriptor == null) {
throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Failed to get field descriptor for field: method!");
throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Failed to get field descriptor for field: method!");
} else {
if (!Descriptors.FieldDescriptor.Type.STRING.equals(methodFieldDescriptor.getType())) {
throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Field 'method' has invalid data type. Only string type is supported!");
throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'method' has invalid data type. Only string type is supported!");
}
if (methodFieldDescriptor.isRepeated()) {
throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Field 'method' has invalid label!");
throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'method' has invalid label!");
}
}
Descriptors.FieldDescriptor requestIdFieldDescriptor = rpcRequestDynamicMessageDescriptor.findFieldByName("requestId");
if (requestIdFieldDescriptor == null) {
throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Failed to get field descriptor for field: requestId!");
throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Failed to get field descriptor for field: requestId!");
} else {
if (!Descriptors.FieldDescriptor.Type.INT32.equals(requestIdFieldDescriptor.getType())) {
throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Field 'requestId' has invalid data type. Only int32 type is supported!");
throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'requestId' has invalid data type. Only int32 type is supported!");
}
if (requestIdFieldDescriptor.isRepeated()) {
throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Field 'requestId' has invalid label!");
throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'requestId' has invalid label!");
}
}
Descriptors.FieldDescriptor paramsFieldDescriptor = rpcRequestDynamicMessageDescriptor.findFieldByName("params");
if (paramsFieldDescriptor == null) {
throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Failed to get field descriptor for field: params!");
throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Failed to get field descriptor for field: params!");
} else {
if (paramsFieldDescriptor.isRepeated()) {
throw new DataValidationException(invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA) + " Field 'params' has invalid label!");
throw new DataValidationException(DynamicProtoUtils.invalidSchemaProvidedMessage(RPC_REQUEST_PROTO_SCHEMA, EXCEPTION_PREFIX) + " Field 'params' has invalid label!");
}
}
}

24
dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java

@ -79,14 +79,22 @@ public class JpaAlarmDao extends JpaAbstractDao<AlarmEntity, Alarm> implements A
}
@Override
public ListenableFuture<Alarm> findLatestByOriginatorAndType(TenantId tenantId, EntityId originator, String type) {
return service.submit(() -> {
List<AlarmEntity> latest = alarmRepository.findLatestByOriginatorAndType(
originator.getId(),
type,
PageRequest.of(0, 1));
return latest.isEmpty() ? null : DaoUtil.getData(latest.get(0));
});
public Alarm findLatestByOriginatorAndType(TenantId tenantId, EntityId originator, String type) {
List<AlarmEntity> latest = alarmRepository.findLatestByOriginatorAndType(
originator.getId(),
type,
PageRequest.of(0, 1));
return latest.isEmpty() ? null : DaoUtil.getData(latest.get(0));
}
@Override
public ListenableFuture<Alarm> findLatestByOriginatorAndTypeAsync(TenantId tenantId, EntityId originator, String type) {
return service.submit(() -> findLatestByOriginatorAndType(tenantId, originator, type));
}
@Override
public Alarm findAlarmById(TenantId tenantId, UUID key) {
return findById(tenantId, key);
}
@Override

18
dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java

@ -23,6 +23,7 @@ import org.springframework.data.repository.query.Param;
import org.thingsboard.server.dao.ExportableEntityRepository;
import org.thingsboard.server.dao.model.sql.AssetEntity;
import org.thingsboard.server.dao.model.sql.AssetInfoEntity;
import org.thingsboard.server.common.data.util.TbPair;
import java.util.List;
import java.util.UUID;
@ -70,9 +71,9 @@ public interface AssetRepository extends JpaRepository<AssetEntity, UUID>, Expor
"AND a.assetProfileId = :profileId " +
"AND LOWER(a.searchText) LIKE LOWER(CONCAT('%', :searchText, '%'))")
Page<AssetEntity> findByTenantIdAndProfileId(@Param("tenantId") UUID tenantId,
@Param("profileId") UUID profileId,
@Param("searchText") String searchText,
Pageable pageable);
@Param("profileId") UUID profileId,
@Param("searchText") String searchText,
Pageable pageable);
@Query("SELECT new org.thingsboard.server.dao.model.sql.AssetInfoEntity(a, c.title, c.additionalInfo, p.name) " +
"FROM AssetEntity a " +
@ -186,14 +187,17 @@ public interface AssetRepository extends JpaRepository<AssetEntity, UUID>, Expor
"AND a.type = :type " +
"AND LOWER(a.searchText) LIKE LOWER(CONCAT('%', :searchText, '%'))")
Page<AssetEntity> findByTenantIdAndEdgeIdAndType(@Param("tenantId") UUID tenantId,
@Param("edgeId") UUID edgeId,
@Param("type") String type,
@Param("searchText") String searchText,
Pageable pageable);
@Param("edgeId") UUID edgeId,
@Param("type") String type,
@Param("searchText") String searchText,
Pageable pageable);
Long countByTenantIdAndTypeIsNot(UUID tenantId, String type);
@Query("SELECT externalId FROM AssetEntity WHERE id = :id")
UUID getExternalIdById(@Param("id") UUID id);
@Query(value = "SELECT DISTINCT new org.thingsboard.server.common.data.util.TbPair(a.tenantId , a.type) FROM AssetEntity a")
Page<TbPair<UUID, String>> getAllAssetTypes(Pageable pageable);
}

9
dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java

@ -28,14 +28,17 @@ import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.page.SortOrder;
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.asset.AssetDao;
import org.thingsboard.server.dao.model.sql.AssetEntity;
import org.thingsboard.server.dao.model.sql.AssetInfoEntity;
import org.thingsboard.server.common.data.util.TbPair;
import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao;
import org.thingsboard.server.dao.util.SqlDao;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
@ -243,6 +246,12 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao<AssetEntity, Asset> im
DaoUtil.toPageable(pageLink)));
}
public PageData<TbPair<UUID, String>> getAllAssetTypes(PageLink pageLink) {
log.debug("Try to find all asset types and pageLink [{}]", pageLink);
return DaoUtil.pageToPageData(assetRepository.getAllAssetTypes(
DaoUtil.toPageable(pageLink, Arrays.asList(new SortOrder("tenantId"), new SortOrder("type")))));
}
@Override
public Long countByTenantId(TenantId tenantId) {
return assetRepository.countByTenantIdAndTypeIsNot(tenantId.getId(), TB_SERVICE_QUEUE);

83
dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java

@ -17,10 +17,11 @@ package org.thingsboard.server.dao.sql.edge;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.edge.EdgeEvent;
@ -31,19 +32,17 @@ import org.thingsboard.server.common.data.page.TimePageLink;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.edge.EdgeEventDao;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.model.sql.EdgeEventEntity;
import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao;
import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent;
import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams;
import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper;
import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository;
import org.thingsboard.server.dao.util.SqlDao;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Comparator;
import java.util.Objects;
import java.util.UUID;
@ -52,18 +51,25 @@ import java.util.function.Function;
import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID;
@Slf4j
@Component
@SqlDao
@RequiredArgsConstructor
@Slf4j
public class JpaBaseEdgeEventDao extends JpaAbstractSearchTextDao<EdgeEventEntity, EdgeEvent> implements EdgeEventDao {
private final UUID systemTenantId = NULL_UUID;
@Autowired
ScheduledLogExecutorComponent logExecutor;
private final ScheduledLogExecutorComponent logExecutor;
private final StatsFactory statsFactory;
private final EdgeEventRepository edgeEventRepository;
private final EdgeEventInsertRepository edgeEventInsertRepository;
@Autowired
private StatsFactory statsFactory;
private final SqlPartitioningRepository partitioningRepository;
private final JdbcTemplate jdbcTemplate;
@Value("${sql.edge_events.batch_size:1000}")
private int batchSize;
@ -74,13 +80,15 @@ public class JpaBaseEdgeEventDao extends JpaAbstractSearchTextDao<EdgeEventEntit
@Value("${sql.edge_events.stats_print_interval_ms:10000}")
private long statsPrintIntervalMs;
private TbSqlBlockingQueueWrapper<EdgeEventEntity> queue;
@Value("${sql.edge_events.partitions_size:168}")
private int partitionSizeInHours;
@Autowired
private EdgeEventRepository edgeEventRepository;
@Value("${sql.ttl.edge_events.edge_events_ttl:2628000}")
private long edge_events_ttl;
@Autowired
private EdgeEventInsertRepository edgeEventInsertRepository;
private static final String TABLE_NAME = ModelConstants.EDGE_EVENT_COLUMN_FAMILY_NAME;
private TbSqlBlockingQueueWrapper<EdgeEventEntity> queue;
@Override
protected Class<EdgeEventEntity> getEntityClass() {
@ -140,6 +148,7 @@ public class JpaBaseEdgeEventDao extends JpaAbstractSearchTextDao<EdgeEventEntit
if (StringUtils.isEmpty(edgeEvent.getUid())) {
edgeEvent.setUid(edgeEvent.getId().toString());
}
partitioningRepository.createPartitionIfNotExists(TABLE_NAME, edgeEvent.getCreatedTime(), TimeUnit.HOURS.toMillis(partitionSizeInHours));
return save(new EdgeEventEntity(edgeEvent));
}
@ -189,20 +198,36 @@ public class JpaBaseEdgeEventDao extends JpaAbstractSearchTextDao<EdgeEventEntit
@Override
public void cleanupEvents(long ttl) {
log.info("Going to cleanup old edge events using ttl: {}s", ttl);
try (Connection connection = dataSource.getConnection();
PreparedStatement stmt = connection.prepareStatement("call cleanup_edge_events_by_ttl(?,?)")) {
stmt.setLong(1, ttl);
stmt.setLong(2, 0);
stmt.setQueryTimeout((int) TimeUnit.HOURS.toSeconds(1));
stmt.execute();
printWarnings(stmt);
try (ResultSet resultSet = stmt.getResultSet()) {
resultSet.next();
log.info("Total edge events removed by TTL: [{}]", resultSet.getLong(1));
}
} catch (SQLException e) {
log.error("SQLException occurred during edge events TTL task execution ", e);
partitioningRepository.dropPartitionsBefore(TABLE_NAME, ttl, TimeUnit.HOURS.toMillis(partitionSizeInHours));
}
@Override
public void migrateEdgeEvents() {
long startTime = edge_events_ttl > 0 ? System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(edge_events_ttl) : 1629158400000L;
long currentTime = System.currentTimeMillis();
var partitionStepInMs = TimeUnit.HOURS.toMillis(partitionSizeInHours);
long numberOfPartitions = (currentTime - startTime) / partitionStepInMs;
if (numberOfPartitions > 1000) {
String error = "Please adjust your edge event partitioning configuration. Configuration with partition size " +
"of " + partitionSizeInHours + " hours and corresponding TTL will use " + numberOfPartitions + " " +
"(> 1000) partitions which is not recommended!";
log.error(error);
throw new RuntimeException(error);
}
while (startTime < currentTime) {
var endTime = startTime + partitionStepInMs;
log.info("Migrating edge event for time period: {} - {}", startTime, endTime);
callMigrationFunction(startTime, endTime, partitionStepInMs);
startTime = endTime;
}
log.info("Event edge migration finished");
jdbcTemplate.execute("DROP TABLE IF EXISTS old_edge_event");
}
private void callMigrationFunction(long startTime, long endTime, long partitionSIzeInMs) {
jdbcTemplate.update("CALL migrate_edge_event(?, ?, ?)", startTime, endTime, partitionSIzeInMs);
}
}

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save