From b1d19eb3ccc3c4ec17d5c873369f285e139d626b Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 14 Feb 2025 16:45:41 +0100 Subject: [PATCH 01/88] enforce 2FA feature draft --- .../server/controller/AuthController.java | 14 ++++++- .../TwoFactorAuthConfigController.java | 13 ++---- .../controller/TwoFactorAuthController.java | 34 ++++++++++----- .../auth/ForceMfaAuthenticationToken.java | 24 +++++++++++ .../auth/mfa/DefaultTwoFactorAuthService.java | 7 ++++ .../auth/mfa/TwoFactorAuthService.java | 3 +- .../mfa/config/DefaultTwoFaConfigManager.java | 5 ++- .../auth/rest/RestAuthenticationProvider.java | 3 ++ ...RestAwareAuthenticationSuccessHandler.java | 26 ++++++++---- .../security/model/token/JwtTokenFactory.java | 13 +++--- .../controller/TwoFactorAuthConfigTest.java | 17 +++++++- .../server/controller/TwoFactorAuthTest.java | 41 +++++++++++++++++++ .../security/auth/JwtTokenFactoryTest.java | 2 +- .../common/data/security/Authority.java | 3 +- .../model/mfa/PlatformTwoFaSettings.java | 1 + 15 files changed, 166 insertions(+), 40 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/security/auth/ForceMfaAuthenticationToken.java diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index 600980316a..521c1c4ef6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.limit.LimitedApi; +import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.security.event.UserCredentialsInvalidationEvent; import org.thingsboard.server.common.data.security.event.UserSessionInvalidationEvent; @@ -48,7 +49,9 @@ import org.thingsboard.server.common.data.security.model.UserPasswordPolicy; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.dao.settings.SecuritySettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.auth.mfa.TwoFactorAuthService; import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails; +import org.thingsboard.server.service.security.auth.rest.RestAwareAuthenticationSuccessHandler; import org.thingsboard.server.service.security.model.ActivateUserRequest; import org.thingsboard.server.service.security.model.ChangePasswordRequest; import org.thingsboard.server.service.security.model.ResetPasswordEmailRequest; @@ -74,7 +77,8 @@ public class AuthController extends BaseController { private final SecuritySettingsService securitySettingsService; private final RateLimitService rateLimitService; private final ApplicationEventPublisher eventPublisher; - + private final TwoFactorAuthService twoFactorAuthService; + private final RestAwareAuthenticationSuccessHandler authenticationSuccessHandler; @ApiOperation(value = "Get current User (getUser)", notes = "Get the information about the User which credentials are used to perform this REST API call.") @@ -221,7 +225,13 @@ public class AuthController extends BaseController { } } - var tokenPair = tokenFactory.createTokenPair(securityUser); + JwtPair tokenPair; + if (twoFactorAuthService.isEnforceTwoFaEnabled(securityUser.getTenantId())) { + tokenPair = authenticationSuccessHandler.createMfaTokenPair(securityUser, Authority.ENFORCE_MFA_TOKEN); + } else { + tokenPair = tokenFactory.createTokenPair(securityUser); + } + systemSecurityService.logLoginAction(user, new RestAuthenticationDetails(request), ActionType.LOGIN, null); return tokenPair; } diff --git a/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthConfigController.java b/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthConfigController.java index 01487b20a0..7a7a47027e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthConfigController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthConfigController.java @@ -56,7 +56,6 @@ public class TwoFactorAuthConfigController extends BaseController { private final TwoFaConfigManager twoFaConfigManager; private final TwoFactorAuthService twoFactorAuthService; - @ApiOperation(value = "Get account 2FA settings (getAccountTwoFaSettings)", notes = "Get user's account 2FA configuration. Configuration contains configs for different 2FA providers." + NEW_LINE + "Example:\n" + @@ -73,7 +72,6 @@ public class TwoFactorAuthConfigController extends BaseController { return twoFaConfigManager.getAccountTwoFaSettings(user.getTenantId(), user.getId()).orElse(null); } - @ApiOperation(value = "Generate 2FA account config (generateTwoFaAccountConfig)", notes = "Generate new 2FA account config template for specified provider type. " + NEW_LINE + "For TOTP, this will return a corresponding account config template " + @@ -99,7 +97,7 @@ public class TwoFactorAuthConfigController extends BaseController { "Will throw an error (Bad Request) if the provider is not configured for usage. " + ControllerConstants.AVAILABLE_FOR_ANY_AUTHORIZED_USER) @PostMapping("/account/config/generate") - @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER', 'ENFORCE_MFA_TOKEN')") public TwoFaAccountConfig generateTwoFaAccountConfig(@Parameter(description = "2FA provider type to generate new account config for", schema = @Schema(defaultValue = "TOTP", requiredMode = Schema.RequiredMode.REQUIRED)) @RequestParam TwoFaProviderType providerType) throws Exception { SecurityUser user = getCurrentUser(); @@ -139,7 +137,7 @@ public class TwoFactorAuthConfigController extends BaseController { "Will throw an error (Bad Request) if the provider is not configured for usage. " + ControllerConstants.AVAILABLE_FOR_ANY_AUTHORIZED_USER) @PostMapping("/account/config") - @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER', 'ENFORCE_MFA_TOKEN')") public AccountTwoFaSettings verifyAndSaveTwoFaAccountConfig(@Valid @RequestBody TwoFaAccountConfig accountConfig, @RequestParam(required = false) String verificationCode) throws Exception { SecurityUser user = getCurrentUser(); @@ -189,7 +187,6 @@ public class TwoFactorAuthConfigController extends BaseController { return twoFaConfigManager.deleteTwoFaAccountConfig(user.getTenantId(), user.getId(), providerType); } - @ApiOperation(value = "Get available 2FA providers (getAvailableTwoFaProviders)", notes = "Get the list of provider types available for user to use (the ones configured by tenant or sysadmin).\n" + "Example of response:\n" + @@ -197,7 +194,7 @@ public class TwoFactorAuthConfigController extends BaseController { ControllerConstants.AVAILABLE_FOR_ANY_AUTHORIZED_USER ) @GetMapping("/providers") - @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER', 'ENFORCE_MFA_TOKEN')") public List getAvailableTwoFaProviders() throws ThingsboardException { return twoFaConfigManager.getPlatformTwoFaSettings(getTenantId(), true) .map(PlatformTwoFaSettings::getProviders).orElse(Collections.emptyList()).stream() @@ -205,7 +202,6 @@ public class TwoFactorAuthConfigController extends BaseController { .collect(Collectors.toList()); } - @ApiOperation(value = "Get platform 2FA settings (getPlatformTwoFaSettings)", notes = "Get platform settings for 2FA. The settings are described for savePlatformTwoFaSettings API method. " + "If 2FA is not configured, then an empty response will be returned." + @@ -260,11 +256,10 @@ public class TwoFactorAuthConfigController extends BaseController { @PostMapping("/settings") @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") public PlatformTwoFaSettings savePlatformTwoFaSettings(@Parameter(description = "Settings value", required = true) - @RequestBody PlatformTwoFaSettings twoFaSettings) throws ThingsboardException { + @RequestBody PlatformTwoFaSettings twoFaSettings) throws ThingsboardException { return twoFaConfigManager.savePlatformTwoFaSettings(getTenantId(), twoFaSettings); } - @Data public static class TwoFaAccountConfigUpdateRequest { private boolean useByDefault; diff --git a/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthController.java b/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthController.java index 4e4b30a542..9d7d6ba897 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthController.java @@ -64,7 +64,6 @@ public class TwoFactorAuthController extends BaseController { private final SystemSecurityService systemSecurityService; private final UserService userService; - @ApiOperation(value = "Request 2FA verification code (requestTwoFaVerificationCode)", notes = "Request 2FA verification code." + NEW_LINE + "To make a request to this endpoint, you need an access token with the scope of PRE_VERIFICATION_TOKEN, " + @@ -91,18 +90,9 @@ public class TwoFactorAuthController extends BaseController { @RequestParam String verificationCode, HttpServletRequest servletRequest) throws Exception { SecurityUser user = getCurrentUser(); boolean verificationSuccess = twoFactorAuthService.checkVerificationCode(user, providerType, verificationCode, true); - if (verificationSuccess) { - systemSecurityService.logLoginAction(user, new RestAuthenticationDetails(servletRequest), ActionType.LOGIN, null); - user = new SecurityUser(userService.findUserById(user.getTenantId(), user.getId()), true, user.getUserPrincipal()); - return tokenFactory.createTokenPair(user); - } else { - ThingsboardException error = new ThingsboardException("Verification code is incorrect", ThingsboardErrorCode.BAD_REQUEST_PARAMS); - systemSecurityService.logLoginAction(user, new RestAuthenticationDetails(servletRequest), ActionType.LOGIN, error); - throw error; - } + return getRegularJwtPair(servletRequest, user, verificationSuccess, "Verification code is incorrect"); } - @ApiOperation(value = "Get available 2FA providers (getAvailableTwoFaProviders)", notes = "Get the list of 2FA provider infos available for user to use. Example:\n" + "```\n[\n" + @@ -139,6 +129,28 @@ public class TwoFactorAuthController extends BaseController { .collect(Collectors.toList()); } + @ApiOperation(value = "Get regular token pair after successfully saved two factor settings", + notes = "Checks 2FA setting saved, and if it success the method returns a regular access and refresh token pair.") + @PostMapping("/login") + @PreAuthorize("hasAuthority('ENFORCE_MFA_TOKEN')") + public JwtPair authorizeByTwoFaEnforceToken(HttpServletRequest servletRequest) throws ThingsboardException { + SecurityUser user = getCurrentUser(); + boolean isEnabled = twoFactorAuthService.isTwoFaEnabled(user.getTenantId(), user.getId()); + return getRegularJwtPair(servletRequest, user, isEnabled, "Two factor settings is not set up!"); + } + + private JwtPair getRegularJwtPair(HttpServletRequest servletRequest, SecurityUser user, boolean isAvailable, String errorMessage) throws ThingsboardException { + if (isAvailable) { + systemSecurityService.logLoginAction(user, new RestAuthenticationDetails(servletRequest), ActionType.LOGIN, null); + user = new SecurityUser(userService.findUserById(user.getTenantId(), user.getId()), true, user.getUserPrincipal()); + return tokenFactory.createTokenPair(user); + } else { + ThingsboardException error = new ThingsboardException(errorMessage, ThingsboardErrorCode.BAD_REQUEST_PARAMS); + systemSecurityService.logLoginAction(user, new RestAuthenticationDetails(servletRequest), ActionType.LOGIN, error); + throw error; + } + } + @Data @AllArgsConstructor @Builder diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/ForceMfaAuthenticationToken.java b/application/src/main/java/org/thingsboard/server/service/security/auth/ForceMfaAuthenticationToken.java new file mode 100644 index 0000000000..5cb0c752ab --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/ForceMfaAuthenticationToken.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth; + +import org.thingsboard.server.service.security.model.SecurityUser; + +public class ForceMfaAuthenticationToken extends AbstractJwtAuthenticationToken { + public ForceMfaAuthenticationToken(SecurityUser securityUser) { + super(securityUser); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/DefaultTwoFactorAuthService.java b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/DefaultTwoFactorAuthService.java index ab7fc19202..33911eef3b 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/DefaultTwoFactorAuthService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/DefaultTwoFactorAuthService.java @@ -67,6 +67,13 @@ public class DefaultTwoFactorAuthService implements TwoFactorAuthService { .orElse(false); } + @Override + public boolean isEnforceTwoFaEnabled(TenantId tenantId) { + return configManager.getPlatformTwoFaSettings(tenantId, true) + .map(PlatformTwoFaSettings::isEnforceTwoFa) + .orElse(false); + } + @Override public void checkProvider(TenantId tenantId, TwoFaProviderType providerType) throws ThingsboardException { getTwoFaProvider(providerType).check(tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/TwoFactorAuthService.java b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/TwoFactorAuthService.java index f1e2e34322..9336bc0d65 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/TwoFactorAuthService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/TwoFactorAuthService.java @@ -27,8 +27,9 @@ public interface TwoFactorAuthService { boolean isTwoFaEnabled(TenantId tenantId, UserId userId); - void checkProvider(TenantId tenantId, TwoFaProviderType providerType) throws ThingsboardException; + boolean isEnforceTwoFaEnabled(TenantId tenantId); + void checkProvider(TenantId tenantId, TwoFaProviderType providerType) throws ThingsboardException; void prepareVerificationCode(SecurityUser user, TwoFaProviderType providerType, boolean checkLimits) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java index a5b44b2add..7e505da11d 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.security.model.mfa.account.AccountTwoF import org.thingsboard.server.common.data.security.model.mfa.account.TwoFaAccountConfig; import org.thingsboard.server.common.data.security.model.mfa.provider.TwoFaProviderConfig; import org.thingsboard.server.common.data.security.model.mfa.provider.TwoFaProviderType; +import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.ConstraintValidator; import org.thingsboard.server.dao.settings.AdminSettingsDao; import org.thingsboard.server.dao.settings.AdminSettingsService; @@ -166,7 +167,9 @@ public class DefaultTwoFaConfigManager implements TwoFaConfigManager { for (TwoFaProviderConfig providerConfig : twoFactorAuthSettings.getProviders()) { twoFactorAuthService.checkProvider(tenantId, providerConfig.getProviderType()); } - + if (twoFactorAuthSettings.isEnforceTwoFa() && twoFactorAuthSettings.getProviders().isEmpty()) { + throw new DataValidationException("At least one 2FA provider is required if enforce enabled!"); + } AdminSettings settings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(tenantId, TWO_FACTOR_AUTH_SETTINGS_KEY)) .orElseGet(() -> { AdminSettings newSettings = new AdminSettings(); diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java index b9fe54deec..08166ab6c9 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java @@ -43,6 +43,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.settings.SecuritySettingsService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.auth.ForceMfaAuthenticationToken; import org.thingsboard.server.service.security.auth.MfaAuthenticationToken; import org.thingsboard.server.service.security.auth.mfa.TwoFactorAuthService; import org.thingsboard.server.service.security.exception.UserPasswordNotValidException; @@ -105,6 +106,8 @@ public class RestAuthenticationProvider implements AuthenticationProvider { securityUser = authenticateByUsernameAndPassword(authentication, userPrincipal, username, password); if (twoFactorAuthService.isTwoFaEnabled(securityUser.getTenantId(), securityUser.getId())) { return new MfaAuthenticationToken(securityUser); + } else if (twoFactorAuthService.isEnforceTwoFaEnabled(securityUser.getTenantId())) { + return new ForceMfaAuthenticationToken(securityUser); } else { systemSecurityService.logLoginAction(securityUser, authentication.getDetails(), ActionType.LOGIN, null); } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java index 5dbcac0f84..ebaab2dd5e 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java @@ -29,6 +29,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.model.JwtPair; +import org.thingsboard.server.service.security.auth.ForceMfaAuthenticationToken; import org.thingsboard.server.service.security.auth.MfaAuthenticationToken; import org.thingsboard.server.service.security.auth.mfa.config.TwoFaConfigManager; import org.thingsboard.server.service.security.model.SecurityUser; @@ -48,16 +49,13 @@ public class RestAwareAuthenticationSuccessHandler implements AuthenticationSucc public void onAuthenticationSuccess(HttpServletRequest request, HttpServletResponse response, Authentication authentication) throws IOException, ServletException { SecurityUser securityUser = (SecurityUser) authentication.getPrincipal(); - JwtPair tokenPair = new JwtPair(); + JwtPair tokenPair; if (authentication instanceof MfaAuthenticationToken) { - int preVerificationTokenLifetime = twoFaConfigManager.getPlatformTwoFaSettings(securityUser.getTenantId(), true) - .flatMap(settings -> Optional.ofNullable(settings.getTotalAllowedTimeForVerification()) - .filter(time -> time > 0)) - .orElse((int) TimeUnit.MINUTES.toSeconds(30)); - tokenPair.setToken(tokenFactory.createPreVerificationToken(securityUser, preVerificationTokenLifetime).getToken()); - tokenPair.setRefreshToken(null); - tokenPair.setScope(Authority.PRE_VERIFICATION_TOKEN); + tokenPair = createMfaTokenPair(securityUser, Authority.PRE_VERIFICATION_TOKEN); + } + else if (authentication instanceof ForceMfaAuthenticationToken) { + tokenPair = createMfaTokenPair(securityUser, Authority.ENFORCE_MFA_TOKEN); } else { tokenPair = tokenFactory.createTokenPair(securityUser); } @@ -69,6 +67,18 @@ public class RestAwareAuthenticationSuccessHandler implements AuthenticationSucc clearAuthenticationAttributes(request); } + public JwtPair createMfaTokenPair(SecurityUser securityUser, Authority scope) { + JwtPair tokenPair = new JwtPair(); + int preVerificationTokenLifetime = twoFaConfigManager.getPlatformTwoFaSettings(securityUser.getTenantId(), true) + .flatMap(settings -> Optional.ofNullable(settings.getTotalAllowedTimeForVerification()) + .filter(time -> time > 0)) + .orElse((int) TimeUnit.MINUTES.toSeconds(30)); + tokenPair.setToken(tokenFactory.createMfaToken(securityUser, scope, preVerificationTokenLifetime).getToken()); + tokenPair.setRefreshToken(null); + tokenPair.setScope(scope); + return tokenPair; + } + /** * Removes temporary authentication-related data which may have been stored * in the session during the authentication process.. diff --git a/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java b/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java index f68d954aa1..52ba9981bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java @@ -115,13 +115,16 @@ public class JwtTokenFactory { throw new IllegalArgumentException("JWT Token doesn't have any scopes"); } + Authority authority = Authority.parse(scopes.get(0)); + SecurityUser securityUser = new SecurityUser(new UserId(UUID.fromString(claims.get(USER_ID, String.class)))); securityUser.setEmail(subject); - securityUser.setAuthority(Authority.parse(scopes.get(0))); + securityUser.setAuthority(authority); String tenantId = claims.get(TENANT_ID, String.class); + if (tenantId != null) { securityUser.setTenantId(TenantId.fromUUID(UUID.fromString(tenantId))); - } else if (securityUser.getAuthority() == Authority.SYS_ADMIN) { + } else if (authority == Authority.SYS_ADMIN) { securityUser.setTenantId(TenantId.SYS_TENANT_ID); } String customerId = claims.get(CUSTOMER_ID, String.class); @@ -133,7 +136,7 @@ public class JwtTokenFactory { } UserPrincipal principal; - if (securityUser.getAuthority() != Authority.PRE_VERIFICATION_TOKEN) { + if (authority != Authority.PRE_VERIFICATION_TOKEN && authority != Authority.ENFORCE_MFA_TOKEN) { securityUser.setFirstName(claims.get(FIRST_NAME, String.class)); securityUser.setLastName(claims.get(LAST_NAME, String.class)); securityUser.setEnabled(claims.get(ENABLED, Boolean.class)); @@ -179,8 +182,8 @@ public class JwtTokenFactory { return securityUser; } - public JwtToken createPreVerificationToken(SecurityUser user, Integer expirationTime) { - JwtBuilder jwtBuilder = setUpToken(user, Collections.singletonList(Authority.PRE_VERIFICATION_TOKEN.name()), expirationTime) + public JwtToken createMfaToken(SecurityUser user, Authority scope, Integer expirationTime) { + JwtBuilder jwtBuilder = setUpToken(user, Collections.singletonList(scope.name()), expirationTime) .claim(TENANT_ID, user.getTenantId().toString()); if (user.getCustomerId() != null) { jwtBuilder.claim(CUSTOMER_ID, user.getCustomerId().toString()); diff --git a/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthConfigTest.java b/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthConfigTest.java index a8101c7801..f5703ea13a 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthConfigTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthConfigTest.java @@ -85,7 +85,6 @@ public class TwoFactorAuthConfigTest extends AbstractControllerTest { twoFaConfigManager.deletePlatformTwoFaSettings(tenantId); } - @Test public void testSavePlatformTwoFaSettings() throws Exception { loginSysAdmin(); @@ -102,6 +101,7 @@ public class TwoFactorAuthConfigTest extends AbstractControllerTest { twoFaSettings.setVerificationCodeCheckRateLimit("3:900"); twoFaSettings.setMaxVerificationFailuresBeforeUserLockout(10); twoFaSettings.setTotalAllowedTimeForVerification(3600); + twoFaSettings.setEnforceTwoFa(true); doPost("/api/2fa/settings", twoFaSettings).andExpect(status().isOk()); @@ -111,6 +111,21 @@ public class TwoFactorAuthConfigTest extends AbstractControllerTest { assertThat(savedTwoFaSettings.getProviders()).contains(totpTwoFaProviderConfig, smsTwoFaProviderConfig); } + @Test + public void testSavePlatformTwoFaSettingsWithEnforceTwoFaWithoutProviders() throws Exception { + loginSysAdmin(); + + PlatformTwoFaSettings twoFaSettings = new PlatformTwoFaSettings(); + twoFaSettings.setProviders(List.of()); + twoFaSettings.setMinVerificationCodeSendPeriod(5); + twoFaSettings.setVerificationCodeCheckRateLimit("3:900"); + twoFaSettings.setMaxVerificationFailuresBeforeUserLockout(10); + twoFaSettings.setTotalAllowedTimeForVerification(3600); + twoFaSettings.setEnforceTwoFa(true); + + doPost("/api/2fa/settings", twoFaSettings).andExpect(status().isBadRequest()); + } + @Test public void testSavePlatformTwoFaSettings_validationError() throws Exception { loginSysAdmin(); diff --git a/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java b/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java index 7fa848f95d..2f21e1afe6 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java @@ -25,6 +25,7 @@ import org.mockito.ArgumentCaptor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.web.util.UriComponentsBuilder; import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.User; @@ -66,6 +67,10 @@ import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -395,6 +400,42 @@ public class TwoFactorAuthTest extends AbstractControllerTest { assertThat(providersInfos.get(TwoFaProviderType.EMAIL).isDefault()).isFalse(); } + @Test + public void testEnforceTwoFactorSetting() throws Exception { + TotpTwoFaProviderConfig totpTwoFaProviderConfig = new TotpTwoFaProviderConfig(); + totpTwoFaProviderConfig.setIssuerName("tb"); + + PlatformTwoFaSettings twoFaSettings = new PlatformTwoFaSettings(); + twoFaSettings.setProviders(Arrays.stream(new TwoFaProviderConfig[]{totpTwoFaProviderConfig}).collect(Collectors.toList())); + twoFaSettings.setMinVerificationCodeSendPeriod(5); + twoFaSettings.setTotalAllowedTimeForVerification(100); + twoFaSettings.setEnforceTwoFa(true); + twoFaSettings = twoFaConfigManager.savePlatformTwoFaSettings(TenantId.SYS_TENANT_ID, twoFaSettings); + + JsonNode node = readResponse(doPost("/api/auth/login", new LoginRequest(username, password)).andExpect(status().isOk()), JsonNode.class); + assertNotNull(node.get("token").asText()); + assertNull(node.get("refreshToken")); + assertEquals(node.get("scope").asText(), Authority.ENFORCE_MFA_TOKEN.name()); + + this.token = node.get("token").asText(); + TotpTwoFaAccountConfig totpTwoFaAccountConfig = (TotpTwoFaAccountConfig) twoFactorAuthService.generateNewAccountConfig(user, totpTwoFaProviderConfig.getProviderType()); + String secret = UriComponentsBuilder.fromUriString(totpTwoFaAccountConfig.getAuthUrl()).build() + .getQueryParams().getFirst("secret"); + String verificationCode = new Totp(secret).now(); + readResponse(doPost("/api/2fa/account/config?verificationCode=" + verificationCode, totpTwoFaAccountConfig).andExpect(status().isOk()), JsonNode.class); + + JwtPair tokenPair = readResponse(doPost("/api/auth/2fa/login").andExpect(status().isOk()), JwtPair.class); + assertNotNull(tokenPair); + + this.token = tokenPair.getToken(); + this.refreshToken = tokenPair.getRefreshToken(); + + doGet("/api/user/" + user.getId()).andExpect(status().isOk()); + + twoFaSettings.setEnforceTwoFa(false); + twoFaConfigManager.savePlatformTwoFaSettings(TenantId.SYS_TENANT_ID, twoFaSettings); + } + private void logInWithPreVerificationToken(String username, String password) throws Exception { LoginRequest loginRequest = new LoginRequest(username, password); diff --git a/application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java b/application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java index e9d874084d..07dd3a422c 100644 --- a/application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java @@ -125,7 +125,7 @@ public class JwtTokenFactoryTest { public void testCreateAndParsePreVerificationJwtToken() { SecurityUser securityUser = createSecurityUser(); int tokenLifetime = (int) TimeUnit.MINUTES.toSeconds(30); - JwtToken preVerificationToken = tokenFactory.createPreVerificationToken(securityUser, tokenLifetime); + JwtToken preVerificationToken = tokenFactory.createMfaToken(securityUser, Authority.PRE_VERIFICATION_TOKEN, tokenLifetime); checkExpirationTime(preVerificationToken, tokenLifetime); SecurityUser parsedSecurityUser = tokenFactory.parseAccessJwtToken(preVerificationToken.getToken()); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/security/Authority.java b/common/data/src/main/java/org/thingsboard/server/common/data/security/Authority.java index 699decabd9..ccebd43ccb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/security/Authority.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/security/Authority.java @@ -21,7 +21,8 @@ public enum Authority { TENANT_ADMIN(1), CUSTOMER_USER(2), REFRESH_TOKEN(10), - PRE_VERIFICATION_TOKEN(11); + PRE_VERIFICATION_TOKEN(11), + ENFORCE_MFA_TOKEN(12); private int code; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/security/model/mfa/PlatformTwoFaSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/security/model/mfa/PlatformTwoFaSettings.java index bd2cd1fad7..35a2fac352 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/security/model/mfa/PlatformTwoFaSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/security/model/mfa/PlatformTwoFaSettings.java @@ -46,6 +46,7 @@ public class PlatformTwoFaSettings { @Min(value = 60) private Integer totalAllowedTimeForVerification; + private boolean enforceTwoFa; public Optional getProviderConfig(TwoFaProviderType providerType) { return Optional.ofNullable(providers) From 16102d22aa59d4f447ca5789b76ed6c681ddb96d Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 24 Jun 2025 11:43:09 +0300 Subject: [PATCH 02/88] Add users filter to enforce 2FA for --- .../server/controller/AuthController.java | 4 +- .../TwoFactorAuthConfigController.java | 6 +- .../controller/TwoFactorAuthController.java | 44 ++++++---- ...nToken.java => MfaConfigurationToken.java} | 6 +- .../auth/mfa/DefaultTwoFactorAuthService.java | 15 +++- .../auth/mfa/TwoFactorAuthService.java | 2 +- .../mfa/config/DefaultTwoFaConfigManager.java | 4 +- .../auth/rest/RestAuthenticationProvider.java | 9 +-- ...RestAwareAuthenticationSuccessHandler.java | 14 ++-- .../security/model/token/JwtTokenFactory.java | 2 +- .../server/controller/AbstractWebTest.java | 12 ++- .../server/controller/TwoFactorAuthTest.java | 50 ++++++------ .../dao/tenant/TbTenantProfileCache.java | 1 - .../server/dao/user/UserService.java | 6 ++ .../targets/platform/AllUsersFilter.java | 2 +- .../platform/SystemAdministratorsFilter.java | 2 +- .../platform/SystemLevelUsersFilter.java | 19 +++++ .../platform/TenantAdministratorsFilter.java | 2 +- .../common/data/security/Authority.java | 3 +- .../model/mfa/PlatformTwoFaSettings.java | 2 + .../DefaultNotificationTargetService.java | 51 +----------- .../server/dao/tenant/TenantServiceImpl.java | 1 + .../server/dao/user/UserServiceImpl.java | 80 +++++++++++++++++++ 23 files changed, 209 insertions(+), 128 deletions(-) rename application/src/main/java/org/thingsboard/server/service/security/auth/{ForceMfaAuthenticationToken.java => MfaConfigurationToken.java} (78%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/notification/targets/platform/SystemLevelUsersFilter.java diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index bb14bf76c5..b15b650c33 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -226,8 +226,8 @@ public class AuthController extends BaseController { } JwtPair tokenPair; - if (twoFactorAuthService.isEnforceTwoFaEnabled(securityUser.getTenantId())) { - tokenPair = authenticationSuccessHandler.createMfaTokenPair(securityUser, Authority.ENFORCE_MFA_TOKEN); + if (twoFactorAuthService.isEnforceTwoFaEnabled(securityUser.getTenantId(), user)) { + tokenPair = authenticationSuccessHandler.createMfaTokenPair(securityUser, Authority.MFA_CONFIGURATION_TOKEN); } else { tokenPair = tokenFactory.createTokenPair(securityUser); } diff --git a/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthConfigController.java b/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthConfigController.java index eeb0b4553b..fd78e5c945 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthConfigController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthConfigController.java @@ -97,7 +97,7 @@ public class TwoFactorAuthConfigController extends BaseController { "Will throw an error (Bad Request) if the provider is not configured for usage. " + ControllerConstants.AVAILABLE_FOR_ANY_AUTHORIZED_USER) @PostMapping("/account/config/generate") - @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER', 'ENFORCE_MFA_TOKEN')") + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER', 'MFA_CONFIGURATION_TOKEN')") public TwoFaAccountConfig generateTwoFaAccountConfig(@Parameter(description = "2FA provider type to generate new account config for", schema = @Schema(defaultValue = "TOTP", requiredMode = Schema.RequiredMode.REQUIRED)) @RequestParam TwoFaProviderType providerType) throws Exception { SecurityUser user = getCurrentUser(); @@ -137,7 +137,7 @@ public class TwoFactorAuthConfigController extends BaseController { "Will throw an error (Bad Request) if the provider is not configured for usage. " + ControllerConstants.AVAILABLE_FOR_ANY_AUTHORIZED_USER) @PostMapping("/account/config") - @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER', 'ENFORCE_MFA_TOKEN')") + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER', 'MFA_CONFIGURATION_TOKEN')") public AccountTwoFaSettings verifyAndSaveTwoFaAccountConfig(@Valid @RequestBody TwoFaAccountConfig accountConfig, @RequestParam(required = false) String verificationCode) throws Exception { SecurityUser user = getCurrentUser(); @@ -194,7 +194,7 @@ public class TwoFactorAuthConfigController extends BaseController { ControllerConstants.AVAILABLE_FOR_ANY_AUTHORIZED_USER ) @GetMapping("/providers") - @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER', 'ENFORCE_MFA_TOKEN')") + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER', 'MFA_CONFIGURATION_TOKEN')") public List getAvailableTwoFaProviders() throws ThingsboardException { return twoFaConfigManager.getPlatformTwoFaSettings(getTenantId(), true) .map(PlatformTwoFaSettings::getProviders).orElse(Collections.emptyList()).stream() diff --git a/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthController.java b/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthController.java index b31db74095..ec5bf0e054 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthController.java @@ -28,7 +28,6 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.security.model.JwtPair; import org.thingsboard.server.common.data.security.model.mfa.PlatformTwoFaSettings; @@ -90,7 +89,14 @@ public class TwoFactorAuthController extends BaseController { @RequestParam String verificationCode, HttpServletRequest servletRequest) throws Exception { SecurityUser user = getCurrentUser(); boolean verificationSuccess = twoFactorAuthService.checkVerificationCode(user, providerType, verificationCode, true); - return getRegularJwtPair(servletRequest, user, verificationSuccess, "Verification code is incorrect"); + if (verificationSuccess) { + logLogInAction(servletRequest, user, null); + return createTokenPair(user); + } else { + IllegalArgumentException error = new IllegalArgumentException("Verification code is incorrect"); + logLogInAction(servletRequest, user, error); + throw error; + } } @ApiOperation(value = "Get available 2FA providers (getAvailableTwoFaProviders)", notes = @@ -129,28 +135,32 @@ public class TwoFactorAuthController extends BaseController { .collect(Collectors.toList()); } - @ApiOperation(value = "Get regular token pair after successfully saved two factor settings", - notes = "Checks 2FA setting saved, and if it success the method returns a regular access and refresh token pair.") + @ApiOperation(value = "Get regular token pair after successfully configuring 2FA", + notes = "Checks 2FA is configured, returning token pair on success.") @PostMapping("/login") - @PreAuthorize("hasAuthority('ENFORCE_MFA_TOKEN')") - public JwtPair authorizeByTwoFaEnforceToken(HttpServletRequest servletRequest) throws ThingsboardException { + @PreAuthorize("hasAuthority('MFA_CONFIGURATION_TOKEN')") + public JwtPair authenticateByTwoFaConfigurationToken(HttpServletRequest servletRequest) throws ThingsboardException { SecurityUser user = getCurrentUser(); - boolean isEnabled = twoFactorAuthService.isTwoFaEnabled(user.getTenantId(), user.getId()); - return getRegularJwtPair(servletRequest, user, isEnabled, "Two factor settings is not set up!"); - } - - private JwtPair getRegularJwtPair(HttpServletRequest servletRequest, SecurityUser user, boolean isAvailable, String errorMessage) throws ThingsboardException { - if (isAvailable) { - systemSecurityService.logLoginAction(user, new RestAuthenticationDetails(servletRequest), ActionType.LOGIN, null); - user = new SecurityUser(userService.findUserById(user.getTenantId(), user.getId()), true, user.getUserPrincipal()); - return tokenFactory.createTokenPair(user); + if (twoFactorAuthService.isTwoFaEnabled(user.getTenantId(), user.getId())) { + logLogInAction(servletRequest, user, null); + return createTokenPair(user); } else { - ThingsboardException error = new ThingsboardException(errorMessage, ThingsboardErrorCode.BAD_REQUEST_PARAMS); - systemSecurityService.logLoginAction(user, new RestAuthenticationDetails(servletRequest), ActionType.LOGIN, error); + IllegalArgumentException error = new IllegalArgumentException("2FA is not configured"); + logLogInAction(servletRequest, user, error); throw error; } } + private JwtPair createTokenPair(SecurityUser user) { + log.debug("[{}][{}] Creating token pair for user", user.getTenantId(), user.getId()); + user = new SecurityUser(userService.findUserById(user.getTenantId(), user.getId()), true, user.getUserPrincipal()); + return tokenFactory.createTokenPair(user); + } + + private void logLogInAction(HttpServletRequest servletRequest, SecurityUser user, Exception error) { + systemSecurityService.logLoginAction(user, new RestAuthenticationDetails(servletRequest), ActionType.LOGIN, error); + } + @Data @AllArgsConstructor @Builder diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/ForceMfaAuthenticationToken.java b/application/src/main/java/org/thingsboard/server/service/security/auth/MfaConfigurationToken.java similarity index 78% rename from application/src/main/java/org/thingsboard/server/service/security/auth/ForceMfaAuthenticationToken.java rename to application/src/main/java/org/thingsboard/server/service/security/auth/MfaConfigurationToken.java index 5cb0c752ab..b52404bac2 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/ForceMfaAuthenticationToken.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/MfaConfigurationToken.java @@ -1,5 +1,5 @@ /** - * Copyright © 2016-2024 The Thingsboard Authors + * Copyright © 2016-2025 The Thingsboard Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,8 +17,8 @@ package org.thingsboard.server.service.security.auth; import org.thingsboard.server.service.security.model.SecurityUser; -public class ForceMfaAuthenticationToken extends AbstractJwtAuthenticationToken { - public ForceMfaAuthenticationToken(SecurityUser securityUser) { +public class MfaConfigurationToken extends AbstractJwtAuthenticationToken { + public MfaConfigurationToken(SecurityUser securityUser) { super(securityUser); } } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/DefaultTwoFactorAuthService.java b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/DefaultTwoFactorAuthService.java index 2b172f3588..c4683f5822 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/DefaultTwoFactorAuthService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/DefaultTwoFactorAuthService.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.limit.LimitedApi; +import org.thingsboard.server.common.data.notification.targets.platform.SystemLevelUsersFilter; import org.thingsboard.server.common.data.security.model.mfa.PlatformTwoFaSettings; import org.thingsboard.server.common.data.security.model.mfa.account.TwoFaAccountConfig; import org.thingsboard.server.common.data.security.model.mfa.provider.TwoFaProviderConfig; @@ -68,10 +69,16 @@ public class DefaultTwoFactorAuthService implements TwoFactorAuthService { } @Override - public boolean isEnforceTwoFaEnabled(TenantId tenantId) { - return configManager.getPlatformTwoFaSettings(tenantId, true) - .map(PlatformTwoFaSettings::isEnforceTwoFa) - .orElse(false); + public boolean isEnforceTwoFaEnabled(TenantId tenantId, User user) { + SystemLevelUsersFilter enforcedUsersFilter = configManager.getPlatformTwoFaSettings(TenantId.SYS_TENANT_ID, true) + .filter(PlatformTwoFaSettings::isEnforceTwoFa) + .map(PlatformTwoFaSettings::getEnforcedUsersFilter) + .orElse(null); + if (enforcedUsersFilter == null) { + return false; + } + + return userService.matchesFilter(tenantId, enforcedUsersFilter, user); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/TwoFactorAuthService.java b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/TwoFactorAuthService.java index b8cc08bd1e..d77d8d65a0 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/TwoFactorAuthService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/TwoFactorAuthService.java @@ -27,7 +27,7 @@ public interface TwoFactorAuthService { boolean isTwoFaEnabled(TenantId tenantId, UserId userId); - boolean isEnforceTwoFaEnabled(TenantId tenantId); + boolean isEnforceTwoFaEnabled(TenantId tenantId, User user); void checkProvider(TenantId tenantId, TwoFaProviderType providerType) throws ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java index a9ef1e4a47..f3afc7fdc2 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java @@ -167,8 +167,8 @@ public class DefaultTwoFaConfigManager implements TwoFaConfigManager { for (TwoFaProviderConfig providerConfig : twoFactorAuthSettings.getProviders()) { twoFactorAuthService.checkProvider(tenantId, providerConfig.getProviderType()); } - if (twoFactorAuthSettings.isEnforceTwoFa() && twoFactorAuthSettings.getProviders().isEmpty()) { - throw new DataValidationException("At least one 2FA provider is required if enforce enabled!"); + if (tenantId.isSysTenantId() && twoFactorAuthSettings.isEnforceTwoFa() && twoFactorAuthSettings.getProviders().isEmpty()) { + throw new DataValidationException("At least one 2FA provider is required if enforcing is enabled"); } AdminSettings settings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(tenantId, TWO_FACTOR_AUTH_SETTINGS_KEY)) .orElseGet(() -> { diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java index 651067f045..dc753400e4 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java @@ -43,7 +43,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.settings.SecuritySettingsService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.security.auth.ForceMfaAuthenticationToken; +import org.thingsboard.server.service.security.auth.MfaConfigurationToken; import org.thingsboard.server.service.security.auth.MfaAuthenticationToken; import org.thingsboard.server.service.security.auth.mfa.TwoFactorAuthService; import org.thingsboard.server.service.security.exception.UserPasswordNotValidException; @@ -83,11 +83,10 @@ public class RestAuthenticationProvider implements AuthenticationProvider { Assert.notNull(authentication, "No authentication data provided"); Object principal = authentication.getPrincipal(); - if (!(principal instanceof UserPrincipal)) { + if (!(principal instanceof UserPrincipal userPrincipal)) { throw new BadCredentialsException("Authentication Failed. Bad user principal."); } - UserPrincipal userPrincipal = (UserPrincipal) principal; SecurityUser securityUser; if (userPrincipal.getType() == UserPrincipal.Type.USER_NAME) { String username = userPrincipal.getValue(); @@ -106,8 +105,8 @@ public class RestAuthenticationProvider implements AuthenticationProvider { securityUser = authenticateByUsernameAndPassword(authentication, userPrincipal, username, password); if (twoFactorAuthService.isTwoFaEnabled(securityUser.getTenantId(), securityUser.getId())) { return new MfaAuthenticationToken(securityUser); - } else if (twoFactorAuthService.isEnforceTwoFaEnabled(securityUser.getTenantId())) { - return new ForceMfaAuthenticationToken(securityUser); + } else if (twoFactorAuthService.isEnforceTwoFaEnabled(securityUser.getTenantId(), securityUser)) { + return new MfaConfigurationToken(securityUser); } else { systemSecurityService.logLoginAction(securityUser, authentication.getDetails(), ActionType.LOGIN, null); } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java index 9b60f29d0b..a21aab3f66 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java @@ -15,11 +15,11 @@ */ package org.thingsboard.server.service.security.auth.rest; -import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; import jakarta.servlet.http.HttpSession; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.security.core.Authentication; @@ -29,8 +29,8 @@ import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.model.JwtPair; -import org.thingsboard.server.service.security.auth.ForceMfaAuthenticationToken; import org.thingsboard.server.service.security.auth.MfaAuthenticationToken; +import org.thingsboard.server.service.security.auth.MfaConfigurationToken; import org.thingsboard.server.service.security.auth.mfa.config.TwoFaConfigManager; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; @@ -39,7 +39,7 @@ import java.io.IOException; import java.util.Optional; import java.util.concurrent.TimeUnit; -@Component(value = "defaultAuthenticationSuccessHandler") +@Slf4j @Component(value = "defaultAuthenticationSuccessHandler") @RequiredArgsConstructor public class RestAwareAuthenticationSuccessHandler implements AuthenticationSuccessHandler { private final JwtTokenFactory tokenFactory; @@ -47,15 +47,14 @@ public class RestAwareAuthenticationSuccessHandler implements AuthenticationSucc @Override public void onAuthenticationSuccess(HttpServletRequest request, HttpServletResponse response, - Authentication authentication) throws IOException, ServletException { + Authentication authentication) throws IOException { SecurityUser securityUser = (SecurityUser) authentication.getPrincipal(); JwtPair tokenPair; if (authentication instanceof MfaAuthenticationToken) { tokenPair = createMfaTokenPair(securityUser, Authority.PRE_VERIFICATION_TOKEN); - } - else if (authentication instanceof ForceMfaAuthenticationToken) { - tokenPair = createMfaTokenPair(securityUser, Authority.ENFORCE_MFA_TOKEN); + } else if (authentication instanceof MfaConfigurationToken) { + tokenPair = createMfaTokenPair(securityUser, Authority.MFA_CONFIGURATION_TOKEN); } else { tokenPair = tokenFactory.createTokenPair(securityUser); } @@ -68,6 +67,7 @@ public class RestAwareAuthenticationSuccessHandler implements AuthenticationSucc } public JwtPair createMfaTokenPair(SecurityUser securityUser, Authority scope) { + log.debug("[{}][{}] Creating {} token", securityUser.getTenantId(), securityUser.getId(), scope); JwtPair tokenPair = new JwtPair(); int preVerificationTokenLifetime = twoFaConfigManager.getPlatformTwoFaSettings(securityUser.getTenantId(), true) .flatMap(settings -> Optional.ofNullable(settings.getTotalAllowedTimeForVerification()) diff --git a/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java b/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java index fb38e251d7..9dac000d4b 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java @@ -136,7 +136,7 @@ public class JwtTokenFactory { } UserPrincipal principal; - if (authority != Authority.PRE_VERIFICATION_TOKEN && authority != Authority.ENFORCE_MFA_TOKEN) { + if (authority != Authority.PRE_VERIFICATION_TOKEN && authority != Authority.MFA_CONFIGURATION_TOKEN) { securityUser.setFirstName(claims.get(FIRST_NAME, String.class)); securityUser.setLastName(claims.get(LAST_NAME, String.class)); securityUser.setEnabled(claims.get(ENABLED, Boolean.class)); diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index b93ff0f623..43cab3ca3f 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -138,6 +138,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; +import org.thingsboard.server.common.data.security.model.JwtPair; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.msg.session.FeatureType; @@ -203,7 +204,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { protected static final String TENANT_ADMIN_PASSWORD = "tenant"; protected static final String DIFFERENT_TENANT_ADMIN_EMAIL = "testdifftenant@thingsboard.org"; - private static final String DIFFERENT_TENANT_ADMIN_PASSWORD = "difftenant"; + protected static final String DIFFERENT_TENANT_ADMIN_PASSWORD = "difftenant"; protected static final String CUSTOMER_USER_EMAIL = "testcustomer@thingsboard.org"; private static final String CUSTOMER_USER_PASSWORD = "customer"; @@ -596,8 +597,13 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { Assert.assertNotNull(tokenInfo); Assert.assertTrue(tokenInfo.has("token")); Assert.assertTrue(tokenInfo.has("refreshToken")); - String token = tokenInfo.get("token").asText(); - String refreshToken = tokenInfo.get("refreshToken").asText(); + validateAndSetJwtToken(JacksonUtil.treeToValue(tokenInfo, JwtPair.class), username); + } + + protected void validateAndSetJwtToken(JwtPair jwtPair, String username) { + Assert.assertNotNull(jwtPair); + String token = jwtPair.getToken(); + String refreshToken = jwtPair.getRefreshToken(); validateJwtToken(token, username); validateJwtToken(refreshToken, username); this.token = token; diff --git a/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java b/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java index c5aea9a773..ca578f3830 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.audit.AuditLog; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.notification.targets.platform.TenantAdministratorsFilter; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.page.TimePageLink; @@ -59,6 +60,7 @@ import java.time.Duration; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -67,10 +69,6 @@ import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -121,7 +119,7 @@ public class TwoFactorAuthTest extends AbstractControllerTest { public void testTwoFa_totp() throws Exception { TotpTwoFaAccountConfig totpTwoFaAccountConfig = configureTotpTwoFa(); - logInWithPreVerificationToken(username, password); + logInWithMfaToken(username, password, Authority.PRE_VERIFICATION_TOKEN); doPost("/api/auth/2fa/verification/send?providerType=TOTP") .andExpect(status().isOk()); @@ -141,7 +139,7 @@ public class TwoFactorAuthTest extends AbstractControllerTest { public void testTwoFa_sms() throws Exception { configureSmsTwoFa(); - logInWithPreVerificationToken(username, password); + logInWithMfaToken(username, password, Authority.PRE_VERIFICATION_TOKEN); doPost("/api/auth/2fa/verification/send?providerType=SMS") .andExpect(status().isOk()); @@ -165,7 +163,7 @@ public class TwoFactorAuthTest extends AbstractControllerTest { twoFaSettings.setTotalAllowedTimeForVerification(65); }); - logInWithPreVerificationToken(username, password); + logInWithMfaToken(username, password, Authority.PRE_VERIFICATION_TOKEN); await("expiration of the pre-verification token") .atLeast(Duration.ofSeconds(30).plusMillis(500)) @@ -182,7 +180,7 @@ public class TwoFactorAuthTest extends AbstractControllerTest { twoFaSettings.setMaxVerificationFailuresBeforeUserLockout(10); }); - logInWithPreVerificationToken(username, password); + logInWithMfaToken(username, password, Authority.PRE_VERIFICATION_TOKEN); Stream.generate(() -> StringUtils.randomNumeric(6)) .limit(9) @@ -211,7 +209,7 @@ public class TwoFactorAuthTest extends AbstractControllerTest { twoFaSettings.setMinVerificationCodeSendPeriod(10); }); - logInWithPreVerificationToken(username, password); + logInWithMfaToken(username, password, Authority.PRE_VERIFICATION_TOKEN); doPost("/api/auth/2fa/verification/send?providerType=TOTP") .andExpect(status().isOk()); @@ -235,7 +233,7 @@ public class TwoFactorAuthTest extends AbstractControllerTest { twoFaSettings.setVerificationCodeCheckRateLimit("3:10"); }); - logInWithPreVerificationToken(username, password); + logInWithMfaToken(username, password, Authority.PRE_VERIFICATION_TOKEN); for (int i = 0; i < 3; i++) { String incorrectVerificationCodeError = getErrorMessage(doPost("/api/auth/2fa/verification/check?providerType=TOTP&verificationCode=incorrect") @@ -263,7 +261,7 @@ public class TwoFactorAuthTest extends AbstractControllerTest { @Test public void testCheckVerificationCode_invalidVerificationCode() throws Exception { configureTotpTwoFa(); - logInWithPreVerificationToken(username, password); + logInWithMfaToken(username, password, Authority.PRE_VERIFICATION_TOKEN); for (String invalidVerificationCode : new String[]{"1234567", "ab1212", "12311 ", "oewkriwejqf"}) { String errorMessage = getErrorMessage(doPost("/api/auth/2fa/verification/check?providerType=TOTP&verificationCode=" + invalidVerificationCode) @@ -278,7 +276,7 @@ public class TwoFactorAuthTest extends AbstractControllerTest { smsTwoFaProviderConfig.setVerificationCodeLifetime(10); }); - logInWithPreVerificationToken(username, password); + logInWithMfaToken(username, password, Authority.PRE_VERIFICATION_TOKEN); ArgumentCaptor verificationCodeCaptor = ArgumentCaptor.forClass(String.class); doPost("/api/auth/2fa/verification/send?providerType=SMS").andExpect(status().isOk()); @@ -301,7 +299,7 @@ public class TwoFactorAuthTest extends AbstractControllerTest { public void testTwoFa_logLoginAction() throws Exception { TotpTwoFaAccountConfig totpTwoFaAccountConfig = configureTotpTwoFa(); - logInWithPreVerificationToken(username, password); + logInWithMfaToken(username, password, Authority.PRE_VERIFICATION_TOKEN); await("async audit log saving").during(1, TimeUnit.SECONDS); doPost("/api/auth/2fa/verification/check?providerType=TOTP&verificationCode=incorrect") @@ -383,7 +381,7 @@ public class TwoFactorAuthTest extends AbstractControllerTest { emailTwoFaAccountConfig.setEmail(twoFaUser.getEmail()); twoFaConfigManager.saveTwoFaAccountConfig(tenantId, twoFaUser.getId(), emailTwoFaAccountConfig); - logInWithPreVerificationToken(twoFaUser.getEmail(), "12345678"); + logInWithMfaToken(twoFaUser.getEmail(), "12345678", Authority.PRE_VERIFICATION_TOKEN); Map providersInfos = readResponse(doGet("/api/auth/2fa/providers").andExpect(status().isOk()), new TypeReference>() {}).stream() .collect(Collectors.toMap(TwoFactorAuthController.TwoFaProviderInfo::getType, v -> v)); @@ -401,7 +399,7 @@ public class TwoFactorAuthTest extends AbstractControllerTest { } @Test - public void testEnforceTwoFactorSetting() throws Exception { + public void testEnforceTwoFa() throws Exception { TotpTwoFaProviderConfig totpTwoFaProviderConfig = new TotpTwoFaProviderConfig(); totpTwoFaProviderConfig.setIssuerName("tb"); @@ -410,14 +408,13 @@ public class TwoFactorAuthTest extends AbstractControllerTest { twoFaSettings.setMinVerificationCodeSendPeriod(5); twoFaSettings.setTotalAllowedTimeForVerification(100); twoFaSettings.setEnforceTwoFa(true); + TenantAdministratorsFilter enforcedUsersFilter = new TenantAdministratorsFilter(); + enforcedUsersFilter.setTenantsIds(Set.of(tenantId.getId())); + twoFaSettings.setEnforcedUsersFilter(enforcedUsersFilter); twoFaSettings = twoFaConfigManager.savePlatformTwoFaSettings(TenantId.SYS_TENANT_ID, twoFaSettings); - JsonNode node = readResponse(doPost("/api/auth/login", new LoginRequest(username, password)).andExpect(status().isOk()), JsonNode.class); - assertNotNull(node.get("token").asText()); - assertNull(node.get("refreshToken")); - assertEquals(node.get("scope").asText(), Authority.ENFORCE_MFA_TOKEN.name()); + logInWithMfaToken(username, password, Authority.MFA_CONFIGURATION_TOKEN); - this.token = node.get("token").asText(); TotpTwoFaAccountConfig totpTwoFaAccountConfig = (TotpTwoFaAccountConfig) twoFactorAuthService.generateNewAccountConfig(user, totpTwoFaProviderConfig.getProviderType()); String secret = UriComponentsBuilder.fromUriString(totpTwoFaAccountConfig.getAuthUrl()).build() .getQueryParams().getFirst("secret"); @@ -425,24 +422,27 @@ public class TwoFactorAuthTest extends AbstractControllerTest { readResponse(doPost("/api/2fa/account/config?verificationCode=" + verificationCode, totpTwoFaAccountConfig).andExpect(status().isOk()), JsonNode.class); JwtPair tokenPair = readResponse(doPost("/api/auth/2fa/login").andExpect(status().isOk()), JwtPair.class); - assertNotNull(tokenPair); + assertThat(tokenPair.getToken()).isNotEmpty(); + assertThat(tokenPair.getRefreshToken()).isNotEmpty(); + validateAndSetJwtToken(tokenPair, username); - this.token = tokenPair.getToken(); - this.refreshToken = tokenPair.getRefreshToken(); + doGet("/api/user/" + user.getId()).andExpect(status().isOk()); + // verifying enforced users filter + createDifferentTenant(); doGet("/api/user/" + user.getId()).andExpect(status().isOk()); twoFaSettings.setEnforceTwoFa(false); twoFaConfigManager.savePlatformTwoFaSettings(TenantId.SYS_TENANT_ID, twoFaSettings); } - private void logInWithPreVerificationToken(String username, String password) throws Exception { + private void logInWithMfaToken(String username, String password, Authority expectedScope) throws Exception { LoginRequest loginRequest = new LoginRequest(username, password); JwtPair response = readResponse(doPost("/api/auth/login", loginRequest).andExpect(status().isOk()), JwtPair.class); assertThat(response.getToken()).isNotNull(); assertThat(response.getRefreshToken()).isNull(); - assertThat(response.getScope()).isEqualTo(Authority.PRE_VERIFICATION_TOKEN); + assertThat(response.getScope()).isEqualTo(expectedScope); this.token = response.getToken(); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TbTenantProfileCache.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TbTenantProfileCache.java index 8acfbca542..273b52810a 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TbTenantProfileCache.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TbTenantProfileCache.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.dao.tenant; -import org.thingsboard.server.common.data.SystemParams; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java index c016631064..654de3edae 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java @@ -23,6 +23,8 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.id.UserCredentialsId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.mobile.MobileSessionInfo; +import org.thingsboard.server.common.data.notification.targets.platform.SystemLevelUsersFilter; +import org.thingsboard.server.common.data.notification.targets.platform.UsersFilter; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.UserCredentials; @@ -109,4 +111,8 @@ public interface UserService extends EntityDaoService { void removeMobileSession(TenantId tenantId, String mobileToken); + PageData findUsersByFilter(TenantId tenantId, UsersFilter filter, PageLink pageLink); + + boolean matchesFilter(TenantId tenantId, SystemLevelUsersFilter filter, User user); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/targets/platform/AllUsersFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/targets/platform/AllUsersFilter.java index 79c7dafb1d..0d629fe970 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/targets/platform/AllUsersFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/targets/platform/AllUsersFilter.java @@ -18,7 +18,7 @@ package org.thingsboard.server.common.data.notification.targets.platform; import lombok.Data; @Data -public class AllUsersFilter implements UsersFilter { +public class AllUsersFilter implements SystemLevelUsersFilter { @Override public UsersFilterType getType() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/targets/platform/SystemAdministratorsFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/targets/platform/SystemAdministratorsFilter.java index c178ee1159..4d35488ec1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/targets/platform/SystemAdministratorsFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/targets/platform/SystemAdministratorsFilter.java @@ -18,7 +18,7 @@ package org.thingsboard.server.common.data.notification.targets.platform; import lombok.Data; @Data -public class SystemAdministratorsFilter implements UsersFilter { +public class SystemAdministratorsFilter implements SystemLevelUsersFilter { @Override public UsersFilterType getType() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/targets/platform/SystemLevelUsersFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/targets/platform/SystemLevelUsersFilter.java new file mode 100644 index 0000000000..20233f7e49 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/targets/platform/SystemLevelUsersFilter.java @@ -0,0 +1,19 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.notification.targets.platform; + +public interface SystemLevelUsersFilter extends UsersFilter { +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/targets/platform/TenantAdministratorsFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/targets/platform/TenantAdministratorsFilter.java index 1f78790788..2f72e66076 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/targets/platform/TenantAdministratorsFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/targets/platform/TenantAdministratorsFilter.java @@ -21,7 +21,7 @@ import java.util.Set; import java.util.UUID; @Data -public class TenantAdministratorsFilter implements UsersFilter { +public class TenantAdministratorsFilter implements SystemLevelUsersFilter { private Set tenantsIds; private Set tenantProfilesIds; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/security/Authority.java b/common/data/src/main/java/org/thingsboard/server/common/data/security/Authority.java index 532bf13e55..deaaf263ca 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/security/Authority.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/security/Authority.java @@ -20,9 +20,10 @@ public enum Authority { SYS_ADMIN(0), TENANT_ADMIN(1), CUSTOMER_USER(2), + REFRESH_TOKEN(10), PRE_VERIFICATION_TOKEN(11), - ENFORCE_MFA_TOKEN(12); + MFA_CONFIGURATION_TOKEN(12); private int code; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/security/model/mfa/PlatformTwoFaSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/security/model/mfa/PlatformTwoFaSettings.java index 6d9eded2ae..c1c632f016 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/security/model/mfa/PlatformTwoFaSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/security/model/mfa/PlatformTwoFaSettings.java @@ -21,6 +21,7 @@ import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.Pattern; import lombok.Data; +import org.thingsboard.server.common.data.notification.targets.platform.SystemLevelUsersFilter; import org.thingsboard.server.common.data.security.model.mfa.provider.TwoFaProviderConfig; import org.thingsboard.server.common.data.security.model.mfa.provider.TwoFaProviderType; @@ -47,6 +48,7 @@ public class PlatformTwoFaSettings { private Integer totalAllowedTimeForVerification; private boolean enforceTwoFa; + private SystemLevelUsersFilter enforcedUsersFilter; public Optional getProviderConfig(TwoFaProviderType providerType) { return Optional.ofNullable(providers) diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTargetService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTargetService.java index e873b18c76..1c947552ce 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTargetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTargetService.java @@ -25,17 +25,13 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.NotificationTargetId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.notification.NotificationRequestStatus; import org.thingsboard.server.common.data.notification.NotificationType; import org.thingsboard.server.common.data.notification.info.RuleOriginatedNotificationInfo; import org.thingsboard.server.common.data.notification.targets.NotificationTarget; import org.thingsboard.server.common.data.notification.targets.NotificationTargetConfig; -import org.thingsboard.server.common.data.notification.targets.platform.CustomerUsersFilter; import org.thingsboard.server.common.data.notification.targets.platform.PlatformUsersNotificationTargetConfig; -import org.thingsboard.server.common.data.notification.targets.platform.TenantAdministratorsFilter; -import org.thingsboard.server.common.data.notification.targets.platform.UserListFilter; import org.thingsboard.server.common.data.notification.targets.platform.UsersFilter; import org.thingsboard.server.common.data.notification.targets.platform.UsersFilterType; import org.thingsboard.server.common.data.page.PageData; @@ -50,9 +46,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.stream.Collectors; - -import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; @Service @Slf4j @@ -115,49 +108,7 @@ public class DefaultNotificationTargetService extends AbstractEntityService impl @Override public PageData findRecipientsForNotificationTargetConfig(TenantId tenantId, PlatformUsersNotificationTargetConfig targetConfig, PageLink pageLink) { UsersFilter usersFilter = targetConfig.getUsersFilter(); - switch (usersFilter.getType()) { - case USER_LIST: { - List users = ((UserListFilter) usersFilter).getUsersIds().stream() - .limit(pageLink.getPageSize()) - .map(UserId::new).map(userId -> userService.findUserById(tenantId, userId)) - .filter(Objects::nonNull).collect(Collectors.toList()); - return new PageData<>(users, 1, users.size(), false); - } - case CUSTOMER_USERS: { - if (tenantId.equals(TenantId.SYS_TENANT_ID)) { - throw new IllegalArgumentException("Customer users target is not supported for system administrator"); - } - CustomerUsersFilter filter = (CustomerUsersFilter) usersFilter; - return userService.findCustomerUsers(tenantId, new CustomerId(filter.getCustomerId()), pageLink); - } - case TENANT_ADMINISTRATORS: { - TenantAdministratorsFilter filter = (TenantAdministratorsFilter) usersFilter; - if (!tenantId.equals(TenantId.SYS_TENANT_ID)) { - return userService.findTenantAdmins(tenantId, pageLink); - } else { - if (isNotEmpty(filter.getTenantsIds())) { - return userService.findTenantAdminsByTenantsIds(filter.getTenantsIds().stream() - .map(TenantId::fromUUID).collect(Collectors.toList()), pageLink); - } else if (isNotEmpty(filter.getTenantProfilesIds())) { - return userService.findTenantAdminsByTenantProfilesIds(filter.getTenantProfilesIds().stream() - .map(TenantProfileId::new).collect(Collectors.toList()), pageLink); - } else { - return userService.findAllTenantAdmins(pageLink); - } - } - } - case SYSTEM_ADMINISTRATORS: - return userService.findSysAdmins(pageLink); - case ALL_USERS: { - if (!tenantId.equals(TenantId.SYS_TENANT_ID)) { - return userService.findUsersByTenantId(tenantId, pageLink); - } else { - return userService.findAllUsers(pageLink); - } - } - default: - throw new IllegalArgumentException("Recipient type not supported"); - } + return userService.findUsersByFilter(tenantId, usersFilter, pageLink); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index b9d09e55ba..338ca9594e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -78,6 +78,7 @@ public class TenantServiceImpl extends AbstractCachedEntityService userValidator; private final DataValidator userCredentialsValidator; private final ApplicationEventPublisher eventPublisher; @@ -496,6 +505,77 @@ public class UserServiceImpl extends AbstractCachedEntityService findUsersByFilter(TenantId tenantId, UsersFilter filter, PageLink pageLink) { + switch (filter.getType()) { + case USER_LIST -> { + List users = ((UserListFilter) filter).getUsersIds().stream() + .limit(pageLink.getPageSize()) + .map(UserId::new).map(userId -> findUserById(tenantId, userId)) + .filter(Objects::nonNull).collect(Collectors.toList()); + return new PageData<>(users, 1, users.size(), false); + } + case CUSTOMER_USERS -> { + if (tenantId.equals(TenantId.SYS_TENANT_ID)) { + throw new IllegalArgumentException("Customer users target is not supported for system administrator"); + } + CustomerUsersFilter customerUsersFilter = (CustomerUsersFilter) filter; + return findCustomerUsers(tenantId, new CustomerId(customerUsersFilter.getCustomerId()), pageLink); + } + case TENANT_ADMINISTRATORS -> { + TenantAdministratorsFilter tenantAdministratorsFilter = (TenantAdministratorsFilter) filter; + if (!tenantId.equals(TenantId.SYS_TENANT_ID)) { + return findTenantAdmins(tenantId, pageLink); + } else { + if (isNotEmpty(tenantAdministratorsFilter.getTenantsIds())) { + return findTenantAdminsByTenantsIds(tenantAdministratorsFilter.getTenantsIds().stream() + .map(TenantId::fromUUID).collect(Collectors.toList()), pageLink); + } else if (isNotEmpty(tenantAdministratorsFilter.getTenantProfilesIds())) { + return findTenantAdminsByTenantProfilesIds(tenantAdministratorsFilter.getTenantProfilesIds().stream() + .map(TenantProfileId::new).collect(Collectors.toList()), pageLink); + } else { + return findAllTenantAdmins(pageLink); + } + } + } + case SYSTEM_ADMINISTRATORS -> { + return findSysAdmins(pageLink); + } + case ALL_USERS -> { + if (!tenantId.equals(TenantId.SYS_TENANT_ID)) { + return findUsersByTenantId(tenantId, pageLink); + } else { + return findAllUsers(pageLink); + } + } + default -> throw new IllegalArgumentException("Recipient type not supported"); + } + } + + @Override + public boolean matchesFilter(TenantId tenantId, SystemLevelUsersFilter filter, User user) { + switch (filter.getType()) { + case TENANT_ADMINISTRATORS -> { + TenantAdministratorsFilter tenantAdministratorsFilter = (TenantAdministratorsFilter) filter; + if (isNotEmpty(tenantAdministratorsFilter.getTenantsIds())) { + return tenantAdministratorsFilter.getTenantsIds().contains(user.getTenantId().getId()); + } else if (isNotEmpty(tenantAdministratorsFilter.getTenantProfilesIds())) { + return tenantAdministratorsFilter.getTenantProfilesIds().contains(tenantProfileCache.get(user.getTenantId()).getUuidId()); + } else { + return user.getAuthority() == Authority.TENANT_ADMIN; + } + } + case SYSTEM_ADMINISTRATORS -> { + return user.getAuthority() == Authority.SYS_ADMIN; + } + case ALL_USERS -> { + return true; + } + default -> throw new IllegalArgumentException("Recipient type not supported"); + } + + } + private void updatePasswordHistory(UserCredentials userCredentials) { JsonNode additionalInfo = userCredentials.getAdditionalInfo(); if (!(additionalInfo instanceof ObjectNode)) { From 85804837db28c9432082f0f221e83e6834203184 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 24 Jun 2025 11:57:19 +0300 Subject: [PATCH 03/88] 2FA enforcement: add more validation, fix test --- .../mfa/config/DefaultTwoFaConfigManager.java | 15 +++++++++++++-- .../server/controller/TwoFactorAuthTest.java | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java index f3afc7fdc2..24b3d59440 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java @@ -167,9 +167,20 @@ public class DefaultTwoFaConfigManager implements TwoFaConfigManager { for (TwoFaProviderConfig providerConfig : twoFactorAuthSettings.getProviders()) { twoFactorAuthService.checkProvider(tenantId, providerConfig.getProviderType()); } - if (tenantId.isSysTenantId() && twoFactorAuthSettings.isEnforceTwoFa() && twoFactorAuthSettings.getProviders().isEmpty()) { - throw new DataValidationException("At least one 2FA provider is required if enforcing is enabled"); + if (tenantId.isSysTenantId()) { + if (twoFactorAuthSettings.isEnforceTwoFa()) { + if (twoFactorAuthSettings.getProviders().isEmpty()) { + throw new DataValidationException("At least one 2FA provider is required if enforcing is enabled"); + } + if (twoFactorAuthSettings.getEnforcedUsersFilter() == null) { + throw new DataValidationException("Users filter to enforce 2FA for is required"); + } + } + } else { + twoFactorAuthSettings.setEnforceTwoFa(false); + twoFactorAuthSettings.setEnforcedUsersFilter(null); } + AdminSettings settings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(tenantId, TWO_FACTOR_AUTH_SETTINGS_KEY)) .orElseGet(() -> { AdminSettings newSettings = new AdminSettings(); diff --git a/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java b/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java index ca578f3830..0da218db67 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java @@ -430,7 +430,7 @@ public class TwoFactorAuthTest extends AbstractControllerTest { // verifying enforced users filter createDifferentTenant(); - doGet("/api/user/" + user.getId()).andExpect(status().isOk()); + doGet("/api/user/" + savedDifferentTenantUser.getId()).andExpect(status().isOk()); twoFaSettings.setEnforceTwoFa(false); twoFaConfigManager.savePlatformTwoFaSettings(TenantId.SYS_TENANT_ID, twoFaSettings); From c4c3d255b0e04f2b8eceaa06d26f9ebeb788b35e Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 24 Jun 2025 13:08:02 +0300 Subject: [PATCH 04/88] Add validation for 2FA account settings when enforcement is enabled --- .../TwoFactorAuthConfigController.java | 30 ++-- .../controller/TwoFactorAuthController.java | 14 +- .../security/auth/AuthExceptionHandler.java | 13 ++ .../auth/mfa/DefaultTwoFactorAuthService.java | 8 +- .../auth/mfa/TwoFactorAuthService.java | 3 +- .../mfa/config/DefaultTwoFaConfigManager.java | 35 +++-- .../auth/mfa/config/TwoFaConfigManager.java | 10 +- .../impl/BackupCodeTwoFaProvider.java | 2 +- .../auth/rest/RestAuthenticationProvider.java | 4 +- .../controller/TwoFactorAuthConfigTest.java | 135 ++++++++++-------- .../server/controller/TwoFactorAuthTest.java | 15 +- 11 files changed, 153 insertions(+), 116 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthConfigController.java b/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthConfigController.java index fd78e5c945..00829df047 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthConfigController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthConfigController.java @@ -69,7 +69,7 @@ public class TwoFactorAuthConfigController extends BaseController { @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") public AccountTwoFaSettings getAccountTwoFaSettings() throws ThingsboardException { SecurityUser user = getCurrentUser(); - return twoFaConfigManager.getAccountTwoFaSettings(user.getTenantId(), user.getId()).orElse(null); + return twoFaConfigManager.getAccountTwoFaSettings(user.getTenantId(), user).orElse(null); } @ApiOperation(value = "Generate 2FA account config (generateTwoFaAccountConfig)", @@ -141,7 +141,7 @@ public class TwoFactorAuthConfigController extends BaseController { public AccountTwoFaSettings verifyAndSaveTwoFaAccountConfig(@Valid @RequestBody TwoFaAccountConfig accountConfig, @RequestParam(required = false) String verificationCode) throws Exception { SecurityUser user = getCurrentUser(); - if (twoFaConfigManager.getTwoFaAccountConfig(user.getTenantId(), user.getId(), accountConfig.getProviderType()).isPresent()) { + if (twoFaConfigManager.getTwoFaAccountConfig(user.getTenantId(), user, accountConfig.getProviderType()).isPresent()) { throw new IllegalArgumentException("2FA provider is already configured"); } @@ -152,7 +152,7 @@ public class TwoFactorAuthConfigController extends BaseController { verificationSuccess = true; } if (verificationSuccess) { - return twoFaConfigManager.saveTwoFaAccountConfig(user.getTenantId(), user.getId(), accountConfig); + return twoFaConfigManager.saveTwoFaAccountConfig(user.getTenantId(), user, accountConfig); } else { throw new IllegalArgumentException("Verification code is incorrect"); } @@ -160,38 +160,38 @@ public class TwoFactorAuthConfigController extends BaseController { @ApiOperation(value = "Update 2FA account config (updateTwoFaAccountConfig)", notes = "Update config for a given provider type. \n" + - "Update request example:\n" + - "```\n{\n \"useByDefault\": true\n}\n```\n" + - "Returns whole account's 2FA settings object.\n" + - ControllerConstants.AVAILABLE_FOR_ANY_AUTHORIZED_USER) + "Update request example:\n" + + "```\n{\n \"useByDefault\": true\n}\n```\n" + + "Returns whole account's 2FA settings object.\n" + + ControllerConstants.AVAILABLE_FOR_ANY_AUTHORIZED_USER) @PutMapping("/account/config") @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") public AccountTwoFaSettings updateTwoFaAccountConfig(@RequestParam TwoFaProviderType providerType, @RequestBody TwoFaAccountConfigUpdateRequest updateRequest) throws ThingsboardException { SecurityUser user = getCurrentUser(); - TwoFaAccountConfig accountConfig = twoFaConfigManager.getTwoFaAccountConfig(user.getTenantId(), user.getId(), providerType) + TwoFaAccountConfig accountConfig = twoFaConfigManager.getTwoFaAccountConfig(user.getTenantId(), user, providerType) .orElseThrow(() -> new IllegalArgumentException("Config for " + providerType + " 2FA provider not found")); accountConfig.setUseByDefault(updateRequest.isUseByDefault()); - return twoFaConfigManager.saveTwoFaAccountConfig(user.getTenantId(), user.getId(), accountConfig); + return twoFaConfigManager.saveTwoFaAccountConfig(user.getTenantId(), user, accountConfig); } @ApiOperation(value = "Delete 2FA account config (deleteTwoFaAccountConfig)", notes = "Delete 2FA config for a given 2FA provider type. \n" + - "Returns whole account's 2FA settings object.\n" + - ControllerConstants.AVAILABLE_FOR_ANY_AUTHORIZED_USER) + "Returns whole account's 2FA settings object.\n" + + ControllerConstants.AVAILABLE_FOR_ANY_AUTHORIZED_USER) @DeleteMapping("/account/config") @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") public AccountTwoFaSettings deleteTwoFaAccountConfig(@RequestParam TwoFaProviderType providerType) throws ThingsboardException { SecurityUser user = getCurrentUser(); - return twoFaConfigManager.deleteTwoFaAccountConfig(user.getTenantId(), user.getId(), providerType); + return twoFaConfigManager.deleteTwoFaAccountConfig(user.getTenantId(), user, providerType); } @ApiOperation(value = "Get available 2FA providers (getAvailableTwoFaProviders)", notes = "Get the list of provider types available for user to use (the ones configured by tenant or sysadmin).\n" + - "Example of response:\n" + - "```\n[\n \"TOTP\",\n \"EMAIL\",\n \"SMS\"\n]\n```" + - ControllerConstants.AVAILABLE_FOR_ANY_AUTHORIZED_USER + "Example of response:\n" + + "```\n[\n \"TOTP\",\n \"EMAIL\",\n \"SMS\"\n]\n```" + + ControllerConstants.AVAILABLE_FOR_ANY_AUTHORIZED_USER ) @GetMapping("/providers") @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER', 'MFA_CONFIGURATION_TOKEN')") diff --git a/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthController.java b/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthController.java index ec5bf0e054..0d2af1a4f1 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthController.java @@ -101,17 +101,17 @@ public class TwoFactorAuthController extends BaseController { @ApiOperation(value = "Get available 2FA providers (getAvailableTwoFaProviders)", notes = "Get the list of 2FA provider infos available for user to use. Example:\n" + - "```\n[\n" + - " {\n \"type\": \"EMAIL\",\n \"default\": true,\n \"contact\": \"ab*****ko@gmail.com\"\n },\n" + - " {\n \"type\": \"TOTP\",\n \"default\": false,\n \"contact\": null\n },\n" + - " {\n \"type\": \"SMS\",\n \"default\": false,\n \"contact\": \"+38********12\"\n }\n" + - "]\n```") + "```\n[\n" + + " {\n \"type\": \"EMAIL\",\n \"default\": true,\n \"contact\": \"ab*****ko@gmail.com\"\n },\n" + + " {\n \"type\": \"TOTP\",\n \"default\": false,\n \"contact\": null\n },\n" + + " {\n \"type\": \"SMS\",\n \"default\": false,\n \"contact\": \"+38********12\"\n }\n" + + "]\n```") @GetMapping("/providers") @PreAuthorize("hasAuthority('PRE_VERIFICATION_TOKEN')") public List getAvailableTwoFaProviders() throws ThingsboardException { SecurityUser user = getCurrentUser(); Optional platformTwoFaSettings = twoFaConfigManager.getPlatformTwoFaSettings(user.getTenantId(), true); - return twoFaConfigManager.getAccountTwoFaSettings(user.getTenantId(), user.getId()) + return twoFaConfigManager.getAccountTwoFaSettings(user.getTenantId(), user) .map(settings -> settings.getConfigs().values()).orElse(Collections.emptyList()) .stream().map(config -> { String contact = null; @@ -141,7 +141,7 @@ public class TwoFactorAuthController extends BaseController { @PreAuthorize("hasAuthority('MFA_CONFIGURATION_TOKEN')") public JwtPair authenticateByTwoFaConfigurationToken(HttpServletRequest servletRequest) throws ThingsboardException { SecurityUser user = getCurrentUser(); - if (twoFactorAuthService.isTwoFaEnabled(user.getTenantId(), user.getId())) { + if (twoFactorAuthService.isTwoFaEnabled(user.getTenantId(), user)) { logLogInAction(servletRequest, user, null); return createTokenPair(user); } else { diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/AuthExceptionHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/AuthExceptionHandler.java index 27ed2996d1..ff39038453 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/AuthExceptionHandler.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/AuthExceptionHandler.java @@ -18,8 +18,10 @@ package org.thingsboard.server.service.security.auth; import jakarta.servlet.FilterChain; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; +import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.security.core.AuthenticationException; import org.springframework.stereotype.Component; import org.springframework.web.filter.OncePerRequestFilter; @@ -32,6 +34,10 @@ public class AuthExceptionHandler extends OncePerRequestFilter { private final ThingsboardErrorResponseHandler errorResponseHandler; + @Value("${server.log_controller_error_stack_trace}") + @Getter + private boolean logControllerErrorStackTrace; + @Override protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) { try { @@ -39,8 +45,15 @@ public class AuthExceptionHandler extends OncePerRequestFilter { } catch (AuthenticationException e) { throw e; } catch (Exception e) { + log(e); errorResponseHandler.handle(e, response); } } + private void log(Exception e) { + if (logControllerErrorStackTrace) { + log.error("Auth error", e); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/DefaultTwoFactorAuthService.java b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/DefaultTwoFactorAuthService.java index c4683f5822..325203314f 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/DefaultTwoFactorAuthService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/DefaultTwoFactorAuthService.java @@ -62,8 +62,8 @@ public class DefaultTwoFactorAuthService implements TwoFactorAuthService { private static final ThingsboardException TOO_MANY_REQUESTS_ERROR = new ThingsboardException("Too many requests", ThingsboardErrorCode.TOO_MANY_REQUESTS); @Override - public boolean isTwoFaEnabled(TenantId tenantId, UserId userId) { - return configManager.getAccountTwoFaSettings(tenantId, userId) + public boolean isTwoFaEnabled(TenantId tenantId, User user) { + return configManager.getAccountTwoFaSettings(tenantId, user) .map(settings -> !settings.getConfigs().isEmpty()) .orElse(false); } @@ -89,7 +89,7 @@ public class DefaultTwoFactorAuthService implements TwoFactorAuthService { @Override public void prepareVerificationCode(SecurityUser user, TwoFaProviderType providerType, boolean checkLimits) throws Exception { - TwoFaAccountConfig accountConfig = configManager.getTwoFaAccountConfig(user.getTenantId(), user.getId(), providerType) + TwoFaAccountConfig accountConfig = configManager.getTwoFaAccountConfig(user.getTenantId(), user, providerType) .orElseThrow(() -> ACCOUNT_NOT_CONFIGURED_ERROR); prepareVerificationCode(user, accountConfig, checkLimits); } @@ -118,7 +118,7 @@ public class DefaultTwoFactorAuthService implements TwoFactorAuthService { @Override public boolean checkVerificationCode(SecurityUser user, TwoFaProviderType providerType, String verificationCode, boolean checkLimits) throws ThingsboardException { - TwoFaAccountConfig accountConfig = configManager.getTwoFaAccountConfig(user.getTenantId(), user.getId(), providerType) + TwoFaAccountConfig accountConfig = configManager.getTwoFaAccountConfig(user.getTenantId(), user, providerType) .orElseThrow(() -> ACCOUNT_NOT_CONFIGURED_ERROR); return checkVerificationCode(user, verificationCode, accountConfig, checkLimits); } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/TwoFactorAuthService.java b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/TwoFactorAuthService.java index d77d8d65a0..62fdb973d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/TwoFactorAuthService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/TwoFactorAuthService.java @@ -18,14 +18,13 @@ package org.thingsboard.server.service.security.auth.mfa; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.model.mfa.account.TwoFaAccountConfig; import org.thingsboard.server.common.data.security.model.mfa.provider.TwoFaProviderType; import org.thingsboard.server.service.security.model.SecurityUser; public interface TwoFactorAuthService { - boolean isTwoFaEnabled(TenantId tenantId, UserId userId); + boolean isTwoFaEnabled(TenantId tenantId, User user); boolean isEnforceTwoFaEnabled(TenantId tenantId, User user); diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java index 24b3d59440..ad04d3e962 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java @@ -21,9 +21,9 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.UserAuthSettings; import org.thingsboard.server.common.data.security.model.mfa.PlatformTwoFaSettings; import org.thingsboard.server.common.data.security.model.mfa.account.AccountTwoFaSettings; @@ -56,9 +56,9 @@ public class DefaultTwoFaConfigManager implements TwoFaConfigManager { @Override - public Optional getAccountTwoFaSettings(TenantId tenantId, UserId userId) { + public Optional getAccountTwoFaSettings(TenantId tenantId, User user) { PlatformTwoFaSettings platformTwoFaSettings = getPlatformTwoFaSettings(tenantId, true).orElse(null); - return Optional.ofNullable(userAuthSettingsDao.findByUserId(userId)) + return Optional.ofNullable(userAuthSettingsDao.findByUserId(user.getId())) .map(userAuthSettings -> { AccountTwoFaSettings twoFaSettings = userAuthSettings.getTwoFaSettings(); if (twoFaSettings == null) return null; @@ -80,17 +80,22 @@ public class DefaultTwoFaConfigManager implements TwoFaConfigManager { } if (updateNeeded) { - twoFaSettings = saveAccountTwoFaSettings(tenantId, userId, twoFaSettings); + twoFaSettings = saveAccountTwoFaSettings(tenantId, user, twoFaSettings); } return twoFaSettings; }); } - protected AccountTwoFaSettings saveAccountTwoFaSettings(TenantId tenantId, UserId userId, AccountTwoFaSettings settings) { - UserAuthSettings userAuthSettings = Optional.ofNullable(userAuthSettingsDao.findByUserId(userId)) + protected AccountTwoFaSettings saveAccountTwoFaSettings(TenantId tenantId, User user, AccountTwoFaSettings settings) { + if (settings.getConfigs().isEmpty()) { + if (twoFactorAuthService.isEnforceTwoFaEnabled(tenantId, user)) { + throw new DataValidationException("At least one 2FA provider is required"); + } + } + UserAuthSettings userAuthSettings = Optional.ofNullable(userAuthSettingsDao.findByUserId(user.getId())) .orElseGet(() -> { UserAuthSettings newUserAuthSettings = new UserAuthSettings(); - newUserAuthSettings.setUserId(userId); + newUserAuthSettings.setUserId(user.getId()); return newUserAuthSettings; }); userAuthSettings.setTwoFaSettings(settings); @@ -102,18 +107,18 @@ public class DefaultTwoFaConfigManager implements TwoFaConfigManager { @Override - public Optional getTwoFaAccountConfig(TenantId tenantId, UserId userId, TwoFaProviderType providerType) { - return getAccountTwoFaSettings(tenantId, userId) + public Optional getTwoFaAccountConfig(TenantId tenantId, User user, TwoFaProviderType providerType) { + return getAccountTwoFaSettings(tenantId, user) .map(AccountTwoFaSettings::getConfigs) .flatMap(configs -> Optional.ofNullable(configs.get(providerType))); } @Override - public AccountTwoFaSettings saveTwoFaAccountConfig(TenantId tenantId, UserId userId, TwoFaAccountConfig accountConfig) { + public AccountTwoFaSettings saveTwoFaAccountConfig(TenantId tenantId, User user, TwoFaAccountConfig accountConfig) { getTwoFaProviderConfig(tenantId, accountConfig.getProviderType()) .orElseThrow(() -> new IllegalArgumentException("2FA provider is not configured")); - AccountTwoFaSettings settings = getAccountTwoFaSettings(tenantId, userId).orElseGet(() -> { + AccountTwoFaSettings settings = getAccountTwoFaSettings(tenantId, user).orElseGet(() -> { AccountTwoFaSettings newSettings = new AccountTwoFaSettings(); newSettings.setConfigs(new LinkedHashMap<>()); return newSettings; @@ -129,12 +134,12 @@ public class DefaultTwoFaConfigManager implements TwoFaConfigManager { if (configs.values().stream().noneMatch(TwoFaAccountConfig::isUseByDefault)) { configs.values().stream().findFirst().ifPresent(config -> config.setUseByDefault(true)); } - return saveAccountTwoFaSettings(tenantId, userId, settings); + return saveAccountTwoFaSettings(tenantId, user, settings); } @Override - public AccountTwoFaSettings deleteTwoFaAccountConfig(TenantId tenantId, UserId userId, TwoFaProviderType providerType) { - AccountTwoFaSettings settings = getAccountTwoFaSettings(tenantId, userId) + public AccountTwoFaSettings deleteTwoFaAccountConfig(TenantId tenantId, User user, TwoFaProviderType providerType) { + AccountTwoFaSettings settings = getAccountTwoFaSettings(tenantId, user) .orElseThrow(() -> new IllegalArgumentException("2FA not configured")); settings.getConfigs().remove(providerType); if (settings.getConfigs().size() == 1) { @@ -146,7 +151,7 @@ public class DefaultTwoFaConfigManager implements TwoFaConfigManager { .min(Comparator.comparing(TwoFaAccountConfig::getProviderType)) .ifPresent(config -> config.setUseByDefault(true)); } - return saveAccountTwoFaSettings(tenantId, userId, settings); + return saveAccountTwoFaSettings(tenantId, user, settings); } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/TwoFaConfigManager.java b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/TwoFaConfigManager.java index 92ac638298..e0ffd6e510 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/TwoFaConfigManager.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/TwoFaConfigManager.java @@ -15,9 +15,9 @@ */ package org.thingsboard.server.service.security.auth.mfa.config; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.model.mfa.PlatformTwoFaSettings; import org.thingsboard.server.common.data.security.model.mfa.account.AccountTwoFaSettings; import org.thingsboard.server.common.data.security.model.mfa.account.TwoFaAccountConfig; @@ -27,14 +27,14 @@ import java.util.Optional; public interface TwoFaConfigManager { - Optional getAccountTwoFaSettings(TenantId tenantId, UserId userId); + Optional getAccountTwoFaSettings(TenantId tenantId, User user); - Optional getTwoFaAccountConfig(TenantId tenantId, UserId userId, TwoFaProviderType providerType); + Optional getTwoFaAccountConfig(TenantId tenantId, User user, TwoFaProviderType providerType); - AccountTwoFaSettings saveTwoFaAccountConfig(TenantId tenantId, UserId userId, TwoFaAccountConfig accountConfig); + AccountTwoFaSettings saveTwoFaAccountConfig(TenantId tenantId, User user, TwoFaAccountConfig accountConfig); - AccountTwoFaSettings deleteTwoFaAccountConfig(TenantId tenantId, UserId userId, TwoFaProviderType providerType); + AccountTwoFaSettings deleteTwoFaAccountConfig(TenantId tenantId, User user, TwoFaProviderType providerType); Optional getPlatformTwoFaSettings(TenantId tenantId, boolean sysadminSettingsAsDefault); diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/provider/impl/BackupCodeTwoFaProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/provider/impl/BackupCodeTwoFaProvider.java index 745b651408..672db5dddd 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/provider/impl/BackupCodeTwoFaProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/provider/impl/BackupCodeTwoFaProvider.java @@ -58,7 +58,7 @@ public class BackupCodeTwoFaProvider implements TwoFaProvider { - UriComponents otpAuthUrl = UriComponentsBuilder.fromUriString(accountConfig.getAuthUrl()).build(); - assertThat(otpAuthUrl.getScheme()).isEqualTo("otpauth"); - assertThat(otpAuthUrl.getHost()).isEqualTo("totp"); - assertThat(otpAuthUrl.getQueryParams().getFirst("issuer")).isEqualTo(totpTwoFaProviderConfig.getIssuerName()); - assertThat(otpAuthUrl.getPath()).isEqualTo("/%s:%s", totpTwoFaProviderConfig.getIssuerName(), TENANT_ADMIN_EMAIL); - assertThat(otpAuthUrl.getQueryParams().getFirst("secret")).satisfies(secretKey -> { - assertDoesNotThrow(() -> Base32.decode(secretKey)); - }); - }); - return (TotpTwoFaAccountConfig) generatedTwoFaAccountConfig; - } - @Test public void testGetTwoFaAccountConfig_whenProviderNotConfigured() throws Exception { testVerifyAndSaveTotpTwoFaAccountConfig(); @@ -434,6 +409,56 @@ public class TwoFactorAuthConfigTest extends AbstractControllerTest { assertThat(accountConfig).isEqualTo(initialSmsTwoFaAccountConfig); } + @Test + public void testIsTwoFaEnabled() throws Exception { + configureSmsTwoFaProvider("${code}"); + SmsTwoFaAccountConfig accountConfig = new SmsTwoFaAccountConfig(); + accountConfig.setPhoneNumber("+38050505050"); + twoFaConfigManager.saveTwoFaAccountConfig(tenantId, tenantAdminUser, accountConfig); + + assertThat(twoFactorAuthService.isTwoFaEnabled(tenantId, tenantAdminUser)).isTrue(); + } + + @Test + public void testDeleteTwoFaAccountConfig() throws Exception { + configureSmsTwoFaProvider("${code}"); + loginTenantAdmin(); + SmsTwoFaAccountConfig accountConfig = new SmsTwoFaAccountConfig(); + accountConfig.setPhoneNumber("+38050505050"); + twoFaConfigManager.saveTwoFaAccountConfig(tenantId, tenantAdminUser, accountConfig); + + AccountTwoFaSettings accountTwoFaSettings = readResponse(doGet("/api/2fa/account/settings").andExpect(status().isOk()), AccountTwoFaSettings.class); + TwoFaAccountConfig savedAccountConfig = accountTwoFaSettings.getConfigs().get(TwoFaProviderType.SMS); + assertThat(savedAccountConfig).isEqualTo(accountConfig); + + PlatformTwoFaSettings twoFaSettings = twoFaConfigManager.getPlatformTwoFaSettings(TenantId.SYS_TENANT_ID, true).get(); + twoFaSettings.setEnforceTwoFa(true); + TenantAdministratorsFilter enforcedUsersFilter = new TenantAdministratorsFilter(); + enforcedUsersFilter.setTenantsIds(Set.of(tenantId.getId())); + twoFaSettings.setEnforcedUsersFilter(enforcedUsersFilter); + twoFaConfigManager.savePlatformTwoFaSettings(TenantId.SYS_TENANT_ID, twoFaSettings); + + String errorMessage = getErrorMessage(doDelete("/api/2fa/account/config?providerType=SMS") + .andExpect(status().isBadRequest())); + assertThat(errorMessage).isEqualTo("At least one 2FA provider is required"); + + twoFaSettings.setEnforceTwoFa(false); + twoFaConfigManager.savePlatformTwoFaSettings(TenantId.SYS_TENANT_ID, twoFaSettings); + + doDelete("/api/2fa/account/config?providerType=SMS").andExpect(status().isOk()); + + assertThat(readResponse(doGet("/api/2fa/account/settings").andExpect(status().isOk()), AccountTwoFaSettings.class).getConfigs()) + .doesNotContainKey(TwoFaProviderType.SMS); + } + + private PlatformTwoFaSettings findTwoFaSettings() throws Exception { + return doGet("/api/2fa/settings", PlatformTwoFaSettings.class); + } + + private void saveTwoFaSettings(PlatformTwoFaSettings twoFaSettings) throws Exception { + doPost("/api/2fa/settings", twoFaSettings).andExpect(status().isOk()); + } + private TotpTwoFaProviderConfig configureTotpTwoFaProvider() throws Exception { TotpTwoFaProviderConfig totpTwoFaProviderConfig = new TotpTwoFaProviderConfig(); totpTwoFaProviderConfig.setIssuerName("tb"); @@ -456,37 +481,35 @@ public class TwoFactorAuthConfigTest extends AbstractControllerTest { twoFaSettings.setProviders(Arrays.stream(providerConfigs).collect(Collectors.toList())); twoFaSettings.setMinVerificationCodeSendPeriod(5); twoFaSettings.setTotalAllowedTimeForVerification(100); - doPost("/api/2fa/settings", twoFaSettings).andExpect(status().isOk()); + saveTwoFaSettings(twoFaSettings); } - @Test - public void testIsTwoFaEnabled() throws Exception { - configureSmsTwoFaProvider("${code}"); - SmsTwoFaAccountConfig accountConfig = new SmsTwoFaAccountConfig(); - accountConfig.setPhoneNumber("+38050505050"); - twoFaConfigManager.saveTwoFaAccountConfig(tenantId, tenantAdminUserId, accountConfig); + private TotpTwoFaAccountConfig generateTotpTwoFaAccountConfig(TotpTwoFaProviderConfig totpTwoFaProviderConfig) throws Exception { + TwoFaAccountConfig generatedTwoFaAccountConfig = readResponse(doPost("/api/2fa/account/config/generate?providerType=TOTP") + .andExpect(status().isOk()), TwoFaAccountConfig.class); + assertThat(generatedTwoFaAccountConfig).isInstanceOf(TotpTwoFaAccountConfig.class); - assertThat(twoFactorAuthService.isTwoFaEnabled(tenantId, tenantAdminUserId)).isTrue(); + assertThat(((TotpTwoFaAccountConfig) generatedTwoFaAccountConfig)).satisfies(accountConfig -> { + UriComponents otpAuthUrl = UriComponentsBuilder.fromUriString(accountConfig.getAuthUrl()).build(); + assertThat(otpAuthUrl.getScheme()).isEqualTo("otpauth"); + assertThat(otpAuthUrl.getHost()).isEqualTo("totp"); + assertThat(otpAuthUrl.getQueryParams().getFirst("issuer")).isEqualTo(totpTwoFaProviderConfig.getIssuerName()); + assertThat(otpAuthUrl.getPath()).isEqualTo("/%s:%s", totpTwoFaProviderConfig.getIssuerName(), TENANT_ADMIN_EMAIL); + assertThat(otpAuthUrl.getQueryParams().getFirst("secret")).satisfies(secretKey -> { + assertDoesNotThrow(() -> Base32.decode(secretKey)); + }); + }); + return (TotpTwoFaAccountConfig) generatedTwoFaAccountConfig; } - @Test - public void testDeleteTwoFaAccountConfig() throws Exception { - configureSmsTwoFaProvider("${code}"); - SmsTwoFaAccountConfig accountConfig = new SmsTwoFaAccountConfig(); - accountConfig.setPhoneNumber("+38050505050"); - - loginTenantAdmin(); - - twoFaConfigManager.saveTwoFaAccountConfig(tenantId, tenantAdminUserId, accountConfig); - - AccountTwoFaSettings accountTwoFaSettings = readResponse(doGet("/api/2fa/account/settings").andExpect(status().isOk()), AccountTwoFaSettings.class); - TwoFaAccountConfig savedAccountConfig = accountTwoFaSettings.getConfigs().get(TwoFaProviderType.SMS); - assertThat(savedAccountConfig).isEqualTo(accountConfig); - - doDelete("/api/2fa/account/config?providerType=SMS").andExpect(status().isOk()); + private String savePlatformTwoFaSettingsAndGetError(TwoFaProviderConfig invalidTwoFaProviderConfig) throws Exception { + PlatformTwoFaSettings twoFaSettings = new PlatformTwoFaSettings(); + twoFaSettings.setProviders(Collections.singletonList(invalidTwoFaProviderConfig)); + twoFaSettings.setMinVerificationCodeSendPeriod(5); + twoFaSettings.setTotalAllowedTimeForVerification(100); - assertThat(readResponse(doGet("/api/2fa/account/settings").andExpect(status().isOk()), AccountTwoFaSettings.class).getConfigs()) - .doesNotContainKey(TwoFaProviderType.SMS); + return getErrorMessage(doPost("/api/2fa/settings", twoFaSettings) + .andExpect(status().isBadRequest())); } } diff --git a/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java b/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java index 0da218db67..1138846c08 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java @@ -337,7 +337,7 @@ public class TwoFactorAuthTest extends AbstractControllerTest { @Test public void testAuthWithoutTwoFaAccountConfig() throws ThingsboardException { configureTotpTwoFa(); - twoFaConfigManager.deleteTwoFaAccountConfig(tenantId, user.getId(), TwoFaProviderType.TOTP); + twoFaConfigManager.deleteTwoFaAccountConfig(tenantId, user, TwoFaProviderType.TOTP); assertDoesNotThrow(() -> { login(username, password); @@ -371,15 +371,15 @@ public class TwoFactorAuthTest extends AbstractControllerTest { TotpTwoFaAccountConfig totpTwoFaAccountConfig = (TotpTwoFaAccountConfig) twoFactorAuthService.generateNewAccountConfig(twoFaUser, TwoFaProviderType.TOTP); totpTwoFaAccountConfig.setUseByDefault(true); - twoFaConfigManager.saveTwoFaAccountConfig(tenantId, twoFaUser.getId(), totpTwoFaAccountConfig); + twoFaConfigManager.saveTwoFaAccountConfig(tenantId, twoFaUser, totpTwoFaAccountConfig); SmsTwoFaAccountConfig smsTwoFaAccountConfig = new SmsTwoFaAccountConfig(); smsTwoFaAccountConfig.setPhoneNumber("+38012312322"); - twoFaConfigManager.saveTwoFaAccountConfig(tenantId, twoFaUser.getId(), smsTwoFaAccountConfig); + twoFaConfigManager.saveTwoFaAccountConfig(tenantId, twoFaUser, smsTwoFaAccountConfig); EmailTwoFaAccountConfig emailTwoFaAccountConfig = new EmailTwoFaAccountConfig(); emailTwoFaAccountConfig.setEmail(twoFaUser.getEmail()); - twoFaConfigManager.saveTwoFaAccountConfig(tenantId, twoFaUser.getId(), emailTwoFaAccountConfig); + twoFaConfigManager.saveTwoFaAccountConfig(tenantId, twoFaUser, emailTwoFaAccountConfig); logInWithMfaToken(twoFaUser.getEmail(), "12345678", Authority.PRE_VERIFICATION_TOKEN); @@ -431,9 +431,6 @@ public class TwoFactorAuthTest extends AbstractControllerTest { // verifying enforced users filter createDifferentTenant(); doGet("/api/user/" + savedDifferentTenantUser.getId()).andExpect(status().isOk()); - - twoFaSettings.setEnforceTwoFa(false); - twoFaConfigManager.savePlatformTwoFaSettings(TenantId.SYS_TENANT_ID, twoFaSettings); } private void logInWithMfaToken(String username, String password, Authority expectedScope) throws Exception { @@ -459,7 +456,7 @@ public class TwoFactorAuthTest extends AbstractControllerTest { twoFaConfigManager.savePlatformTwoFaSettings(TenantId.SYS_TENANT_ID, twoFaSettings); TotpTwoFaAccountConfig totpTwoFaAccountConfig = (TotpTwoFaAccountConfig) twoFactorAuthService.generateNewAccountConfig(user, TwoFaProviderType.TOTP); - twoFaConfigManager.saveTwoFaAccountConfig(tenantId, user.getId(), totpTwoFaAccountConfig); + twoFaConfigManager.saveTwoFaAccountConfig(tenantId, user, totpTwoFaAccountConfig); return totpTwoFaAccountConfig; } @@ -477,7 +474,7 @@ public class TwoFactorAuthTest extends AbstractControllerTest { SmsTwoFaAccountConfig smsTwoFaAccountConfig = new SmsTwoFaAccountConfig(); smsTwoFaAccountConfig.setPhoneNumber("+38050505050"); - twoFaConfigManager.saveTwoFaAccountConfig(tenantId, user.getId(), smsTwoFaAccountConfig); + twoFaConfigManager.saveTwoFaAccountConfig(tenantId, user, smsTwoFaAccountConfig); return smsTwoFaAccountConfig; } From 1b04ba09440aec801b2ca3e56048227be8889a77 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 24 Jun 2025 13:22:02 +0300 Subject: [PATCH 05/88] Refactoring for token factory --- .../service/security/model/token/JwtTokenFactory.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java b/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java index 9dac000d4b..e7699e4950 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java @@ -135,18 +135,15 @@ public class JwtTokenFactory { securityUser.setSessionId(claims.get(SESSION_ID, String.class)); } - UserPrincipal principal; + boolean isPublic = false; if (authority != Authority.PRE_VERIFICATION_TOKEN && authority != Authority.MFA_CONFIGURATION_TOKEN) { securityUser.setFirstName(claims.get(FIRST_NAME, String.class)); securityUser.setLastName(claims.get(LAST_NAME, String.class)); securityUser.setEnabled(claims.get(ENABLED, Boolean.class)); - boolean isPublic = claims.get(IS_PUBLIC, Boolean.class); - principal = new UserPrincipal(isPublic ? UserPrincipal.Type.PUBLIC_ID : UserPrincipal.Type.USER_NAME, subject); - } else { - principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, subject); + isPublic = claims.get(IS_PUBLIC, Boolean.class); } + UserPrincipal principal = new UserPrincipal(isPublic ? UserPrincipal.Type.PUBLIC_ID : UserPrincipal.Type.USER_NAME, subject); securityUser.setUserPrincipal(principal); - return securityUser; } From 3010d99eed73354548a6f3b5d7e491b3d62c081b Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 24 Jun 2025 15:08:57 +0300 Subject: [PATCH 06/88] Refactor permissions for MFA configuration --- .../permission/CustomerUserPermissions.java | 2 +- .../DefaultAccessControlService.java | 13 ++++----- .../MfaConfigurationPermissions.java | 28 +++++++++++++++++++ .../permission/SysAdminPermissions.java | 2 +- .../permission/TenantAdminPermissions.java | 2 +- 5 files changed, 37 insertions(+), 10 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/security/permission/MfaConfigurationPermissions.java diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java index 8124671cd7..8c71bab9bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java @@ -28,7 +28,7 @@ import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.service.security.model.SecurityUser; -@Component(value = "customerUserPermissions") +@Component public class CustomerUserPermissions extends AbstractPermissions { public CustomerUserPermissions() { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java b/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java index a5feb1c502..6c54bbe68f 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/DefaultAccessControlService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.security.permission; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; @@ -33,18 +32,18 @@ import java.util.Optional; @Slf4j public class DefaultAccessControlService implements AccessControlService { - private static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; private static final String YOU_DON_T_HAVE_PERMISSION_TO_PERFORM_THIS_OPERATION = "You don't have permission to perform this operation!"; private final Map authorityPermissions = new HashMap<>(); - public DefaultAccessControlService( - @Qualifier("sysAdminPermissions") Permissions sysAdminPermissions, - @Qualifier("tenantAdminPermissions") Permissions tenantAdminPermissions, - @Qualifier("customerUserPermissions") Permissions customerUserPermissions) { + public DefaultAccessControlService(SysAdminPermissions sysAdminPermissions, + TenantAdminPermissions tenantAdminPermissions, + CustomerUserPermissions customerUserPermissions, + MfaConfigurationPermissions mfaConfigurationPermissions) { authorityPermissions.put(Authority.SYS_ADMIN, sysAdminPermissions); authorityPermissions.put(Authority.TENANT_ADMIN, tenantAdminPermissions); authorityPermissions.put(Authority.CUSTOMER_USER, customerUserPermissions); + authorityPermissions.put(Authority.MFA_CONFIGURATION_TOKEN, mfaConfigurationPermissions); } @Override @@ -58,7 +57,7 @@ public class DefaultAccessControlService implements AccessControlService { @Override @SuppressWarnings("unchecked") public void checkPermission(SecurityUser user, Resource resource, - Operation operation, I entityId, T entity) throws ThingsboardException { + Operation operation, I entityId, T entity) throws ThingsboardException { PermissionChecker permissionChecker = getPermissionChecker(user.getAuthority(), resource); if (!permissionChecker.hasPermission(user, operation, entityId, entity)) { permissionDenied(); diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/MfaConfigurationPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/MfaConfigurationPermissions.java new file mode 100644 index 0000000000..b901030a67 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/MfaConfigurationPermissions.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.permission; + +import org.springframework.stereotype.Component; + +@Component +public class MfaConfigurationPermissions extends AbstractPermissions { + + public MfaConfigurationPermissions() { + super(); + // for compatibility with PE + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java index 6bd7aacf54..2593040a12 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.service.security.model.SecurityUser; -@Component(value = "sysAdminPermissions") +@Component public class SysAdminPermissions extends AbstractPermissions { public SysAdminPermissions() { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index 58023be34d..bbad7b52d4 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.service.security.model.SecurityUser; -@Component(value = "tenantAdminPermissions") +@Component public class TenantAdminPermissions extends AbstractPermissions { public TenantAdminPermissions() { From b84d818b28caba27c9e67610da75e8bdf8b46e5f Mon Sep 17 00:00:00 2001 From: ArtemDzhereleiko Date: Tue, 23 Sep 2025 16:53:19 +0300 Subject: [PATCH 07/88] UI: Enforce 2FA --- ui-ngx/src/app/core/auth/auth.service.ts | 17 +- ui-ngx/src/app/core/guards/auth.guard.ts | 10 + .../two-factor-auth-settings.component.html | 351 ++++++++++-------- .../two-factor-auth-settings.component.scss | 7 - .../two-factor-auth-settings.component.ts | 49 ++- .../app/modules/login/login-routing.module.ts | 11 + ui-ngx/src/app/modules/login/login.module.ts | 4 +- ...force-two-factor-auth-login.component.html | 296 +++++++++++++++ ...force-two-factor-auth-login.component.scss | 110 ++++++ .../force-two-factor-auth-login.component.ts | 300 +++++++++++++++ .../src/app/shared/models/authority.enum.ts | 3 +- .../shared/models/two-factor-auth.models.ts | 68 ++++ .../assets/locale/locale.constant-en_US.json | 35 +- 13 files changed, 1093 insertions(+), 168 deletions(-) create mode 100644 ui-ngx/src/app/modules/login/pages/login/force-two-factor-auth-login.component.html create mode 100644 ui-ngx/src/app/modules/login/pages/login/force-two-factor-auth-login.component.scss create mode 100644 ui-ngx/src/app/modules/login/pages/login/force-two-factor-auth-login.component.ts diff --git a/ui-ngx/src/app/core/auth/auth.service.ts b/ui-ngx/src/app/core/auth/auth.service.ts index f8ffd5e8b6..3b6a15688c 100644 --- a/ui-ngx/src/app/core/auth/auth.service.ts +++ b/ui-ngx/src/app/core/auth/auth.service.ts @@ -68,6 +68,7 @@ export class AuthService { redirectUrl: string; oauth2Clients: Array = null; twoFactorAuthProviders: Array = null; + forceTwoFactorAuthProviders: Array = null; private refreshTokenSubject: ReplaySubject = null; private jwtHelper = new JwtHelperService(); @@ -117,6 +118,9 @@ export class AuthService { if (loginResponse.scope === Authority.PRE_VERIFICATION_TOKEN) { this.router.navigateByUrl(`login/mfa`); } + if (loginResponse.scope === Authority.MFA_CONFIGURATION_TOKEN) { + this.router.navigateByUrl(`login/force-mfa`); + } } )); } @@ -239,6 +243,15 @@ export class AuthService { ); } + public getAvailableTwoFaProviders(): Observable> { + return this.http.get>(`/api/2fa/providers`, defaultHttpOptions()).pipe( + catchError(() => of([])), + tap((providers) => { + this.forceTwoFactorAuthProviders = providers; + }) + ); + } + public forceDefaultPlace(authState?: AuthState, path?: string, params?: any): boolean { if (authState && authState.authUser) { if (authState.authUser.authority === Authority.TENANT_ADMIN || authState.authUser.authority === Authority.CUSTOMER_USER) { @@ -266,6 +279,8 @@ export class AuthService { if (isAuthenticated) { if (authState.authUser.authority === Authority.PRE_VERIFICATION_TOKEN) { result = this.router.parseUrl('login/mfa'); + } else if (authState.authUser.authority === Authority.MFA_CONFIGURATION_TOKEN) { + result = this.router.parseUrl('login/force-mfa'); } else if (!path || path === 'login' || this.forceDefaultPlace(authState, path, params)) { if (this.redirectUrl) { const redirectUrl = this.redirectUrl; @@ -399,7 +414,7 @@ export class AuthService { loadUserSubject.error(err); } ); - } else if (authPayload.authUser?.authority === Authority.PRE_VERIFICATION_TOKEN) { + } else if (authPayload.authUser?.authority === Authority.PRE_VERIFICATION_TOKEN || authPayload.authUser?.authority === Authority.MFA_CONFIGURATION_TOKEN) { loadUserSubject.next(authPayload); loadUserSubject.complete(); } else if (authPayload.authUser?.userId) { diff --git a/ui-ngx/src/app/core/guards/auth.guard.ts b/ui-ngx/src/app/core/guards/auth.guard.ts index f6d8753882..f8bbcc3241 100644 --- a/ui-ngx/src/app/core/guards/auth.guard.ts +++ b/ui-ngx/src/app/core/guards/auth.guard.ts @@ -104,6 +104,16 @@ export class AuthGuard { } this.authService.logout(); return of(this.authService.defaultUrl(false)); + } else if (path === 'login.force-mfa') { + if (authState.authUser?.authority === Authority.MFA_CONFIGURATION_TOKEN) { + return this.authService.getAvailableTwoFaProviders().pipe( + map(() => { + return true; + }) + ); + } + this.authService.logout(); + return of(this.authService.defaultUrl(false)); } else { return of(true); } diff --git a/ui-ngx/src/app/modules/home/pages/admin/two-factor-auth-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/two-factor-auth-settings.component.html index 8605921040..faf1904427 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/two-factor-auth-settings.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/two-factor-auth-settings.component.html @@ -30,163 +30,216 @@
-
-
- admin.2fa.available-providers - - - - - - {{ twoFactorAuthProvidersData.get(provider.value.providerType).name | translate }} - - - - - - - - - admin.2fa.issuer-name - - - {{ "admin.2fa.issuer-name-required" | translate }} - - - -
- - admin.2fa.verification-message-template - - - {{ "admin.2fa.verification-message-template-required" | translate }} - - - {{ "admin.2fa.verification-message-template-pattern" | translate }} - - - - - admin.2fa.verification-code-lifetime - - - {{ "admin.2fa.verification-code-lifetime-required" | translate }} - - - {{ "admin.2fa.verification-code-lifetime-pattern" | translate }} - - -
-
- - admin.2fa.verification-code-lifetime - - - {{ "admin.2fa.verification-code-lifetime-required" | translate }} - - - {{ "admin.2fa.verification-code-lifetime-pattern" | translate }} - - -
-
- - admin.2fa.number-of-codes - - - {{ "admin.2fa.number-of-codes-required" | translate }} - - - {{ "admin.2fa.number-of-codes-pattern" | translate }} - - -
- - - - - -
-
- admin.2fa.verification-limitations -
- - admin.2fa.total-allowed-time-for-verification - - - {{ 'admin.2fa.total-allowed-time-for-verification-required' | translate }} - - - {{ 'admin.2fa.total-allowed-time-for-verification-pattern' | translate }} - - - - admin.2fa.retry-verification-code-period - - - {{ 'admin.2fa.retry-verification-code-period-required' | translate }} - - - {{ 'admin.2fa.retry-verification-code-period-pattern' | translate }} - - - - admin.2fa.max-verification-failures-before-user-lockout - - - {{ 'admin.2fa.max-verification-failures-before-user-lockout-pattern' | translate }} - - -
- - +
+
+ - - + + + {{ 'admin.2fa.force-2fa' | translate }} - {{ 'admin.2fa.verification-code-check-rate-limit' | translate }} -
- - admin.2fa.number-of-checking-attempts - - - {{ 'admin.2fa.number-of-checking-attempts-required' | translate }} - - - {{ 'admin.2fa.number-of-checking-attempts-pattern' | translate }} - +
+ + admin.2fa.enforce-for + + + {{ notificationTargetConfigTypeInfoMap.get(type).name | translate }} + + - - admin.2fa.within-time - - - {{ 'admin.2fa.within-time-required' | translate }} - - - {{ 'admin.2fa.within-time-pattern' | translate }} - - -
+
+
+ + {{ 'tenant.tenant' | translate }} + {{ 'tenant-profile.tenant-profile' | translate }} + +
+ + + + + + + + +
+ -
+
+ +
+
admin.2fa.available-providers
+ +
+ + + + + {{ twoFactorAuthProvidersData.get(provider.value.providerType).name | translate }} + + + + + + + + + admin.2fa.issuer-name + + + {{ "admin.2fa.issuer-name-required" | translate }} + + + +
+ + admin.2fa.verification-message-template + + + {{ "admin.2fa.verification-message-template-required" | translate }} + + + {{ "admin.2fa.verification-message-template-pattern" | translate }} + + + + +
+
+ + +
+
+ + admin.2fa.number-of-codes + + + {{ "admin.2fa.number-of-codes-required" | translate }} + + + {{ "admin.2fa.number-of-codes-pattern" | translate }} + + +
+ + + +
+ +
+
+
admin.2fa.verification-limitations
+
+
+ + + + + + admin.2fa.max-verification-failures-before-user-lockout + + + {{ 'admin.2fa.max-verification-failures-before-user-lockout-pattern' | translate }} + + +
+
+ + + + + {{ 'admin.2fa.verification-code-check-rate-limit' | translate }} + + + + +
+ + admin.2fa.number-of-checking-attempts + + + {{ 'admin.2fa.number-of-checking-attempts-required' | translate }} + + + {{ 'admin.2fa.number-of-checking-attempts-pattern' | translate }} + + + + +
+ + +
+
+
+ {{ (config ? 'login.two-fa' :'login.two-fa-required') | translate }} + + + +
+

{{ (config ? 'login.set-up-verification-method-login' :'login.set-up-verification-method') | translate }}

+ + + + @if (config) { + + } +
+ + + } + @case (ForceTwoFAState.AUTHENTICATOR_APP) { + @switch (appState()) { + @case (ProvidersState.INPUT) { + + } + @case (ProvidersState.ENTER_CODE) { + + } + @case (ProvidersState.SUCCESS) { + + } + } + } + @case (ForceTwoFAState.SMS) { + @switch (smsState()) { + @case (ProvidersState.INPUT) { + + } + @case (ProvidersState.ENTER_CODE) { + + } + @case (ProvidersState.SUCCESS) { + + } + } + } + @case (ForceTwoFAState.EMAIL) { + @switch (emailState()) { + @case (ProvidersState.INPUT) { + + } + @case (ProvidersState.ENTER_CODE) { + + } + @case (ProvidersState.SUCCESS) { + + } + } + } + @case (ForceTwoFAState.BACKUP_CODE) { + @switch (backupCodeState()) { + @case (BackupCodeState.CODE) { + + } + @case (BackupCodeState.SUCCESS) { + + } + } + } + } +
+ + + + + + + diff --git a/ui-ngx/src/app/modules/login/pages/login/force-two-factor-auth-login.component.scss b/ui-ngx/src/app/modules/login/pages/login/force-two-factor-auth-login.component.scss new file mode 100644 index 0000000000..d62d7f1628 --- /dev/null +++ b/ui-ngx/src/app/modules/login/pages/login/force-two-factor-auth-login.component.scss @@ -0,0 +1,110 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import '../../../../../scss/constants'; + +:host { + display: flex; + flex: 1 1 0; + width: 100%; + height: 100%; + + .tb-two-factor-auth-login-content { + background-color: #eee; + + .tb-two-factor-auth-login-card { + max-height: 100vh; + overflow: auto; + padding: 48px 48px 48px 16px; + + @media #{$mat-xs} { + height: 100%; + } + + @media #{$mat-gt-xs} { + width: 450px !important; + } + + .mat-mdc-card-title { + font: 400 28px / 36px Roboto, "Helvetica Neue", sans-serif; + } + + .mat-mdc-card-header { + padding: 0; + } + + .mat-mdc-card-content { + margin-top: 34px; + margin-left: 40px; + padding: 0; + } + + .mat-body { + letter-spacing: 0.25px; + line-height: 16px; + } + + .backup-code { + p { + text-align: justify; + } + + .container { + border: 1px solid; + border-radius: 4px; + gap: 16px; + display: grid; + grid-template-columns: 1fr 1fr; + justify-items: center; + padding: 16px 0; + margin-bottom: 16px; + + .code { + letter-spacing: 0.25px; + font-family: Roboto Mono, "Helvetica Neue", monospace; + } + } + + .action-buttons { + margin-bottom: 40px; + } + } + } + } + + ::ng-deep { + .tb-two-factor-auth-login-content { + .tb-two-factor-auth-login-card { + button.mat-mdc-icon-button { + .mat-icon { + color: rgba(255, 255, 255, 0.8); + } + } + } + .mat-mdc-form-field .mat-mdc-form-field-hint-wrapper { + color: rgba(255, 255, 255, 0.8); + } + } + + button.provider, button.navigation { + text-align: start; + font-weight: 400; + color: rgba(255, 255, 255, 0.8); + &:not([disabled][disabled]) { + border-color: rgba(255, 255, 255, .8); + } + } + } +} diff --git a/ui-ngx/src/app/modules/login/pages/login/force-two-factor-auth-login.component.ts b/ui-ngx/src/app/modules/login/pages/login/force-two-factor-auth-login.component.ts new file mode 100644 index 0000000000..ef1917ba24 --- /dev/null +++ b/ui-ngx/src/app/modules/login/pages/login/force-two-factor-auth-login.component.ts @@ -0,0 +1,300 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, ElementRef, OnDestroy, OnInit, signal, ViewChild } from '@angular/core'; +import { AuthService } from '@core/auth/auth.service'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { PageComponent } from '@shared/components/page.component'; +import { UntypedFormBuilder, UntypedFormGroup, Validators } from '@angular/forms'; +import { TwoFactorAuthenticationService } from '@core/http/two-factor-authentication.service'; +import { + AccountTwoFaSettings, + BackupCodeTwoFactorAuthAccountConfig, + TotpTwoFactorAuthAccountConfig, + TwoFactorAuthAccountConfig, + twoFactorAuthProvidersEnterCodeCardTranslate, + twoFactorAuthProvidersLoginData, + twoFactorAuthProvidersSuccessCardTranslate, + TwoFactorAuthProviderType +} from '@shared/models/two-factor-auth.models'; +import { phoneNumberPattern } from '@shared/models/settings.models'; +import { deepClone, isDefinedAndNotNull, unwrapModule } from '@core/utils'; +import { MatDialog } from '@angular/material/dialog'; +import { DialogService } from '@core/services/dialog.service'; +import { getCurrentAuthUser } from '@core/auth/auth.selectors'; +import printTemplate from '@home/pages/security/authentication-dialog/backup-code-print-template.raw'; +import { ImportExportService } from '@shared/import-export/import-export.service'; +import { mergeMap, tap } from 'rxjs/operators'; + +enum ForceTwoFAState { + SETUP = 'setup', + AUTHENTICATOR_APP = 'authenticatorApp', + SMS = 'sms', + EMAIL = 'email', + BACKUP_CODE = 'backupCode', +} + +enum ProvidersState { + INPUT = 'INPUT', + ENTER_CODE = 'ENTER_CODE', + SUCCESS = 'SUCCESS', +} + +enum BackupCodeState { + CODE = 'CODE', + SUCCESS = 'SUCCESS', +} + +@Component({ + selector: 'tb-force-two-factor-auth-login', + templateUrl: './force-two-factor-auth-login.component.html', + styleUrls: ['./force-two-factor-auth-login.component.scss'] +}) +export class ForceTwoFactorAuthLoginComponent extends PageComponent implements OnInit, OnDestroy { + + TwoFactorAuthProviderType = TwoFactorAuthProviderType; + providersData = twoFactorAuthProvidersLoginData; + allowProviders: TwoFactorAuthProviderType[] = []; + config: AccountTwoFaSettings; + + twoFactorAuthProvidersEnterCodeCardTranslate = twoFactorAuthProvidersEnterCodeCardTranslate; + twoFactorAuthProvidersSuccessCardTranslate = twoFactorAuthProvidersSuccessCardTranslate; + + ForceTwoFAState = ForceTwoFAState; + ProvidersState = ProvidersState; + BackupCodeState = BackupCodeState + + state = signal(ForceTwoFAState.SETUP); + appState = signal(ProvidersState.INPUT); + smsState = signal(ProvidersState.INPUT); + emailState = signal(ProvidersState.INPUT); + backupCodeState = signal(BackupCodeState.CODE); + + totpAuthURL: string; + totpAuthURLSecret: string; + backupCode: BackupCodeTwoFactorAuthAccountConfig; + + configForm: UntypedFormGroup; + smsConfigForm: UntypedFormGroup; + emailConfigForm: UntypedFormGroup; + + private providersInfo: TwoFactorAuthProviderType[]; + private authAccountConfig: TwoFactorAuthAccountConfig; + private useByDefault: boolean = true; + + @ViewChild('canvas', {static: false}) canvasRef: ElementRef; + + constructor(protected store: Store, + private authService: AuthService, + private twoFaService: TwoFactorAuthenticationService, + private importExportService: ImportExportService, + public dialog: MatDialog, + public dialogService: DialogService, + private fb: UntypedFormBuilder) { + super(store); + } + + ngOnInit() { + this.providersInfo = this.authService.forceTwoFactorAuthProviders; + this.allowedProviders(); + this.configForm = this.fb.group({ + verificationCode: ['', [ + Validators.required, + Validators.minLength(6), + Validators.maxLength(6), + Validators.pattern(/^\d*$/) + ]] + }); + + this.smsConfigForm = this.fb.group({ + phone: ['', [Validators.required, Validators.pattern(phoneNumberPattern)]] + }); + + this.emailConfigForm = this.fb.group({ + email: [getCurrentAuthUser(this.store).sub, [Validators.required, Validators.email]] + }); + + this.twoFaService.getAccountTwoFaSettings().subscribe(accountConfig => { + if (accountConfig) { + this.config = accountConfig; + this.useByDefault = false; + } + }); + } + + goBackByType(type: TwoFactorAuthProviderType) { + switch (type) { + case TwoFactorAuthProviderType.TOTP: + this.appState.set(ProvidersState.INPUT); + this.updateQRCode(); + break; + case TwoFactorAuthProviderType.SMS: + this.smsState.set(ProvidersState.INPUT); + break; + case TwoFactorAuthProviderType.EMAIL: + this.emailState.set(ProvidersState.INPUT); + break; + } + } + + get isAnyProviderAvailable() { + return this.config?.configs ? Object.keys(this.config?.configs)?.length < this.allowProviders?.length : true; + } + + private allowedProviders() { + if (isDefinedAndNotNull(this.config)) { + this.allowProviders = this.providersInfo; + } else { + this.allowProviders = this.providersInfo.filter(provider => provider !== TwoFactorAuthProviderType.BACKUP_CODE); + } + } + + updateState(type: TwoFactorAuthProviderType) { + switch (type) { + case TwoFactorAuthProviderType.TOTP: + this.state.set(ForceTwoFAState.AUTHENTICATOR_APP); + this.twoFaService.generateTwoFaAccountConfig(TwoFactorAuthProviderType.TOTP).subscribe(accountConfig => { + this.authAccountConfig = accountConfig as TotpTwoFactorAuthAccountConfig; + this.totpAuthURL = this.authAccountConfig.authUrl; + this.totpAuthURLSecret = new URL(this.totpAuthURL).searchParams.get('secret'); + this.authAccountConfig.useByDefault = this.useByDefault; + this.useByDefault = false; + this.updateQRCode(); + }); + break; + case TwoFactorAuthProviderType.SMS: + this.state.set(ForceTwoFAState.SMS); + break; + case TwoFactorAuthProviderType.EMAIL: + this.state.set(ForceTwoFAState.EMAIL); + break; + case TwoFactorAuthProviderType.BACKUP_CODE: + this.state.set(ForceTwoFAState.BACKUP_CODE); + this.twoFaService.generateTwoFaAccountConfig(TwoFactorAuthProviderType.BACKUP_CODE).pipe( + tap((data: BackupCodeTwoFactorAuthAccountConfig) => this.backupCode = data), + mergeMap(data => this.twoFaService.verifyAndSaveTwoFaAccountConfig(data, null, {ignoreLoading: true})) + ).subscribe((config) => { + this.config = config; + }); + break; + } + } + + sendSmsCode() { + if (this.smsConfigForm.valid) { + this.authAccountConfig = { + providerType: TwoFactorAuthProviderType.SMS, + useByDefault: this.useByDefault, + phoneNumber: this.smsConfigForm.get('phone').value as string + }; + this.useByDefault = false; + this.twoFaService.submitTwoFaAccountConfig(this.authAccountConfig).subscribe(() => this.smsState.set(ProvidersState.ENTER_CODE)); + } + } + + sendEmailCode() { + if (this.emailConfigForm.valid) { + this.authAccountConfig = { + providerType: TwoFactorAuthProviderType.EMAIL, + useByDefault: this.useByDefault, + email: this.emailConfigForm.get('email').value as string + }; + this.useByDefault = false; + this.twoFaService.submitTwoFaAccountConfig(this.authAccountConfig).subscribe(() => this.emailState.set(ProvidersState.ENTER_CODE)); + } + } + + tryAnotherWay(type: TwoFactorAuthProviderType) { + this.state.set(ForceTwoFAState.SETUP); + this.configForm.reset(); + switch (type) { + case TwoFactorAuthProviderType.TOTP: + this.appState.set(ProvidersState.INPUT); + break; + case TwoFactorAuthProviderType.SMS: + this.smsState.set(ProvidersState.INPUT); + this.smsConfigForm.reset(); + break; + case TwoFactorAuthProviderType.EMAIL: + this.emailState.set(ProvidersState.INPUT) + this.emailConfigForm.get('email').reset(getCurrentAuthUser(this.store).sub); + break; + } + } + + saveConfig(type: TwoFactorAuthProviderType) { + if (this.configForm.valid) { + this.twoFaService.verifyAndSaveTwoFaAccountConfig(this.authAccountConfig, + this.configForm.get('verificationCode').value).subscribe((config) => { + switch (type) { + case TwoFactorAuthProviderType.TOTP: + this.appState.set(ProvidersState.SUCCESS); + break; + case TwoFactorAuthProviderType.SMS: + this.smsState.set(ProvidersState.SUCCESS); + break; + case TwoFactorAuthProviderType.EMAIL: + this.emailState.set(ProvidersState.SUCCESS); + break; + } + this.config = config; + this.authAccountConfig = null; + this.allowedProviders(); + }); + } + } + + private updateQRCode() { + import('qrcode').then((QRCode) => { + unwrapModule(QRCode).toCanvas(this.canvasRef.nativeElement, this.totpAuthURL); + this.canvasRef.nativeElement.style.width = 'auto'; + this.canvasRef.nativeElement.style.height = 'auto'; + }); + } + + ngOnDestroy() { + super.ngOnDestroy(); + } + + cancelLogin() { + this.authService.logout(); + } + + downloadFile() { + this.importExportService.exportText(this.backupCode.codes, 'backup-codes'); + } + + printCode() { + const codeTemplate = deepClone(this.backupCode.codes) + .map(code => `
${code}
`).join(''); + const printPage = printTemplate.replace('${codesBlock}', codeTemplate); + const newWindow = window.open('', 'Print backup code'); + + newWindow.document.open(); + newWindow.document.write(printPage); + + setTimeout(() => { + newWindow.print(); + + newWindow.document.close(); + + setTimeout(() => { + newWindow.close(); + }, 10); + }, 0); + } +} diff --git a/ui-ngx/src/app/shared/models/authority.enum.ts b/ui-ngx/src/app/shared/models/authority.enum.ts index 8b18beb887..d91ecf777a 100644 --- a/ui-ngx/src/app/shared/models/authority.enum.ts +++ b/ui-ngx/src/app/shared/models/authority.enum.ts @@ -20,5 +20,6 @@ export enum Authority { CUSTOMER_USER = 'CUSTOMER_USER', REFRESH_TOKEN = 'REFRESH_TOKEN', ANONYMOUS = 'ANONYMOUS', - PRE_VERIFICATION_TOKEN = 'PRE_VERIFICATION_TOKEN' + PRE_VERIFICATION_TOKEN = 'PRE_VERIFICATION_TOKEN', + MFA_CONFIGURATION_TOKEN = 'MFA_CONFIGURATION_TOKEN' } diff --git a/ui-ngx/src/app/shared/models/two-factor-auth.models.ts b/ui-ngx/src/app/shared/models/two-factor-auth.models.ts index f2f392bd04..96a8d8186e 100644 --- a/ui-ngx/src/app/shared/models/two-factor-auth.models.ts +++ b/ui-ngx/src/app/shared/models/two-factor-auth.models.ts @@ -14,7 +14,11 @@ /// limitations under the License. /// +import { UsersFilter } from '@shared/models/notification.models'; + export interface TwoFactorAuthSettings { + enforceTwoFa: boolean; + enforcedUsersFilter: UsersFilter; maxVerificationFailuresBeforeUserLockout: number; providers: Array; totalAllowedTimeForVerification: number; @@ -24,12 +28,18 @@ export interface TwoFactorAuthSettings { } export interface TwoFactorAuthSettingsForm extends TwoFactorAuthSettings{ + enforceTwoFa: boolean; + enforcedUsersFilter: UsersFilterWithFilterByTenant; providers: Array; verificationCodeCheckRateLimitEnable: boolean; verificationCodeCheckRateLimitNumber: number; verificationCodeCheckRateLimitTime: number; } +export interface UsersFilterWithFilterByTenant extends UsersFilter{ + filterByTenants?: boolean; +} + export type TwoFactorAuthProviderConfig = Partial; @@ -183,3 +193,61 @@ export const twoFactorAuthProvidersLoginData = new Map>( + [ + [ + TwoFactorAuthProviderType.TOTP, { + name: 'login.enable-authenticator-app', + description: 'login.enable-authenticator-app-description' + } + ], + [ + TwoFactorAuthProviderType.SMS, { + name: 'login.enable-authenticator-sms', + description: 'login.enable-authenticator-sms-description' + } + ], + [ + TwoFactorAuthProviderType.EMAIL, { + name: 'login.enable-authenticator-email', + description: 'login.enable-authenticator-email-description' + } + ], + [ + TwoFactorAuthProviderType.BACKUP_CODE, { + name: 'security.2fa.provider.backup_code', + description: 'login.backup-code-auth-description' + } + ] + ] +); + +export const twoFactorAuthProvidersSuccessCardTranslate = new Map>( + [ + [ + TwoFactorAuthProviderType.TOTP, { + name: 'login.authenticator-app-success', + description: 'login.authenticator-app-success-description' + } + ], + [ + TwoFactorAuthProviderType.SMS, { + name: 'login.authenticator-sms-success', + description: 'login.authenticator-sms-success-description' + } + ], + [ + TwoFactorAuthProviderType.EMAIL, { + name: 'login.authenticator-email-success', + description: 'login.authenticator-email-success-description' + } + ], + [ + TwoFactorAuthProviderType.BACKUP_CODE, { + name: 'login.authenticator-backup-code-success', + description: 'login.authenticator-backup-code-success-description' + } + ] + ] +); diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index dbacae6ad3..9ff922db75 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -496,15 +496,15 @@ "number-of-codes-pattern": "Number of codes must be a positive integer.", "number-of-codes-required": "Number of codes is required.", "provider": "Provider", - "retry-verification-code-period": "Retry verification code period (sec)", + "retry-verification-code-period": "Retry verification code period", "retry-verification-code-period-pattern": "Minimal period time is 5 sec", "retry-verification-code-period-required": "Retry verification code period is required.", - "total-allowed-time-for-verification": "Total allowed time for verification (sec)", + "total-allowed-time-for-verification": "Total allowed time for verification", "total-allowed-time-for-verification-pattern": "Minimal total allowed time is 60 sec", "total-allowed-time-for-verification-required": "Total allowed time is required.", "use-system-two-factor-auth-settings": "Use system two factor auth settings", "verification-code-check-rate-limit": "Verification code check rate limit", - "verification-code-lifetime": "Verification code lifetime (sec)", + "verification-code-lifetime": "Verification code lifetime", "verification-code-lifetime-pattern": "Verification code lifetime must be a positive integer.", "verification-code-lifetime-required": "Verification code lifetime is required.", "verification-message-template": "Verification message template", @@ -513,7 +513,9 @@ "verification-message-template-required": "Verification message template is required.", "within-time": "Within time (sec)", "within-time-pattern": "Time must be a positive integer.", - "within-time-required": "Time is required." + "within-time-required": "Time is required.", + "force-2fa": "Force two-factor authentication", + "enforce-for": "Enforce for" }, "jwt": { "security-settings": "JWT security settings", @@ -3884,7 +3886,30 @@ "activation-link-expired": "Activation link has expired", "activation-link-expired-message": "The link to activate your profile has expired. You can return to the login page to receive a new email.", "reset-password-link-expired": "Password reset link has expired", - "reset-password-link-expired-message": "The link to reset your password has expired. You can return to the login page to receive a new email." + "reset-password-link-expired-message": "The link to reset your password has expired. You can return to the login page to receive a new email.", + "two-fa": "Two-factor authentication", + "two-fa-required": "Two-factor authentication is required", + "set-up-verification-method": "Set up a verification method to continue", + "set-up-verification-method-login": "Set up a verification method or login", + "enable-authenticator-app": "Enable authenticator app", + "enable-authenticator-app-description": "Please enter the security code from your authenticator app", + "enable-authenticator-sms": "Enable SMS authenticator", + "enable-authenticator-sms-description": "Enter a 6-digit code we just sent to ", + "enable-authenticator-email": "Enable email authenticator", + "enable-authenticator-email-description": "A security code has been sent to your email address at ", + "enter-key-manually": "or enter this 32-digits key manually:", + "continue": "Continue", + "confirm": "Confirm", + "authenticator-app-success": "Authenticator app successfully enabled", + "authenticator-app-success-description": "The next time you log in, you will need to provide a two-factor authentication code", + "authenticator-sms-success": "SMS authenticator successfully enabled", + "authenticator-sms-success-description": "The next time you log in, you will be prompted to enter the security code that will be sent to the phone number", + "authenticator-email-success": "Email authenticator successfully enabled", + "authenticator-email-success-description": "The next time you log in, you will be prompted to enter the security code that will be sent to your email address", + "authenticator-backup-code-success": "Backup code successfully enabled", + "authenticator-backup-code-success-description": "The next time you log in, you will be prompted to enter the security code or use one of backup code.", + "add-verification-method": "Add verification method", + "get-backup-code": "Get backup code" }, "markdown": { "edit": "Edit", From 9ca4ff92f6b54bc8d48ceae2195cfb6e72ba9eb7 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Wed, 24 Sep 2025 12:25:02 +0300 Subject: [PATCH 08/88] 2FA: allow MFA_CONFIGURATION_TOKEN for getting and submitting config --- .../server/controller/TwoFactorAuthConfigController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthConfigController.java b/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthConfigController.java index 00829df047..eef086452f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthConfigController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthConfigController.java @@ -66,7 +66,7 @@ public class TwoFactorAuthConfigController extends BaseController { " }\n}\n```" + ControllerConstants.AVAILABLE_FOR_ANY_AUTHORIZED_USER) @GetMapping("/account/settings") - @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER', 'MFA_CONFIGURATION_TOKEN')") public AccountTwoFaSettings getAccountTwoFaSettings() throws ThingsboardException { SecurityUser user = getCurrentUser(); return twoFaConfigManager.getAccountTwoFaSettings(user.getTenantId(), user).orElse(null); @@ -125,7 +125,7 @@ public class TwoFactorAuthConfigController extends BaseController { "or if the provider is not configured for usage. " + ControllerConstants.AVAILABLE_FOR_ANY_AUTHORIZED_USER) @PostMapping("/account/config/submit") - @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER', 'MFA_CONFIGURATION_TOKEN')") public void submitTwoFaAccountConfig(@Valid @RequestBody TwoFaAccountConfig accountConfig) throws Exception { SecurityUser user = getCurrentUser(); twoFactorAuthService.prepareVerificationCode(user, accountConfig, false); From 4ddc8030ccedbe92a420324b8b46dd9cefa086aa Mon Sep 17 00:00:00 2001 From: ArtemDzhereleiko Date: Wed, 24 Sep 2025 16:35:23 +0300 Subject: [PATCH 09/88] UI: Add secret for totp auth dialog --- .../two-factor-auth-settings.component.ts | 19 +++++++--------- .../totp-auth-dialog.component.html | 13 +++++++++++ .../totp-auth-dialog.component.ts | 2 ++ ...force-two-factor-auth-login.component.html | 22 +++++++++++-------- .../assets/locale/locale.constant-en_US.json | 2 +- 5 files changed, 37 insertions(+), 21 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/admin/two-factor-auth-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/two-factor-auth-settings.component.ts index bea85a4639..38a6230b6f 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/two-factor-auth-settings.component.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/two-factor-auth-settings.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, OnDestroy, OnInit, QueryList, ViewChildren } from '@angular/core'; +import { Component, DestroyRef, OnDestroy, OnInit, QueryList, ViewChildren } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; import { Store } from '@ngrx/store'; @@ -29,11 +29,10 @@ import { TwoFactorAuthSettingsForm } from '@shared/models/two-factor-auth.models'; import { isDefined, isNotEmptyStr } from '@core/utils'; -import { Subject } from 'rxjs'; -import { takeUntil } from 'rxjs/operators'; import { MatExpansionPanel } from '@angular/material/expansion'; import { NotificationTargetConfigType, NotificationTargetConfigTypeInfoMap } from '@shared/models/notification.models'; import { EntityType } from '@shared/models/entity-type.models'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; @Component({ selector: 'tb-2fa-settings', @@ -42,7 +41,6 @@ import { EntityType } from '@shared/models/entity-type.models'; }) export class TwoFactorAuthSettingsComponent extends PageComponent implements OnInit, HasConfirmForm, OnDestroy { - private readonly destroy$ = new Subject(); private readonly posIntValidation = [Validators.required, Validators.min(1), Validators.pattern(/^\d*$/)]; twoFaFormGroup: UntypedFormGroup; @@ -62,7 +60,8 @@ export class TwoFactorAuthSettingsComponent extends PageComponent implements OnI constructor(protected store: Store, private twoFaService: TwoFactorAuthenticationService, - private fb: UntypedFormBuilder) { + private fb: UntypedFormBuilder, + private destroyRef: DestroyRef) { super(store); } @@ -75,8 +74,6 @@ export class TwoFactorAuthSettingsComponent extends PageComponent implements OnI ngOnDestroy() { super.ngOnDestroy(); - this.destroy$.next(); - this.destroy$.complete(); } confirmForm(): UntypedFormGroup { @@ -156,7 +153,7 @@ export class TwoFactorAuthSettingsComponent extends PageComponent implements OnI this.buildProvidersSettingsForm(provider); }); this.twoFaFormGroup.get('verificationCodeCheckRateLimitEnable').valueChanges.pipe( - takeUntil(this.destroy$) + takeUntilDestroyed(this.destroyRef) ).subscribe(value => { if (value) { this.twoFaFormGroup.get('verificationCodeCheckRateLimitNumber').enable({emitEvent: false}); @@ -167,7 +164,7 @@ export class TwoFactorAuthSettingsComponent extends PageComponent implements OnI } }); this.providersForm.valueChanges.pipe( - takeUntil(this.destroy$) + takeUntilDestroyed(this.destroyRef) ).subscribe((value: TwoFactorAuthProviderConfigForm[]) => { const activeProvider = value.filter(provider => provider.enable); const indexBackupCode = Object.values(TwoFactorAuthProviderType).indexOf(TwoFactorAuthProviderType.BACKUP_CODE); @@ -181,7 +178,7 @@ export class TwoFactorAuthSettingsComponent extends PageComponent implements OnI } }); this.twoFaFormGroup.get('enforceTwoFa').valueChanges.pipe( - takeUntil(this.destroy$) + takeUntilDestroyed(this.destroyRef) ).subscribe(value => { if (value) { this.twoFaFormGroup.get('enforcedUsersFilter').enable({emitEvent: false}); @@ -245,7 +242,7 @@ export class TwoFactorAuthSettingsComponent extends PageComponent implements OnI } const newProviders = this.fb.group(formControlConfig); newProviders.get('enable').valueChanges.pipe( - takeUntil(this.destroy$) + takeUntilDestroyed(this.destroyRef) ).subscribe(value => { if (value) { newProviders.enable({emitEvent: false}); diff --git a/ui-ngx/src/app/modules/home/pages/security/authentication-dialog/totp-auth-dialog.component.html b/ui-ngx/src/app/modules/home/pages/security/authentication-dialog/totp-auth-dialog.component.html index 1b33d4df1e..6bf4174268 100644 --- a/ui-ngx/src/app/modules/home/pages/security/authentication-dialog/totp-auth-dialog.component.html +++ b/ui-ngx/src/app/modules/home/pages/security/authentication-dialog/totp-auth-dialog.component.html @@ -52,6 +52,19 @@

security.2fa.dialog.scan-qr-code

+

login.enter-key-manually

+
+ {{ totpAuthURLSecret }} + + +

security.2fa.dialog.enter-verification-code

; @@ -55,6 +56,7 @@ export class TotpAuthDialogComponent extends DialogComponent { this.authAccountConfig = accountConfig as TotpTwoFactorAuthAccountConfig; this.totpAuthURL = this.authAccountConfig.authUrl; + this.totpAuthURLSecret = new URL(this.totpAuthURL).searchParams.get('secret'); this.authAccountConfig.useByDefault = true; import('qrcode').then((QRCode) => { unwrapModule(QRCode).toCanvas(this.canvasRef.nativeElement, this.totpAuthURL); diff --git a/ui-ngx/src/app/modules/login/pages/login/force-two-factor-auth-login.component.html b/ui-ngx/src/app/modules/login/pages/login/force-two-factor-auth-login.component.html index 87c5179a03..e71c04950a 100644 --- a/ui-ngx/src/app/modules/login/pages/login/force-two-factor-auth-login.component.html +++ b/ui-ngx/src/app/modules/login/pages/login/force-two-factor-auth-login.component.html @@ -31,12 +31,12 @@

{{ (config ? 'login.set-up-verification-method-login' :'login.set-up-verification-method') | translate }}

- + @for (provider of allowProviders; track provider) { - + } @if (config) { - {{ 'login.enable-authenticator-sms' | translate }} + {{ 'login.enable-authenticator-email' | translate }}
-

security.2fa.dialog.sms-step-description

+

login.email-description

+ placeholder="{{ 'login.email-label' | translate }}" /> - {{ 'user.email-required' | translate }} + {{ 'login.email-required' | translate }} - {{ 'user.invalid-email-format' | translate }} + {{ 'login.invalid-email-format' | translate }}
-

security.2fa.dialog.backup-code-warn

+

login.backup-code-warn

@@ -257,9 +261,9 @@ maxlength="6" type="text" required inputmode="numeric" pattern="[0-9]*" autocomplete="off" - placeholder="{{ 'security.2fa.dialog.verification-code' | translate }}"> + placeholder="{{ 'login.verification-code' | translate }}"> - {{ 'security.2fa.dialog.verification-code-invalid' | translate }} + {{ 'login.verification-code-invalid' | translate }}
diff --git a/ui-ngx/src/app/modules/login/pages/login/two-factor-auth-login.component.scss b/ui-ngx/src/app/modules/login/pages/login/two-factor-auth-login.component.scss index 68a4dcb4a4..09e3c29c43 100644 --- a/ui-ngx/src/app/modules/login/pages/login/two-factor-auth-login.component.scss +++ b/ui-ngx/src/app/modules/login/pages/login/two-factor-auth-login.component.scss @@ -72,9 +72,19 @@ } ::ng-deep { + .tb-two-factor-auth-login-content { + .tb-two-factor-auth-login-card { + button.mat-mdc-icon-button { + .mat-icon { + color: rgba(255, 255, 255, 0.8); + } + } + } + } button.provider { text-align: start; font-weight: 400; + color: rgba(255, 255, 255, 0.8); &:not([disabled][disabled]) { border-color: rgba(255, 255, 255, .8); } diff --git a/ui-ngx/src/app/shared/components/phone-input.component.html b/ui-ngx/src/app/shared/components/phone-input.component.html index 6bf61ae4b8..1b025e8113 100644 --- a/ui-ngx/src/app/shared/components/phone-input.component.html +++ b/ui-ngx/src/app/shared/components/phone-input.component.html @@ -37,12 +37,12 @@ (focus)="focus()" autocomplete="off" [required]="required"> - + - {{ 'phone-input.phone-input-required' | translate }} + {{ requiredErrorText }} - {{ 'phone-input.phone-input-validation' | translate }} + {{ validationErrorText }}
diff --git a/ui-ngx/src/app/shared/components/phone-input.component.ts b/ui-ngx/src/app/shared/components/phone-input.component.ts index bd1124f97a..5d6d0b21a7 100644 --- a/ui-ngx/src/app/shared/components/phone-input.component.ts +++ b/ui-ngx/src/app/shared/components/phone-input.component.ts @@ -77,6 +77,15 @@ export class PhoneInputComponent implements OnInit, ControlValueAccessor, Valida @Input() label = this.translate.instant('phone-input.phone-input-label'); + @Input() + hint = 'phone-input.phone-input-hint'; + + @Input() + requiredErrorText = this.translate.instant('phone-input.phone-input-required'); + + @Input() + validationErrorText = this.translate.instant('phone-input.phone-input-validation'); + get showFlagSelect(): boolean { return this.enableFlagsSelect && !this.isLegacy; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 7c8989fe98..7f14588403 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3909,7 +3909,25 @@ "authenticator-backup-code-success": "Backup code successfully enabled", "authenticator-backup-code-success-description": "The next time you log in, you will be prompted to enter the security code or use one of backup code.", "add-verification-method": "Add verification method", - "get-backup-code": "Get backup code" + "get-backup-code": "Get backup code", + "copy-key": "Copy key", + "send-code": "Send code", + "email-label": "Email", + "sms-description": "Enter a phone number to use as your authenticator.", + "backup-code-description": "Print out the codes so you have them handy when you need to use them to log in to your account. You can use each backup code once.", + "backup-code-warn": "Once you leave this page, these codes cannot be shown again. Store them safely using the options below.", + "download-txt": "Download (txt)", + "print": "Print", + "verification-code": "6-digit code", + "verification-code-invalid": "Invalid verification code format", + "scan-qr-code": "Scan this QR code with your verification app", + "phone-input": { + "phone-input-label": "Phone number", + "phone-input-required": "Phone number is required", + "phone-input-validation": "Phone number is invalid or not possible", + "phone-input-pattern": "Invalid phone number. Should be in E.164 format, ex. {{phoneNumber}}", + "phone-input-hint": "Phone Number in E.164 format, ex. {{phoneNumber}}" + } }, "markdown": { "edit": "Edit", From 9f48acf05dbdef18780e4f8f1510f3742db32323 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 25 Sep 2025 15:36:26 +0300 Subject: [PATCH 13/88] preprovisioned device strategy fix: device should exists but now provisioned --- .../server/service/device/DeviceProvisionServiceImpl.java | 2 +- .../thingsboard/server/msa/connectivity/CoapClientTest.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java index 0778d61ee7..ce7abd3bf1 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java @@ -186,7 +186,7 @@ public class DeviceProvisionServiceImpl implements DeviceProvisionService { try { Optional provisionState = attributesService.find(device.getTenantId(), device.getId(), AttributeScope.SERVER_SCOPE, DEVICE_PROVISION_STATE).get(); - if (provisionState != null && provisionState.isPresent() && !provisionState.get().getValueAsString().equals(PROVISIONED_STATE)) { + if (provisionState != null && provisionState.isPresent() && provisionState.get().getValueAsString().equals(PROVISIONED_STATE)) { notify(device, provisionRequest, TbMsgType.PROVISION_FAILURE, false); throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); } else { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/CoapClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/CoapClientTest.java index 995b90e529..bc3d8b1b81 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/CoapClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/CoapClientTest.java @@ -60,6 +60,10 @@ public class CoapClientTest extends AbstractCoapClientTest{ assertThat(provisionResponse.get("credentialsValue").asText()).isEqualTo(expectedDeviceCredentials.getCredentialsId()); assertThat(provisionResponse.get("status").asText()).isEqualTo("SUCCESS"); + // provision second time should fail + JsonNode provisionResponse2 = JacksonUtil.fromBytes(createCoapClientAndPublish(device.getName())); + assertThat(provisionResponse2.get("status").asText()).isEqualTo("FAILURE"); + updateDeviceProfileWithProvisioningStrategy(deviceProfile, DeviceProfileProvisionType.DISABLED); } From e167110c6751ae2165bfd4f6620c6fd744a5a646 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Mon, 29 Sep 2025 14:26:44 +0300 Subject: [PATCH 14/88] fix_lwm2m: convert Time to Long --- .../lwm2m/server/LwM2mTransportServerHelper.java | 11 ++++++++++- .../lwm2m/utils/LwM2mValueConverterImpl.java | 7 ++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServerHelper.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServerHelper.java index ea3358de60..416547ef36 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServerHelper.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServerHelper.java @@ -38,6 +38,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.time.Instant; import java.util.ArrayList; +import java.util.Date; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; @@ -180,8 +181,16 @@ public class LwM2mTransportServerHelper { case BOOLEAN: kvProto.setType(BOOLEAN_V).setBoolV((Boolean) value).build(); break; - case STRING: case TIME: + if (value instanceof Date) { + kvProto.setType(TransportProtos.KeyValueType.LONG_V).setLongV(((Date) value).getTime()); + } else if (value instanceof Integer || value instanceof Long) { + kvProto.setType(TransportProtos.KeyValueType.LONG_V).setLongV((long) (value)); + } else { + kvProto.setType(TransportProtos.KeyValueType.STRING_V).setStringV(value.toString()); + } + break; + case STRING: case OPAQUE: case OBJLNK: kvProto.setType(TransportProtos.KeyValueType.STRING_V).setStringV((String) value); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/LwM2mValueConverterImpl.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/LwM2mValueConverterImpl.java index 91d55305da..9d496a2f88 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/LwM2mValueConverterImpl.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/LwM2mValueConverterImpl.java @@ -33,6 +33,7 @@ import java.util.Date; import static org.eclipse.leshan.core.model.ResourceModel.Type.NONE; import static org.eclipse.leshan.core.model.ResourceModel.Type.OPAQUE; +import static org.eclipse.leshan.core.model.ResourceModel.Type.TIME; @Slf4j public class LwM2mValueConverterImpl implements LwM2mValueConverter { @@ -58,7 +59,7 @@ public class LwM2mValueConverterImpl implements LwM2mValueConverter { currentType = OPAQUE; } - if (currentType == expectedType || currentType == NONE) { + if (currentType == expectedType || currentType == NONE || currentType == TIME) { /** expected type */ return value; } @@ -135,7 +136,7 @@ public class LwM2mValueConverterImpl implements LwM2mValueConverter { **/ } catch (IllegalArgumentException e) { log.debug("Unable to convert string to date", e); - throw new CodecException("Unable to convert string (%s) to date for resource %s", value, + throw new CodecException("Unable to convert string (%s) to %s for resource %s", value, TIME.name(), resourcePath); } default: @@ -149,7 +150,7 @@ public class LwM2mValueConverterImpl implements LwM2mValueConverter { case FLOAT: return String.valueOf(value); case TIME: - String DATE_FORMAT = "MMM d, yyyy HH:mm a"; + String DATE_FORMAT = "yyyy-MM-dd[[ ]['T']HH:mm[:ss[.SSS]][ ][XXX][Z][z][VV][O]]"; Long timeValue; try { timeValue = ((Date) value).getTime(); From 988495b4593b3ea409513c4d5c54caf49e7b8826 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Tue, 30 Sep 2025 13:16:22 +0300 Subject: [PATCH 15/88] fix_lwm2m: add to Front bootstrap Short Server id 0 or 65535 --- application/src/main/resources/thingsboard.yml | 2 +- .../lwm2m-device-config-server.component.html | 5 +++-- .../lwm2m/lwm2m-device-config-server.component.ts | 14 ++++++-------- .../src/assets/locale/locale.constant-en_US.json | 1 + 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 944520c4e5..69f0b29b74 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1193,7 +1193,7 @@ transport: # Enable/disable Bootstrap Server enabled: "${LWM2M_ENABLED_BS:true}" # Default value in LwM2M client after start in mode Bootstrap for the object : name "LWM2M Security" field: "Short Server ID" (deviceProfile: Bootstrap.BOOTSTRAP SERVER.Short ID) - id: "${LWM2M_SERVER_ID_BS:111}" + id: "${LWM2M_SERVER_ID_BS:0}" # LwM2M bootstrap server bind address. Bind to all interfaces by default bind_address: "${LWM2M_BS_BIND_ADDRESS:0.0.0.0}" # LwM2M bootstrap server bind port diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.html b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.html index e4d5135e74..fc1f6adf47 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.html @@ -58,12 +58,13 @@ {{ 'device-profile.lwm2m.short-id' | translate }} help - + + {{ 'device-profile.lwm2m.short-id-required' | translate }} - {{ 'device-profile.lwm2m.short-id-pattern' | translate }} + {{ (isBootstrap ? 'device-profile.lwm2m.short-id-pattern-bs' : 'device-profile.lwm2m.short-id-pattern') | translate }} diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts index 6c0f87c058..b2c5a2fc07 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts @@ -74,8 +74,10 @@ export class Lwm2mDeviceConfigServerComponent implements OnInit, ControlValueAcc currentSecurityMode = null; bootstrapDisabled = false; - shortServerIdMin = 1; - shortServerIdMax = 65534; + readonly shortServerIdMin = 1; + readonly shortServerIdMax = 65534; + readonly shortServerIdBsMin = 0; + readonly shortServerIdBsMax = 65535; @Input() @coerceBoolean() @@ -94,18 +96,13 @@ export class Lwm2mDeviceConfigServerComponent implements OnInit, ControlValueAcc } ngOnInit(): void { - if (this.isBootstrap) { - this.shortServerIdMin = 0; - this.shortServerIdMax = 65535; - } this.serverFormGroup = this.fb.group({ host: ['', Validators.required], port: ['', [Validators.required, Validators.min(1), Validators.max(65535), Validators.pattern('[0-9]*')]], securityMode: [Lwm2mSecurityType.NO_SEC], serverPublicKey: [''], clientHoldOffTime: ['', [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], - shortServerId: ['', - [Validators.required, Validators.min(this.shortServerIdMin), Validators.max(this.shortServerIdMax), Validators.pattern('[0-9]*')]], + shortServerId: ['', [Validators.required, Validators.pattern(this.isBootstrap ? '^(0|65535)$' : '[0-9]*')]], bootstrapServerAccountTimeout: ['', [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], binding: [''], lifetime: [null, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], @@ -129,6 +126,7 @@ export class Lwm2mDeviceConfigServerComponent implements OnInit, ControlValueAcc this.serverFormGroup.get('serverPublicKey').patchValue(serverSecurityConfig.serverCertificate, {emitEvent: false}); } }); + this.serverFormGroup.valueChanges.pipe( takeUntil(this.destroy$) ).subscribe(value => { diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 1866ab7d0b..05a1f5ceb3 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -2302,6 +2302,7 @@ "short-id-required": "Short server ID is required.", "short-id-range": "Short server ID should be in a range from {{ min }} to {{ max }}.", "short-id-pattern": "Short server ID must be a positive integer.", + "short-id-pattern-bs": "Short server ID must be only 0 or 65535", "lifetime": "Client registration lifetime", "lifetime-required": "Client registration lifetime is required.", "lifetime-pattern": "Client registration lifetime must be a positive integer.", From 53ff45d43755343eb26a87fe0322a5f4b9c9307e Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Tue, 30 Sep 2025 14:22:47 +0300 Subject: [PATCH 16/88] fix_lwm2m:comments #1 --- .../lwm2m/lwm2m-device-config-server.component.html | 3 +-- .../device/lwm2m/lwm2m-device-config-server.component.ts | 9 +++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.html b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.html index fc1f6adf47..f1cafd7e81 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.html @@ -58,8 +58,7 @@ {{ 'device-profile.lwm2m.short-id' | translate }} help - - + {{ 'device-profile.lwm2m.short-id-required' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts index b2c5a2fc07..7d0ce0f887 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts @@ -98,11 +98,16 @@ export class Lwm2mDeviceConfigServerComponent implements OnInit, ControlValueAcc ngOnInit(): void { this.serverFormGroup = this.fb.group({ host: ['', Validators.required], - port: ['', [Validators.required, Validators.min(1), Validators.max(65535), Validators.pattern('[0-9]*')]], + port: ['', [Validators.required, Validators.min(5683), Validators.max(5688), Validators.pattern('[0-9]*')]], securityMode: [Lwm2mSecurityType.NO_SEC], serverPublicKey: [''], clientHoldOffTime: ['', [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], - shortServerId: ['', [Validators.required, Validators.pattern(this.isBootstrap ? '^(0|65535)$' : '[0-9]*')]], + // shortServerId: ['', [Validators.required, Validators.pattern(this.isBootstrap ? '^(' + this.shortServerIdBsMin+ '|' + this.shortServerIdBsMax + ')$' : '[0-9]*'), !this.isBootstrap ? [Validators.min(this.shortServerIdMin), Validators.max(this.shortServerIdMax)] : [] ]], + shortServerId: ['', [Validators.required, Validators.pattern(this.isBootstrap ? '^(' + this.shortServerIdBsMin+ '|' + this.shortServerIdBsMax + ')$' : '[0-9]*'), + ...(!this.isBootstrap ? [ + Validators.min(this.shortServerIdMin), + Validators.max(this.shortServerIdMax) + ] : []) ]], bootstrapServerAccountTimeout: ['', [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], binding: [''], lifetime: [null, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], From 4546597339ddd62d60c068424d7dfae6514ef275 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Tue, 30 Sep 2025 15:10:07 +0300 Subject: [PATCH 17/88] UI: Improved validation shortServerId in lwm2m transport configuration --- .../lwm2m/lwm2m-device-config-server.component.html | 4 +++- .../lwm2m/lwm2m-device-config-server.component.ts | 12 +++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.html b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.html index f1cafd7e81..65a2400e16 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.html @@ -58,7 +58,9 @@ {{ 'device-profile.lwm2m.short-id' | translate }} help - + {{ 'device-profile.lwm2m.short-id-required' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts index 7d0ce0f887..1630a1d220 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts @@ -98,16 +98,14 @@ export class Lwm2mDeviceConfigServerComponent implements OnInit, ControlValueAcc ngOnInit(): void { this.serverFormGroup = this.fb.group({ host: ['', Validators.required], - port: ['', [Validators.required, Validators.min(5683), Validators.max(5688), Validators.pattern('[0-9]*')]], + port: ['', [Validators.required, Validators.min(1), Validators.max(65535), Validators.pattern('[0-9]*')]], securityMode: [Lwm2mSecurityType.NO_SEC], serverPublicKey: [''], clientHoldOffTime: ['', [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], - // shortServerId: ['', [Validators.required, Validators.pattern(this.isBootstrap ? '^(' + this.shortServerIdBsMin+ '|' + this.shortServerIdBsMax + ')$' : '[0-9]*'), !this.isBootstrap ? [Validators.min(this.shortServerIdMin), Validators.max(this.shortServerIdMax)] : [] ]], - shortServerId: ['', [Validators.required, Validators.pattern(this.isBootstrap ? '^(' + this.shortServerIdBsMin+ '|' + this.shortServerIdBsMax + ')$' : '[0-9]*'), - ...(!this.isBootstrap ? [ - Validators.min(this.shortServerIdMin), - Validators.max(this.shortServerIdMax) - ] : []) ]], + shortServerId: ['', this.isBootstrap + ? [Validators.required, Validators.pattern('^(' + this.shortServerIdBsMin+ '|' + this.shortServerIdBsMax + ')$' )] + : [Validators.required, Validators.pattern('[0-9]*'),Validators.min(this.shortServerIdMin), Validators.max(this.shortServerIdMax)] + ], bootstrapServerAccountTimeout: ['', [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], binding: [''], lifetime: [null, [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], From 19ef1e357712823daadfda1620d733e5b11b5125 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Fri, 3 Oct 2025 11:18:56 +0300 Subject: [PATCH 18/88] Fix tenant admins filter for enforced 2FA --- .../java/org/thingsboard/server/dao/user/UserServiceImpl.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index 7b5e4da6a0..4a83c71d66 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -556,6 +556,9 @@ public class UserServiceImpl extends AbstractCachedEntityService { + if (user.isSystemAdmin() || user.isCustomerUser()) { + return false; + } TenantAdministratorsFilter tenantAdministratorsFilter = (TenantAdministratorsFilter) filter; if (isNotEmpty(tenantAdministratorsFilter.getTenantsIds())) { return tenantAdministratorsFilter.getTenantsIds().contains(user.getTenantId().getId()); From 6c6ebee77eb4cb3da42477f8d8fc8c654a713c3c Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Thu, 9 Oct 2025 15:18:08 +0300 Subject: [PATCH 19/88] Fix CVE-2025-4641 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1283dcc01c..8cbd01653b 100755 --- a/pom.xml +++ b/pom.xml @@ -129,7 +129,7 @@ 1.20.6 1.0.2 1.12 - 5.8.0 + 6.1.0 2.27.0 2.12.0 From 9ee28a3e7abccb187a4ffd5a895a26597af90d37 Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Thu, 9 Oct 2025 15:29:38 +0300 Subject: [PATCH 20/88] Fix CVE-2025-58056, CVE-2025-58057 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8cbd01653b..ae11511a5a 100755 --- a/pom.xml +++ b/pom.xml @@ -146,7 +146,7 @@ 9.2.0 1.1.10.5 9.10.0 - 4.1.124.Final + 4.1.125.Final From 15aa5fa74ea9d4e128387b8f1d71359b56bcd9fa Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Thu, 9 Oct 2025 15:30:13 +0300 Subject: [PATCH 21/88] Fix CVE-2025-48989, CVE-2025-41242, CVE-2025-41249, CVE-2025-41248 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ae11511a5a..3dd01e29e8 100755 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ ${project.name} /var/log/${pkg.name} /usr/share/${pkg.name} - 3.4.8 + 3.4.10 2.4.0-b180830.0359 5.1.5 0.12.5 From fc6ba08bb25d1250d71477c6c3a0835fb7033f5c Mon Sep 17 00:00:00 2001 From: VIacheslavKlimov Date: Thu, 9 Oct 2025 15:36:14 +0300 Subject: [PATCH 22/88] Add comment for netty.version property --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3dd01e29e8..ce9d666615 100755 --- a/pom.xml +++ b/pom.xml @@ -146,7 +146,7 @@ 9.2.0 1.1.10.5 9.10.0 - 4.1.125.Final + 4.1.125.Final From f19a1ba2a7632a733d7ec909b347f576953db4f6 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Thu, 9 Oct 2025 18:58:27 +0300 Subject: [PATCH 23/88] lwm2m: bootstrap new --- .../lwm2m/AbstractLwM2MIntegrationTest.java | 3 +- .../transport/lwm2m/Lwm2mTestHelper.java | 2 - .../lwm2m/client/LwM2MTestClient.java | 25 +- .../rpc/AbstractRpcLwM2MIntegrationTest.java | 11 +- .../rpc/sql/RpcLwm2mIntegrationReadTest.java | 32 +- .../AbstractSecurityLwM2MIntegrationTest.java | 20 +- .../NoSecLwM2MIntegrationBSNoTriggerTest.java | 40 +++ .../NoSecLwM2MIntegrationBSTriggerTest.java | 61 ++++ .../sql/NoSecLwM2MIntegrationTest.java | 50 --- .../security/sql/PskLwm2mIntegrationTest.java | 4 +- .../security/sql/RpkLwM2MIntegrationTest.java | 4 +- .../sql/X509_NoTrustLwM2MIntegrationTest.java | 5 +- .../LwM2mDefaultBootstrapSessionManager.java | 17 +- ...LwM2MBootstrapConfigStoreTaskProvider.java | 307 ++++++++---------- .../lwm2m/utils/LwM2MTransportUtil.java | 2 + .../lwm2m-device-config-server.component.ts | 2 +- 16 files changed, 313 insertions(+), 272 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSNoTriggerTest.java create mode 100644 application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSTriggerTest.java diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java index ddf8bca43f..50051c08d5 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java @@ -147,7 +147,6 @@ public abstract class AbstractLwM2MIntegrationTest extends AbstractTransportInte public static final Integer shortServerId = 123; public static final Integer shortServerIdBs0 = 0; public static final int serverId = 1; - public static final int serverIdBs = 0; public static final String COAP = "coap://"; public static final String COAPS = "coaps://"; @@ -705,7 +704,7 @@ public abstract class AbstractLwM2MIntegrationTest extends AbstractTransportInte return bootstrap; } - private AbstractLwM2MBootstrapServerCredential getBootstrapServerCredentialNoSec(boolean isBootstrap) { + protected AbstractLwM2MBootstrapServerCredential getBootstrapServerCredentialNoSec(boolean isBootstrap) { AbstractLwM2MBootstrapServerCredential bootstrapServerCredential = new NoSecLwM2MBootstrapServerCredential(); bootstrapServerCredential.setServerPublicKey(""); bootstrapServerCredential.setShortServerId(isBootstrap ? shortServerIdBs0 : shortServerId); diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/Lwm2mTestHelper.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/Lwm2mTestHelper.java index 6159400bb6..ce073d137d 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/Lwm2mTestHelper.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/Lwm2mTestHelper.java @@ -24,8 +24,6 @@ public class Lwm2mTestHelper { public static final int TEMPERATURE_SENSOR = 3303; // Ids in Client - public static final int OBJECT_ID_0 = 0; - public static final int OBJECT_ID_1 = 1; public static final int OBJECT_INSTANCE_ID_0 = 0; public static final int OBJECT_INSTANCE_ID_1 = 1; public static final int OBJECT_INSTANCE_ID_2 = 2; diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java index 2ae1432cac..7c5e2973ac 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java @@ -88,10 +88,7 @@ import static org.eclipse.leshan.core.LwM2mId.SECURITY; import static org.eclipse.leshan.core.LwM2mId.SERVER; import static org.eclipse.leshan.core.LwM2mId.SOFTWARE_MANAGEMENT; import static org.eclipse.leshan.core.node.codec.DefaultLwM2mEncoder.getDefaultPathEncoder; -import static org.thingsboard.server.transport.lwm2m.AbstractLwM2MIntegrationTest.serverId; -import static org.thingsboard.server.transport.lwm2m.AbstractLwM2MIntegrationTest.serverIdBs; import static org.thingsboard.server.transport.lwm2m.AbstractLwM2MIntegrationTest.shortServerId; -import static org.thingsboard.server.transport.lwm2m.AbstractLwM2MIntegrationTest.shortServerIdBs0; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.BINARY_APP_DATA_CONTAINER; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MClientState; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_BOOTSTRAP_FAILURE; @@ -169,31 +166,29 @@ public class LwM2MTestClient { LwM2mModel model = new StaticModel(models); ObjectsInitializer initializer = new ObjectsInitializer(model); if (securityBs != null && security != null) { - // SECURITY - security.setId(serverId); - securityBs.setId(serverIdBs); + // SECURITIES + securityBs.setId(0); + security.setId(1); LwM2mInstanceEnabler[] instances = new LwM2mInstanceEnabler[]{securityBs, security}; - initializer.setClassForObject(SECURITY, Security.class); initializer.setInstancesForObject(SECURITY, instances); // SERVER Server lwm2mServer = new Server(shortServerId, TimeUnit.MINUTES.toSeconds(60)); - lwm2mServer.setId(serverId); - Server serverBs = new Server(shortServerIdBs0, TimeUnit.MINUTES.toSeconds(60)); - serverBs.setId(serverIdBs); - instances = new LwM2mInstanceEnabler[]{serverBs, lwm2mServer}; - initializer.setClassForObject(SERVER, Server.class); + lwm2mServer.setId(0); + instances = new LwM2mInstanceEnabler[]{lwm2mServer}; + initializer.setInstancesForObject(SERVER, instances); } else if (securityBs != null) { // SECURITY - initializer.setInstancesForObject(SECURITY, securityBs); - // SERVER + securityBs.setId(0); initializer.setClassForObject(SERVER, Server.class); + initializer.setInstancesForObject(SECURITY, securityBs); } else { // SECURITY + security.setId(0); initializer.setInstancesForObject(SECURITY, security); // SERVER Server lwm2mServer = new Server(shortServerId, TimeUnit.MINUTES.toSeconds(60)); - lwm2mServer.setId(serverId); + lwm2mServer.setId(0); initializer.setInstancesForObject(SERVER, lwm2mServer); } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/AbstractRpcLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/AbstractRpcLwM2MIntegrationTest.java index be85294f08..1253993e08 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/AbstractRpcLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/AbstractRpcLwM2MIntegrationTest.java @@ -43,12 +43,11 @@ import static org.awaitility.Awaitility.await; import static org.eclipse.leshan.core.LwM2mId.ACCESS_CONTROL; import static org.eclipse.leshan.core.LwM2mId.DEVICE; import static org.eclipse.leshan.core.LwM2mId.FIRMWARE; +import static org.eclipse.leshan.core.LwM2mId.SECURITY; import static org.eclipse.leshan.core.LwM2mId.SERVER; import static org.eclipse.leshan.core.LwM2mId.SOFTWARE_MANAGEMENT; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.BINARY_APP_DATA_CONTAINER; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.NONE; -import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.OBJECT_ID_0; -import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.OBJECT_ID_1; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.OBJECT_INSTANCE_ID_0; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.OBJECT_INSTANCE_ID_1; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.OBJECT_INSTANCE_ID_12; @@ -140,10 +139,10 @@ public abstract class AbstractRpcLwM2MIntegrationTest extends AbstractLwM2MInteg }); } }); - String ver_Id_0 = lwM2MTestClient.getLeshanClient().getObjectTree().getModel().getObjectModel(OBJECT_ID_0).version; - String ver_Id_1 = lwM2MTestClient.getLeshanClient().getObjectTree().getModel().getObjectModel(OBJECT_ID_1).version; - objectIdVer_0 = "/" + OBJECT_ID_0 + "_" + ver_Id_0; - objectIdVer_1 = "/" + OBJECT_ID_1 + "_" + ver_Id_1; + String ver_Id_0 = lwM2MTestClient.getLeshanClient().getObjectTree().getModel().getObjectModel(SECURITY).version; + String ver_Id_1 = lwM2MTestClient.getLeshanClient().getObjectTree().getModel().getObjectModel(SERVER).version; + objectIdVer_0 = "/" + SECURITY + "_" + ver_Id_0; + objectIdVer_1 = "/" + SERVER + "_" + ver_Id_1; objectIdVer_2 = (String) expectedObjectIdVers.stream().filter(path -> ((String) path).startsWith("/" + ACCESS_CONTROL)).findFirst().get(); objectIdVer_3 = (String) expectedObjectIdVers.stream().filter(PREDICATE_3).findFirst().get(); objectIdVer_19 = (String) expectedObjectIdVers.stream().filter(path -> ((String) path).startsWith("/" + BINARY_APP_DATA_CONTAINER)).findFirst().get(); diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationReadTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationReadTest.java index 90b373b16c..c0779ad944 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationReadTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationReadTest.java @@ -21,8 +21,10 @@ import org.eclipse.leshan.core.ResponseCode; import org.eclipse.leshan.core.node.LwM2mPath; import org.junit.Test; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.transport.lwm2m.rpc.AbstractRpcLwM2MIntegrationTest; +import static org.eclipse.leshan.core.LwM2mId.ACCESS_CONTROL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -53,18 +55,18 @@ public class RpcLwm2mIntegrationReadTest extends AbstractRpcLwM2MIntegrationTest try { expectedObjectIdVers.forEach(expected -> { try { - String actualResult = sendRPCById((String) expected); String expectedObjectId = pathIdVerToObjectId((String) expected); LwM2mPath expectedPath = new LwM2mPath(expectedObjectId); - ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - String expectedObjectInstances = "LwM2mObject [id=" + expectedPath.getObjectId() + ", instances={0=LwM2mObjectInstance [id=0, resources="; - if (expectedPath.getObjectId() == 1) { - expectedObjectInstances = "LwM2mObject [id=1, instances={1="; - } else if (expectedPath.getObjectId() == 2) { - expectedObjectInstances = "LwM2mObject [id=2, instances={}]"; + if (expectedPath.getObjectId() > ACCESS_CONTROL) { + String actualResult = sendRPCByIdSync((String) expected); + if (StringUtils.isNoneBlank(actualResult)) { + log.warn(" expectedPath: [{}]", expectedPath); + ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); + assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); + String expectedObjectInstances = "LwM2mObject [id=" + expectedPath.getObjectId() + ", instances={0=LwM2mObjectInstance [id=0, resources="; + assertTrue(rpcActualResult.get("value").asText().contains(expectedObjectInstances)); + } } - assertTrue(rpcActualResult.get("value").asText().contains(expectedObjectInstances)); } catch (Exception e) { e.printStackTrace(); } @@ -83,7 +85,7 @@ public class RpcLwm2mIntegrationReadTest extends AbstractRpcLwM2MIntegrationTest public void testReadAllInstancesInClientById_Result_CONTENT_Value_IsInstances_IsResources() throws Exception { expectedObjectIdVerInstances.forEach(expected -> { try { - String actualResult = sendRPCById((String) expected); + String actualResult = sendRPCByIdAsync((String) expected); String expectedObjectId = pathIdVerToObjectId((String) expected); LwM2mPath expectedPath = new LwM2mPath(expectedObjectId); ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); @@ -104,7 +106,7 @@ public class RpcLwm2mIntegrationReadTest extends AbstractRpcLwM2MIntegrationTest @Test public void testReadMultipleResourceById_Result_CONTENT_Value_IsLwM2mMultipleResource() throws Exception { String expectedIdVer = objectInstanceIdVer_3 + "/" + RESOURCE_ID_11; - String actualResult = sendRPCById(expectedIdVer); + String actualResult = sendRPCByIdAsync(expectedIdVer); ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); String expected = "LwM2mMultipleResource [id=" + RESOURCE_ID_11 + ", values={"; @@ -117,7 +119,7 @@ public class RpcLwm2mIntegrationReadTest extends AbstractRpcLwM2MIntegrationTest @Test public void testReadSingleResourceById_Result_CONTENT_Value_IsLwM2mSingleResource() throws Exception { String expectedIdVer = objectInstanceIdVer_3 + "/" + RESOURCE_ID_14; - String actualResult = sendRPCById(expectedIdVer); + String actualResult = sendRPCByIdAsync(expectedIdVer); ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); String expected = "LwM2mSingleResource [id=" + RESOURCE_ID_14 + ", value="; @@ -228,10 +230,14 @@ public class RpcLwm2mIntegrationReadTest extends AbstractRpcLwM2MIntegrationTest assertEquals(actualValue, expectedValue); } - private String sendRPCById(String path) throws Exception { + private String sendRPCByIdAsync(String path) throws Exception { String setRpcRequest = "{\"method\": \"Read\", \"params\": {\"id\": \"" + path + "\"}}"; return doPostAsync("/api/plugins/rpc/twoway/" + lwM2MTestClient.getDeviceIdStr(), setRpcRequest, String.class, status().isOk()); } + private String sendRPCByIdSync(String path) throws Exception { + String setRpcRequest = "{\"method\": \"Read\", \"params\": {\"id\": \"" + path + "\"}}"; + return doPost("/api/plugins/rpc/twoway/" + lwM2MTestClient.getDeviceIdStr(), setRpcRequest, String.class, status().isOk()); + } private String sendRPCByKey(String key) throws Exception { String setRpcRequest = "{\"method\": \"Read\", \"params\": {\"key\": \"" + key + "\"}}"; diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/AbstractSecurityLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/AbstractSecurityLwM2MIntegrationTest.java index 6dda2de9fe..264c3b84c7 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/AbstractSecurityLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/AbstractSecurityLwM2MIntegrationTest.java @@ -69,6 +69,7 @@ import java.util.concurrent.TimeUnit; import static org.awaitility.Awaitility.await; import static org.eclipse.leshan.client.object.Security.noSecBootstrap; import static org.eclipse.leshan.client.object.Security.psk; +import static org.eclipse.leshan.core.LwM2mId.SERVER; import static org.junit.Assert.assertEquals; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode.PSK; @@ -77,7 +78,7 @@ import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MClient import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_REGISTRATION_STARTED; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_REGISTRATION_SUCCESS; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_UPDATE_SUCCESS; -import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.OBJECT_ID_1; +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.OBJECT_INSTANCE_ID_0; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_9; @DaoSqlTest @@ -191,7 +192,7 @@ public abstract class AbstractSecurityLwM2MIntegrationTest extends AbstractLwM2M transportConfiguration, awaitAlias, expectedStatuses, - true, + false, finishState, false); } @@ -231,8 +232,15 @@ public abstract class AbstractSecurityLwM2MIntegrationTest extends AbstractLwM2M } - public void basicTestConnectionBootstrapRequestTriggerBefore(String clientEndpoint, String awaitAlias, LwM2MProfileBootstrapConfigType type) throws Exception { - Lwm2mDeviceProfileTransportConfiguration transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsNoSec(type)); + public void basicTestConnectionBootstrapRequestTriggerBefore(String clientEndpoint, String awaitAlias, LwM2MProfileBootstrapConfigType type, int cnt) throws Exception { + List bootstrapServerCredentialsNoSec = getBootstrapServerCredentialsNoSec(type); + for (int i = 2; i <= cnt; i++) { + AbstractLwM2MBootstrapServerCredential bsCredential = getBootstrapServerCredentialNoSec(false); + bsCredential.setHost("0.0.0." + i); + bsCredential.setShortServerId(bsCredential.getShortServerId() + i); + bootstrapServerCredentialsNoSec.add(bsCredential); + } + Lwm2mDeviceProfileTransportConfiguration transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, bootstrapServerCredentialsNoSec); LwM2MDeviceCredentials deviceCredentials = getDeviceCredentialsNoSec(createNoSecClientCredentials(clientEndpoint)); this.basicTestConnectionBootstrapRequestTrigger( SECURITY_NO_SEC, @@ -275,8 +283,8 @@ public abstract class AbstractSecurityLwM2MIntegrationTest extends AbstractLwM2M }); Assert.assertTrue(lwM2MTestClient.getClientStates().containsAll(expectedStatusesLwm2m)); - String executedPath = "/" + OBJECT_ID_1 + "_" + lwM2MTestClient.getLeshanClient().getObjectTree().getModel().getObjectModel(OBJECT_ID_1).version - + "/" +serverId + "/" + RESOURCE_ID_9; + String executedPath = "/" + SERVER + "_" + lwM2MTestClient.getLeshanClient().getObjectTree().getModel().getObjectModel(SERVER).version + + "/" + OBJECT_INSTANCE_ID_0 + "/" + RESOURCE_ID_9; lwM2MTestClient.setClientStates(new HashSet<>()); String actualResult = sendRPCSecurityExecuteById(executedPath, deviceIdStr, endpoint); ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSNoTriggerTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSNoTriggerTest.java new file mode 100644 index 0000000000..2622cac18c --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSNoTriggerTest.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.security.sql; + +import org.junit.Test; +import org.thingsboard.server.transport.lwm2m.security.AbstractSecurityLwM2MIntegrationTest; + +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_REGISTRATION_SUCCESS; +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.BOTH; +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.LWM2M_ONLY; + +public class NoSecLwM2MIntegrationBSNoTriggerTest extends AbstractSecurityLwM2MIntegrationTest { + + @Test + public void testWithNoSecConnectBsSuccess_UpdateTwoSectionsBootstrapAndLm2m_ConnectLwm2mSuccess() throws Exception { + String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "NoTrigger" + BOTH.name(); + String awaitAlias = "await on client state (NoSecBS two section)"; + basicTestConnectionBefore(clientEndpoint, awaitAlias, BOTH, expectedStatusesRegistrationBsSuccess, ON_REGISTRATION_SUCCESS); + } + + @Test + public void testWithNoSecConnectBsSuccess_UpdateLwm2mSectionAndLm2m_ConnectLwm2mSuccess() throws Exception { + String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "NoTrigger" + LWM2M_ONLY.name(); + String awaitAlias = "await on client state (NoSecBS Lwm2m section)"; + basicTestConnectionBefore(clientEndpoint, awaitAlias, LWM2M_ONLY, expectedStatusesRegistrationBsSuccess, ON_REGISTRATION_SUCCESS); + } +} diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSTriggerTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSTriggerTest.java new file mode 100644 index 0000000000..af5571864f --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSTriggerTest.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.security.sql; + +import org.junit.Test; +import org.thingsboard.server.transport.lwm2m.security.AbstractSecurityLwM2MIntegrationTest; + +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.BOOTSTRAP_ONLY; +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.BOTH; +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.LWM2M_ONLY; +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.NONE; + +public class NoSecLwM2MIntegrationBSTriggerTest extends AbstractSecurityLwM2MIntegrationTest { + + @Test + public void testWithNoSecConnectLwm2mSuccessBootstrapRequestTriggerConnectBsSuccess_UpdateBootstrapSectionAndLm2m_ConnectLwm2mSuccess() throws Exception { + String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "Trigger" + BOOTSTRAP_ONLY.name(); + String awaitAlias = "await on client state (NoSecBS Trigger Bootstrap section)"; + basicTestConnectionBootstrapRequestTriggerBefore(clientEndpoint, awaitAlias, BOOTSTRAP_ONLY, 1); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessBootstrapRequestTriggerConnectBsSuccess_UpdateTwoSectionAndLm2m_ConnectLwm2mSuccess() throws Exception { + String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "Trigger" + BOTH.name(); + String awaitAlias = "await on client state (NoSecBS Trigger Two section)"; + basicTestConnectionBootstrapRequestTriggerBefore(clientEndpoint, awaitAlias, BOTH, 1); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessBootstrapRequestTriggerConnectBsSuccess_UpdateLwm2mSectionAndLm2m_ConnectLwm2mSuccess() throws Exception { + String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "Trigger" + LWM2M_ONLY.name(); + String awaitAlias = "await on client state (NoSecBS Trigger Lwm2m section)"; + basicTestConnectionBootstrapRequestTriggerBefore(clientEndpoint, awaitAlias, LWM2M_ONLY, 1); + } + @Test + public void testWithNoSecConnectLwm2mSuccessBootstrapRequestTriggerConnectBsSuccess_UpdateLwm2mSection_3_AndLm2m_1_ConnectLwm2mSuccess() throws Exception { + String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "Trigger" + LWM2M_ONLY.name(); + String awaitAlias = "await on client state (NoSecBS Trigger Lwm2m section)"; + basicTestConnectionBootstrapRequestTriggerBefore(clientEndpoint, awaitAlias, LWM2M_ONLY, 3); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessBootstrapRequestTriggerConnectBsSuccess_UpdateNoneSectionAndLm2m_ConnectLwm2mSuccess() throws Exception { + String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "Trigger" + NONE.name(); + String awaitAlias = "await on client state (NoSecBS Trigger None section)"; + basicTestConnectionBootstrapRequestTriggerBefore(clientEndpoint, awaitAlias, NONE, 1); + } +} diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationTest.java index d4a464b9c6..50bb87467f 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationTest.java @@ -19,12 +19,6 @@ import org.junit.Test; import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MDeviceCredentials; import org.thingsboard.server.transport.lwm2m.security.AbstractSecurityLwM2MIntegrationTest; -import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_REGISTRATION_SUCCESS; -import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.BOOTSTRAP_ONLY; -import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.BOTH; -import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.LWM2M_ONLY; -import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.NONE; - public class NoSecLwM2MIntegrationTest extends AbstractSecurityLwM2MIntegrationTest { //Lwm2m only @@ -34,48 +28,4 @@ public class NoSecLwM2MIntegrationTest extends AbstractSecurityLwM2MIntegrationT LwM2MDeviceCredentials clientCredentials = getDeviceCredentialsNoSec(createNoSecClientCredentials(clientEndpoint)); super.basicTestConnectionObserveSingleTelemetry(SECURITY_NO_SEC, clientCredentials, clientEndpoint, false, false); } - - // Bootstrap + Lwm2m - @Test - public void testWithNoSecConnectBsSuccess_UpdateTwoSectionsBootstrapAndLm2m_ConnectLwm2mSuccess() throws Exception { - String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + BOTH.name(); - String awaitAlias = "await on client state (NoSecBS two section)"; - basicTestConnectionBefore(clientEndpoint, awaitAlias, BOTH, expectedStatusesRegistrationBsSuccess, ON_REGISTRATION_SUCCESS); - } - - @Test - public void testWithNoSecConnectBsSuccess_UpdateLwm2mSectionAndLm2m_ConnectLwm2mSuccess() throws Exception { - String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + LWM2M_ONLY.name(); - String awaitAlias = "await on client state (NoSecBS Lwm2m section)"; - basicTestConnectionBefore(clientEndpoint, awaitAlias, LWM2M_ONLY, expectedStatusesRegistrationBsSuccess, ON_REGISTRATION_SUCCESS); - } - - // Bs trigger - @Test - public void testWithNoSecConnectLwm2mSuccessBootstrapRequestTriggerConnectBsSuccess_UpdateTwoSectionAndLm2m_ConnectLwm2mSuccess() throws Exception { - String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "Trigger" + BOTH.name(); - String awaitAlias = "await on client state (NoSecBS Trigger Two section)"; - basicTestConnectionBootstrapRequestTriggerBefore(clientEndpoint, awaitAlias, BOTH); - } - - @Test - public void testWithNoSecConnectLwm2mSuccessBootstrapRequestTriggerConnectBsSuccess_UpdateBootstrapSectionAndLm2m_ConnectLwm2mSuccess() throws Exception { - String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "Trigger" + BOOTSTRAP_ONLY.name(); - String awaitAlias = "await on client state (NoSecBS Trigger Bootstrap section)"; - basicTestConnectionBootstrapRequestTriggerBefore(clientEndpoint, awaitAlias, BOOTSTRAP_ONLY); - } - - @Test - public void testWithNoSecConnectLwm2mSuccessBootstrapRequestTriggerConnectBsSuccess_UpdateLwm2mSectionAndLm2m_ConnectLwm2mSuccess() throws Exception { - String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "Trigger" + LWM2M_ONLY.name(); - String awaitAlias = "await on client state (NoSecBS Trigger Lwm2m section)"; - basicTestConnectionBootstrapRequestTriggerBefore(clientEndpoint, awaitAlias, LWM2M_ONLY); - } - - @Test - public void testWithNoSecConnectLwm2mSuccessBootstrapRequestTriggerConnectBsSuccess_UpdateNoneSectionAndLm2m_ConnectLwm2mSuccess() throws Exception { - String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "Trigger" + NONE.name(); - String awaitAlias = "await on client state (NoSecBS Trigger None section)"; - basicTestConnectionBootstrapRequestTriggerBefore(clientEndpoint, awaitAlias, NONE); - } } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/PskLwm2mIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/PskLwm2mIntegrationTest.java index 3b61dfe49f..e4bf935306 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/PskLwm2mIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/PskLwm2mIntegrationTest.java @@ -175,12 +175,12 @@ public class PskLwm2mIntegrationTest extends AbstractSecurityLwM2MIntegrationTes clientCredentials.setEndpoint(clientEndpoint); clientCredentials.setIdentity(identity); clientCredentials.setKey(keyPsk); - Security securityBs = pskBootstrap(SECURE_URI_BS, + Security securityPskBs = pskBootstrap(SECURE_URI_BS, identity.getBytes(StandardCharsets.UTF_8), Hex.decodeHex(keyPsk.toCharArray())); Lwm2mDeviceProfileTransportConfiguration transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(PSK, BOTH)); LwM2MDeviceCredentials deviceCredentials = getDeviceCredentialsSecure(clientCredentials, null, null, PSK, false); - this.basicTestConnection(null, securityBs, + this.basicTestConnection(null, securityPskBs, deviceCredentials, clientEndpoint, transportConfiguration, diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/RpkLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/RpkLwM2MIntegrationTest.java index e94b8a6319..fbe84a28b2 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/RpkLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/RpkLwM2MIntegrationTest.java @@ -113,13 +113,13 @@ public class RpkLwM2MIntegrationTest extends AbstractSecurityLwM2MIntegrationTes RPKClientCredential clientCredentials = new RPKClientCredential(); clientCredentials.setEndpoint(clientEndpoint); clientCredentials.setKey(Base64.encodeBase64String(certificate.getPublicKey().getEncoded())); - Security securityBs = rpkBootstrap(SECURE_URI_BS, + Security securityRpkBs = rpkBootstrap(SECURE_URI_BS, certificate.getPublicKey().getEncoded(), privateKey.getEncoded(), serverX509CertBs.getPublicKey().getEncoded()); Lwm2mDeviceProfileTransportConfiguration transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(RPK, BOTH)); LwM2MDeviceCredentials deviceCredentials = getDeviceCredentialsSecure(clientCredentials, clientPrivateKeyFromCertTrust, certificate, RPK, false); - this.basicTestConnection(null, securityBs, + this.basicTestConnection(null, securityRpkBs, deviceCredentials, clientEndpoint, transportConfiguration, diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/X509_NoTrustLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/X509_NoTrustLwM2MIntegrationTest.java index 1ba408ac70..d1576ede1b 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/X509_NoTrustLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/X509_NoTrustLwM2MIntegrationTest.java @@ -51,15 +51,14 @@ public class X509_NoTrustLwM2MIntegrationTest extends AbstractSecurityLwM2MInteg X509ClientCredential clientCredentials = new X509ClientCredential(); clientCredentials.setEndpoint(clientEndpoint); clientCredentials.setCert(Base64.getEncoder().encodeToString(certificate.getEncoded())); - Security security = x509(SECURE_URI, + Security securityX509 = x509(SECURE_URI, shortServerId, certificate.getEncoded(), privateKey.getEncoded(), serverX509Cert.getEncoded()); Lwm2mDeviceProfileTransportConfiguration transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(X509, NONE)); LwM2MDeviceCredentials deviceCredentials = getDeviceCredentialsSecure(clientCredentials, privateKey, certificate, X509, false); - this.basicTestConnection(security, - null, + this.basicTestConnection(securityX509, null, deviceCredentials, clientEndpoint, transportConfiguration, diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/secure/LwM2mDefaultBootstrapSessionManager.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/secure/LwM2mDefaultBootstrapSessionManager.java index 516f118630..9adceb2860 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/secure/LwM2mDefaultBootstrapSessionManager.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/secure/LwM2mDefaultBootstrapSessionManager.java @@ -16,6 +16,7 @@ package org.thingsboard.server.transport.lwm2m.bootstrap.secure; import lombok.extern.slf4j.Slf4j; +import org.eclipse.californium.core.coap.Request; import org.eclipse.leshan.core.peer.IpPeer; import org.eclipse.leshan.core.peer.LwM2mPeer; import org.eclipse.leshan.core.peer.PskIdentity; @@ -112,8 +113,10 @@ public class LwM2mDefaultBootstrapSessionManager extends DefaultBootstrapSession } catch (InvalidConfigurationException e){ log.error("Failed put to lwM2MBootstrapSessionClients by endpoint [{}]", request.getEndpointName(), e); } + String msg = String.format("Bootstrap session started... %s", ((Request) request.getCoapRequest()).getLocalAddress().toString()); + log.warn(String.format("%s: %s", request.getEndpointName(), msg)); this.sendLogs(request.getEndpointName(), - String.format("%s: Bootstrap session started...", LOG_LWM2M_INFO, request.getEndpointName())); + String.format("%s: %s", LOG_LWM2M_INFO, msg)); } return session; } @@ -135,7 +138,7 @@ public class LwM2mDefaultBootstrapSessionManager extends DefaultBootstrapSession session.setModel(modelProvider.getObjectModel(session, tasks.supportedObjects)); // set Requests to Send - log.info("tasks.requestsToSend = [{}]", tasks.requestsToSend); + log.warn("tasks.requestsToSend = [{}]", tasks.requestsToSend); session.setRequests(tasks.requestsToSend); // prepare list where we will store Responses @@ -182,14 +185,16 @@ public class LwM2mDefaultBootstrapSessionManager extends DefaultBootstrapSession session.getResponses().add(response); String msg = String.format("%s: receives success response for: %s %s %s", LOG_LWM2M_INFO, request.getClass().getSimpleName(), request.getPath().toString(), response.toString()); + log.warn(msg); this.sendLogs(bsSession.getEndpoint(), msg); // on success for NOT bootstrap finish request we send next request return BootstrapPolicy.continueWith(nextRequest(bsSession)); } else { // on success for bootstrap finish request we stop the session - this.sendLogs(bsSession.getEndpoint(), - String.format("%s: receives success response for bootstrap finish.", LOG_LWM2M_INFO)); + String msg = String.format("%s: receives success response for bootstrap finish.", LOG_LWM2M_INFO); + log.info(msg); + this.sendLogs(bsSession.getEndpoint(), msg); this.tasksProvider.remove(bsSession.getEndpoint()); return BootstrapPolicy.finished(); } @@ -228,7 +233,9 @@ public class LwM2mDefaultBootstrapSessionManager extends DefaultBootstrapSession @Override public void end(BootstrapSession bsSession) { - this.sendLogs(bsSession.getEndpoint(), String.format("%s: Bootstrap session finished.", LOG_LWM2M_INFO)); + String msg = String.format("%s: Bootstrap session finished.", LOG_LWM2M_INFO); + log.warn(msg); + this.sendLogs(bsSession.getEndpoint(), msg); this.tasksProvider.remove(bsSession.getEndpoint()); } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java index f24f068365..3223179486 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java @@ -17,15 +17,12 @@ package org.thingsboard.server.transport.lwm2m.bootstrap.store; import lombok.extern.slf4j.Slf4j; import org.eclipse.leshan.core.link.Link; -import org.eclipse.leshan.core.node.LwM2mObject; import org.eclipse.leshan.core.node.LwM2mPath; import org.eclipse.leshan.core.request.BootstrapDeleteRequest; import org.eclipse.leshan.core.request.BootstrapDiscoverRequest; import org.eclipse.leshan.core.request.BootstrapDownlinkRequest; -import org.eclipse.leshan.core.request.BootstrapReadRequest; import org.eclipse.leshan.core.request.ContentFormat; import org.eclipse.leshan.core.response.BootstrapDiscoverResponse; -import org.eclipse.leshan.core.response.BootstrapReadResponse; import org.eclipse.leshan.core.response.LwM2mResponse; import org.eclipse.leshan.server.bootstrap.BootstrapConfig; import org.eclipse.leshan.server.bootstrap.BootstrapConfigStore; @@ -33,7 +30,6 @@ import org.eclipse.leshan.server.bootstrap.BootstrapSession; import org.eclipse.leshan.server.bootstrap.BootstrapUtil; import org.eclipse.leshan.server.bootstrap.InvalidConfigurationException; -import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -43,14 +39,18 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.Collectors; -import static org.eclipse.leshan.core.model.ResourceModel.Type.OPAQUE; +import static org.eclipse.leshan.core.LwM2mId.ACCESS_CONTROL; +import static org.eclipse.leshan.core.LwM2mId.SECURITY; +import static org.eclipse.leshan.core.LwM2mId.SERVER; import static org.eclipse.leshan.server.bootstrap.BootstrapUtil.toWriteRequest; import static org.thingsboard.server.transport.lwm2m.utils.LwM2MTransportUtil.BOOTSTRAP_DEFAULT_SHORT_ID_0; +import static org.thingsboard.server.transport.lwm2m.utils.LwM2MTransportUtil.LWM2M_DEFAULT_SHORT_ID_1; +import static org.thingsboard.server.transport.lwm2m.utils.LwM2MTransportUtil.LWM2M_DEFAULT_SHORT_ID_65534; @Slf4j public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTaskProvider { @@ -77,11 +77,11 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask @Override public Tasks getTasks(BootstrapSession session, List previousResponse) { // BootstrapConfig config = store.get(session.getEndpoint(), session.getClientTransportData().getIdentity(), session); - BootstrapConfig config = store.get(session); - if (config == null) { + BootstrapConfig configNew = store.get(session); + if (configNew == null) { return null; } - if (previousResponse == null && shouldStartWithDiscover(config)) { + if (previousResponse == null && shouldStartWithDiscover(configNew)) { Tasks tasks = new Tasks(); tasks.requestsToSend = new ArrayList<>(1); tasks.requestsToSend.add(new BootstrapDiscoverRequest()); @@ -96,47 +96,25 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask tasks.supportedObjects = this.supportedObjects; // handle bootstrap discover response if (previousResponse != null) { - if (previousResponse.get(0) instanceof BootstrapDiscoverResponse) { - BootstrapDiscoverResponse discoverResponse = (BootstrapDiscoverResponse) previousResponse.get(0); + if (previousResponse.get(0) instanceof BootstrapDiscoverResponse discoverResponse) { if (discoverResponse.isSuccess()) { this.initAfterBootstrapDiscover(discoverResponse); - findSecurityInstanceId(discoverResponse.getObjectLinks(), session.getEndpoint()); + /// Short Server Ids - in old config + findInstancesIdOldByServerId(discoverResponse, session.getEndpoint()); } else { - log.warn( - "Bootstrap Discover return error {} : to continue bootstrap session without autoIdForSecurityObject mode. {}", - discoverResponse, session); - } - if (this.lwM2MBootstrapSessionClients.get(session.getEndpoint()).getSecurityInstances().get(BOOTSTRAP_DEFAULT_SHORT_ID_0) == null) { log.error( "Unable to find bootstrap server instance in Security Object (0) in response {}: unable to continue bootstrap session with autoIdForSecurityObject mode. {}", discoverResponse, session); return null; } - tasks.requestsToSend = new ArrayList<>(1); - tasks.requestsToSend.add(new BootstrapReadRequest("/1")); - tasks.last = false; - return tasks; - } - BootstrapReadResponse readResponse = (BootstrapReadResponse) previousResponse.get(0); - Integer bootstrapServerIdOld = null; - if (readResponse.isSuccess()) { - findServerInstanceId(readResponse, session.getEndpoint()); - if (this.lwM2MBootstrapSessionClients.get(session.getEndpoint()).getSecurityInstances().size() > 0 && this.lwM2MBootstrapSessionClients.get(session.getEndpoint()).getServerInstances().size() > 0) { - bootstrapServerIdOld = this.findBootstrapServerId(session.getEndpoint()); - } - } else { - log.warn( - "Bootstrap ReadResponse return error {} : to continue bootstrap session without find Server Instance Id. {}", - readResponse, session); } // create requests from config - tasks.requestsToSend = this.toRequests(config, - config.contentFormat != null ? config.contentFormat : session.getContentFormat(), - bootstrapServerIdOld, session.getEndpoint()); + tasks.requestsToSend = this.toRequests(configNew, + configNew.contentFormat != null ? configNew.contentFormat : session.getContentFormat(), session.getEndpoint()); } else { // create requests from config - tasks.requestsToSend = BootstrapUtil.toRequests(config, - config.contentFormat != null ? config.contentFormat : session.getContentFormat()); + tasks.requestsToSend = BootstrapUtil.toRequests(configNew, + configNew.contentFormat != null ? configNew.contentFormat : session.getContentFormat()); } return tasks; } @@ -148,81 +126,57 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask /** * "Short Server ID": This Resource MUST be set when the Bootstrap-Server Resource has a value of 'false'. - * The values ID:0 and ID:65535 values MUST NOT be used for identifying the LwM2M Server. - * "Short Server ID": + * "Short Lwm2m Server ID": * - Link Instance (lwm2m Server) hase linkParams with key = "ssid" value = "shortId" (ver lvm2m = 1.1). - * - Link Instance (bootstrap Server) hase not linkParams with key = "ssid" (ver lvm2m = 1.0). + * The values ID:0 values MUST NOT be used for identifying the LwM2M Server only BS. */ - protected void findSecurityInstanceId(Link[] objectLinks, String endpoint) { - log.info("Object after discover: [{}]", objectLinks); - for (Link link : objectLinks) { - if (link.getUriReference().startsWith("/0/")) { - try { - LwM2mPath path = new LwM2mPath(link.getUriReference()); - if (path.isObjectInstance()) { - if (link.getAttributes().get("ssid") != null) { - int serverId = Integer.parseInt(link.getAttributes().get("ssid").getCoreLinkValue()); - if (!lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().containsKey(serverId)) { - lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().put(serverId, path.getObjectInstanceId()); - } else { - log.error("Invalid lwm2mSecurityInstance by [{}]", path.getObjectInstanceId()); - } - lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().put(serverId, path.getObjectInstanceId()); + protected void findInstancesIdOldByServerId(BootstrapDiscoverResponse discoverResponses, String endpoint) { + log.info("Object after discover: [{}]", Arrays.toString(discoverResponses.getObjectLinks())); + for (Link link : discoverResponses.getObjectLinks()) { + LwM2mPath path = new LwM2mPath(link.getUriReference()); + if (path.isObjectInstance()) { + int lwm2mShortServerId = 0; + if (path.getObjectId() == 0) { + if (link.getAttributes().get("ssid") != null) { + lwm2mShortServerId = Integer.parseInt(link.getAttributes().get("ssid").getCoreLinkValue()); + if (validateLwm2mShortServerId(lwm2mShortServerId)) { + this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().putIfAbsent(lwm2mShortServerId, path.getObjectInstanceId()); + } else { + log.error("Invalid lwm2mSecurityInstance [{}] by short server id [{}]", path.getObjectInstanceId(), lwm2mShortServerId); + } + } else { + this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().putIfAbsent(BOOTSTRAP_DEFAULT_SHORT_ID_0, path.getObjectInstanceId()); + } + } else if (path.getObjectId() == 1) { + if (link.getAttributes().get("ssid") != null) { + lwm2mShortServerId = Integer.parseInt(link.getAttributes().get("ssid").getCoreLinkValue()); + if (validateLwm2mShortServerId(lwm2mShortServerId)) { + this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().putIfAbsent(lwm2mShortServerId, path.getObjectInstanceId()); } else { - if (!this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().containsKey(0)) { - this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().put(BOOTSTRAP_DEFAULT_SHORT_ID_0, path.getObjectInstanceId()); - } else { - log.error("Invalid bootstrapSecurityInstance by [{}]", path.getObjectInstanceId()); - } + log.error("Invalid lwm2mServerInstance [{}] by short server id [{}]", path.getObjectInstanceId(), lwm2mShortServerId); } } - } catch (Exception e) { - // ignore if this is not a LWM2M path - log.error("Invalid LwM2MPath starting by \"/0/\""); } } } } - protected void findServerInstanceId(BootstrapReadResponse readResponse, String endpoint) { - try { - ((LwM2mObject) readResponse.getContent()).getInstances().values().forEach(instance -> { - var shId = OPAQUE.equals(instance.getResource(0).getType()) ? new BigInteger((byte[]) instance.getResource(0).getValue()).intValue() : instance.getResource(0).getValue(); - int shortId; - if (shId instanceof Long) { - shortId = ((Long) shId).intValue(); - } else { - shortId = (int) shId; - } - this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().put(shortId, instance.getId()); - }); - } catch (Exception e) { - log.error("Failed find Server Instance Id. ", e); - } - } - - protected Integer findBootstrapServerId(String endpoint) { - Integer bootstrapServerIdOld = null; - Map filteredMap = this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().entrySet() - .stream().filter(x -> !this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().containsKey(x.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - if (filteredMap.size() > 0) { - bootstrapServerIdOld = filteredMap.keySet().stream().findFirst().get(); - } - return bootstrapServerIdOld; - } - public BootstrapConfigStore getStore() { return this.store; } private void initAfterBootstrapDiscover(BootstrapDiscoverResponse response) { Link[] links = response.getObjectLinks(); + AtomicReference verDefault = new AtomicReference<>("1.0"); Arrays.stream(links).forEach(link -> { LwM2mPath path = new LwM2mPath(link.getUriReference()); - if (!path.isRoot() && path.getObjectId() < 3) { + if (path.isRoot()) { + if (link.hasAttribute() && link.getAttributes().get("lwm2m") != null) { + verDefault.set(link.getAttributes().get("lwm2m").getValue().toString()); + } + } else if (path.getObjectId() <= ACCESS_CONTROL) { if (path.isObject()) { - String ver = link.getAttributes().get("ver") != null ? link.getAttributes().get("ver").getCoreLinkValue() : "1.0"; + String ver = (link.hasAttribute() && link.getAttributes().get("ver") != null) ? link.getAttributes().get("ver").getCoreLinkValue() : verDefault.get(); this.supportedObjects.put(path.getObjectId(), ver); } } @@ -230,84 +184,103 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask } - public List> toRequests(BootstrapConfig bootstrapConfig, + /** Map + * 1) Only Lwm2m Server + * - Short Server ID == 1 - 65534 lwm2m) + * SECURITY = 0; InstanceId = 0 + * SERVER = 1; InstanceId = 0 + * 2) Both + * - Short Server ID == 0 or 65535 bs) + * SECURITY = 0; InstanceId = 0 + * SERVER = 1; InstanceId = null + * - Short Server ID == 1 - 65534 lwm2m) + * SECURITY = 0; InstanceId = 1 + * SERVER = 1; InstanceId = 0 + * */ + public List> toRequests(BootstrapConfig bootstrapConfigNew, ContentFormat contentFormat, - Integer bootstrapServerIdOld, String endpoint) { + Integer bootstrapSecurityInstanceId = this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(BOOTSTRAP_DEFAULT_SHORT_ID_0); List> requests = new ArrayList<>(); Set pathsDelete = new HashSet<>(); - List> requestsWrite = new ArrayList<>(); - boolean isBsServer = false; - boolean isLwServer = false; - /** Map */ - Map instances = new HashMap<>(); - Integer bootstrapServerIdNew = null; - // handle security - int lwm2mSecurityInstanceId = 0; - int bootstrapSecurityInstanceId = this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(BOOTSTRAP_DEFAULT_SHORT_ID_0); - for (BootstrapConfig.ServerSecurity security : new TreeMap<>(bootstrapConfig.security).values()) { - if (security.bootstrapServer) { - requestsWrite.add(toWriteRequest(bootstrapSecurityInstanceId, security, contentFormat)); - isBsServer = true; - bootstrapServerIdNew = security.serverId; - instances.put(security.serverId, bootstrapSecurityInstanceId); - } else { - if (lwm2mSecurityInstanceId == bootstrapSecurityInstanceId) { - lwm2mSecurityInstanceId++; - } - requestsWrite.add(toWriteRequest(lwm2mSecurityInstanceId, security, contentFormat)); - instances.put(security.serverId, lwm2mSecurityInstanceId); - isLwServer = true; - if (!isBsServer && this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().containsKey(security.serverId) && - lwm2mSecurityInstanceId != this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(security.serverId)) { - pathsDelete.add("/0/" + this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(security.serverId)); - } - /** - * If there is an instance in the serverInstances with serverId which we replace in the securityInstances - */ - // find serverId in securityInstances by id (instance) - Integer serverIdOld = null; - for (Map.Entry entry : this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().entrySet()) { - if (entry.getValue().equals(lwm2mSecurityInstanceId)) { - serverIdOld = entry.getKey(); - } - } - if (!isBsServer && serverIdOld != null && this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().containsKey(serverIdOld)) { - pathsDelete.add("/1/" + this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().get(serverIdOld)); - } - lwm2mSecurityInstanceId++; + ConcurrentHashMap> requestsWrite = new ConcurrentHashMap<>(); + + /// handle security & handle + int lwm2mSecurityInstanceIdMax = -1; + int lwm2mServerInstanceIdMax = -1; + // bootstrap Security new - There can only be one instance of bootstrap at a time. + /// bs: handle security only + for (BootstrapConfig.ServerSecurity security : new TreeMap<>(bootstrapConfigNew.security).values()) { + if (security.bootstrapServer && security.serverId == BOOTSTRAP_DEFAULT_SHORT_ID_0) { + // delete old bootstrap Security + String path = "/" + SECURITY + "/" + bootstrapSecurityInstanceId; + pathsDelete.add(path); + // add new bootstrap Security + requestsWrite.put(path, toWriteRequest(bootstrapSecurityInstanceId, security, contentFormat)); } } - // handle server - for (Map.Entry server : bootstrapConfig.servers.entrySet()) { - int securityInstanceId = instances.get(server.getValue().shortId); - requestsWrite.add(toWriteRequest(securityInstanceId, server.getValue(), contentFormat)); - if (!isBsServer) { - /** Delete instance if bootstrapServerIdNew not equals bootstrapServerIdOld or securityInstanceBsIdNew not equals serverInstanceBsIdOld */ - if (bootstrapServerIdNew != null && server.getValue().shortId == bootstrapServerIdNew && - (bootstrapServerIdNew != bootstrapServerIdOld || securityInstanceId != this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().get(bootstrapServerIdOld))) { - pathsDelete.add("/1/" + this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().get(bootstrapServerIdOld)); - /** Delete instance if serverIdNew is present in serverInstances and securityInstanceIdOld by serverIdNew not equals serverInstanceIdOld */ - } else if (this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().containsKey(server.getValue().shortId) && - securityInstanceId != this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().get(server.getValue().shortId)) { - pathsDelete.add("/1/" + this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().get(server.getValue().shortId)); - } + + /** lwm2m servers: Multiple instances of lwm2m servers can run simultaneously by SHORT_ID + if update -> delete and write by InstanceId + if new -> only write with InstanceIdMax++ + */ + + /// lwm2m server: handle security & server + //max Lwm2m Security instance old id if new + for (Integer shortId : this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().keySet()) { + if (shortId >= BOOTSTRAP_DEFAULT_SHORT_ID_0 && shortId <= LWM2M_DEFAULT_SHORT_ID_65534) { + lwm2mSecurityInstanceIdMax = this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(shortId) > + lwm2mSecurityInstanceIdMax ? this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(shortId) : + lwm2mSecurityInstanceIdMax; } } - // handle acl - for (Map.Entry acl : bootstrapConfig.acls.entrySet()) { - requestsWrite.add(toWriteRequest(acl.getKey(), acl.getValue(), contentFormat)); + //max Lwm2m Server instance old id if new + for (Integer shortId : this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().keySet()) { + if (shortId >= LWM2M_DEFAULT_SHORT_ID_1 && shortId <= LWM2M_DEFAULT_SHORT_ID_65534) { + lwm2mServerInstanceIdMax = this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().get(shortId) > + lwm2mServerInstanceIdMax ? this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().get(shortId) : + lwm2mServerInstanceIdMax; + } } - // handle delete - if (isBsServer && isLwServer) { - requests.add(new BootstrapDeleteRequest("/0")); - requests.add(new BootstrapDeleteRequest("/1")); - } else { - pathsDelete.forEach(pathDelete -> requests.add(new BootstrapDeleteRequest(pathDelete))); + // Lwm2m update or new + for (BootstrapConfig.ServerSecurity security : new TreeMap<>(bootstrapConfigNew.security).values()) { + if (!security.bootstrapServer) { + // Security + boolean isUpdate = this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().containsKey(security.serverId); + Integer secureInstanceId; + Integer serverInstanceId; + if (isUpdate) { + secureInstanceId = this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(security.serverId); + serverInstanceId = this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().get(security.serverId); + pathsDelete.add("/" + SECURITY + "/" + secureInstanceId); + pathsDelete.add("/" + SERVER + "/" + serverInstanceId); + } else { + secureInstanceId = ++lwm2mSecurityInstanceIdMax; + serverInstanceId = ++lwm2mServerInstanceIdMax; + } + requestsWrite.put("/" + SECURITY + "/" + secureInstanceId, toWriteRequest(secureInstanceId, security, contentFormat)); + new TreeMap<>(bootstrapConfigNew.servers).values().stream() + .filter(server -> server.shortId == security.serverId) + .findFirst() + .ifPresent(server -> + requestsWrite.put( + "/" + SERVER + "/" + serverInstanceId, + toWriteRequest(serverInstanceId, server, contentFormat) + ) + ); + } } - // handle write - if (requestsWrite.size() > 0) { - requests.addAll(requestsWrite); + + /// handle acl + for (Map.Entry acl : bootstrapConfigNew.acls.entrySet()) { + requestsWrite.put("/" + ACCESS_CONTROL + "/" + acl.getKey(), toWriteRequest(acl.getKey(), acl.getValue(), contentFormat)); + } + /// handle delete + pathsDelete.forEach(pathDelete -> requests.add(new BootstrapDeleteRequest(pathDelete))); + + /// handle write + if (!requestsWrite.isEmpty()) { + requests.addAll(requestsWrite.values()); } return (requests); } @@ -315,9 +288,13 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask private void initSupportedObjectsDefault() { this.supportedObjects = new HashMap<>(); - this.supportedObjects.put(0, "1.1"); - this.supportedObjects.put(1, "1.1"); - this.supportedObjects.put(2, "1.0"); + this.supportedObjects.put(SECURITY, "1.1"); + this.supportedObjects.put(SERVER, "1.1"); + this.supportedObjects.put(ACCESS_CONTROL, "1.0"); + } + + private boolean validateLwm2mShortServerId(int id){ + return id >= LWM2M_DEFAULT_SHORT_ID_1 && id <= LWM2M_DEFAULT_SHORT_ID_65534; } @Override diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/LwM2MTransportUtil.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/LwM2MTransportUtil.java index 160ca3d905..7464361c24 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/LwM2MTransportUtil.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/LwM2MTransportUtil.java @@ -82,6 +82,8 @@ public class LwM2MTransportUtil { public static final String LOG_LWM2M_ERROR = "error"; public static final String LOG_LWM2M_WARN = "warn"; public static final int BOOTSTRAP_DEFAULT_SHORT_ID_0 = 0; + public static final int LWM2M_DEFAULT_SHORT_ID_1 = 1; + public static final int LWM2M_DEFAULT_SHORT_ID_65534 = 65534; public static LwM2mOtaConvert convertOtaUpdateValueToString(String pathIdVer, Object value, ResourceModel.Type currentType) { String path = fromVersionedIdToObjectId(pathIdVer); diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts index 1630a1d220..6756ea2874 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts @@ -104,7 +104,7 @@ export class Lwm2mDeviceConfigServerComponent implements OnInit, ControlValueAcc clientHoldOffTime: ['', [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], shortServerId: ['', this.isBootstrap ? [Validators.required, Validators.pattern('^(' + this.shortServerIdBsMin+ '|' + this.shortServerIdBsMax + ')$' )] - : [Validators.required, Validators.pattern('[0-9]*'),Validators.min(this.shortServerIdMin), Validators.max(this.shortServerIdMax)] + : [Validators.required, Validators.pattern('[0-9]*'), Validators.min(this.shortServerIdMin), Validators.max(this.shortServerIdMax)] ], bootstrapServerAccountTimeout: ['', [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], binding: [''], From 10edf7b623aa670552afb679dc1ab4511de28d22 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 10 Oct 2025 11:52:28 +0300 Subject: [PATCH 24/88] UI: Updated gateway dashboard to fixed XSS vulnerability --- .../src/main/data/resources/dashboards/gateways_dashboard.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/data/resources/dashboards/gateways_dashboard.json b/application/src/main/data/resources/dashboards/gateways_dashboard.json index 381c9f6de0..078f913570 100644 --- a/application/src/main/data/resources/dashboards/gateways_dashboard.json +++ b/application/src/main/data/resources/dashboards/gateways_dashboard.json @@ -650,7 +650,7 @@ "settings": { "useMarkdownTextFunction": true, "markdownTextPattern": "# Markdown/HTML card \\n - **Current entity**: **${entityName}**. \\n - **Current value**: **${Random}**.", - "markdownTextFunction": "var blockData = '';\nvar connectorsIndex = ctx.actionsApi.getActionDescriptors('elementClick').findIndex(action => action.name == \"Connectors\");\nvar logsIndex = ctx.actionsApi.getActionDescriptors('elementClick').findIndex(action => action.name == \"Logs\");\nfunction generateMatHeader(index) {\n if (index !== undefined && index > -1) {\n return ``\n } else {\n return \"\"\n }\n}\nfunction createDataBlock(value, label, dividerStyle, mobile, index) {\n blockData += `\n \n
\n \n ${generateMatHeader(index)}\n ${label}\n \n ${value}\n `;\n}\ncreateDataBlock(data[0].Status, \"Status\", data[0].Status === \"Active\" ? 'divider-green' : 'divider-red');\ncreateDataBlock(data[0].Name, \"Gateway Name\", '', ctx.isMobile);\nif (data[0].Version) {\n createDataBlock(data[0].Version, \"Gateway Version\", '');\n}\ncreateDataBlock(data[0].Type, \"Gateway Type\", '');\ncreateDataBlock(\n `${(data[1] ? data[1].count : 0)} `\n + \" | \" +\n `${(data[2] ? data[2][\"count 2\"] : 0)} `\n , \"Devices (Active | Inactive)\", '');\ncreateDataBlock(\n `${(data[0].active_connectors ? JSON.parse(data[0].active_connectors).length : 0)} `\n + \" | \" +\n `${(data[0].inactive_connectors ? JSON.parse(data[0].inactive_connectors).length : 0)} `\n , \"Connectors (Enabled | Disabled)\", '', '', connectorsIndex);\ncreateDataBlock(data[0].ALL_ERRORS_COUNT || 0, \"Errors\", (data[0].ALL_ERRORS_COUNT || 0) === 0 ? 'divider-green' : 'divider-red', '', logsIndex);\nreturn `
${blockData}
`;", + "markdownTextFunction": "var blockData = '';\nvar connectorsIndex = ctx.actionsApi.getActionDescriptors('elementClick').findIndex(action => action.name == \"Connectors\");\nvar logsIndex = ctx.actionsApi.getActionDescriptors('elementClick').findIndex(action => action.name == \"Logs\");\nfunction generateMatHeader(index) {\n if (index !== undefined && index > -1) {\n return ``\n } else {\n return \"\"\n }\n}\nfunction createDataBlock(value, label, dividerStyle, mobile, index) {\n blockData += `\n \n
\n \n ${generateMatHeader(index)}\n ${label}\n \n ${ctx.sanitizer.sanitize(1, value)}\n `;\n}\ncreateDataBlock(data[0].Status, \"Status\", data[0].Status === \"Active\" ? 'divider-green' : 'divider-red');\ncreateDataBlock(data[0].Name, \"Gateway Name\", '', ctx.isMobile);\nif (data[0].Version) {\n createDataBlock(data[0].Version, \"Gateway Version\", '');\n}\ncreateDataBlock(data[0].Type, \"Gateway Type\", '');\ncreateDataBlock(\n `${(data[1] ? data[1].count : 0)} `\n + \" | \" +\n `${(data[2] ? data[2][\"count 2\"] : 0)} `\n , \"Devices (Active | Inactive)\", '');\ncreateDataBlock(\n `${(data[0].active_connectors ? JSON.parse(data[0].active_connectors).length : 0)} `\n + \" | \" +\n `${(data[0].inactive_connectors ? JSON.parse(data[0].inactive_connectors).length : 0)} `\n , \"Connectors (Enabled | Disabled)\", '', '', connectorsIndex);\ncreateDataBlock(data[0].ALL_ERRORS_COUNT || 0, \"Errors\", (data[0].ALL_ERRORS_COUNT || 0) === 0 ? 'divider-green' : 'divider-red', '', logsIndex);\nreturn `
${blockData}
`;", "applyDefaultMarkdownStyle": false, "markdownCss": ".divider {\n position: absolute;\n width: 3px;\n top: 8px;\n border-radius: 2px;\n bottom: 8px;\n border: 1px solid rgba(31, 70, 144, 1);\n background-color: rgba(31, 70, 144, 1);\n left: 10px;\n}\n.divider-green .divider {\n border: 1px solid rgb(25,128,56);\n background-color: rgb(25,128,56);\n}\n\n.divider-green .mat-mdc-card-content {\n color: rgb(25,128,56);\n}\n\n.divider-red .divider {\n border: 1px solid rgb(203,37,48);\n background-color: rgb(203,37,48);\n}\n\n.divider-red .mat-mdc-card-content {\n color: rgb(203,37,48);\n}\n\n.mdc-card {\n position: relative;\n padding-left: 10px;\n margin-bottom: 1px;\n}\n\n.mat-mdc-card-subtitle {\n font-weight: 400;\n font-size: 12px;\n}\n\n.mat-mdc-card-header {\n padding: 8px 16px 0;\n}\n\n.mat-mdc-card-content:last-child {\n padding-bottom: 8px;\n font-size: 16px;\n}\n\n.cards-container {\n height: calc(100% - 1px);\n justify-content: stretch;\n align-items: center;\n margin-bottom: 1px;\n}\n\n::ng-deep.tb-home-widget-link > div {\n flex-grow: 1;\n cursor: pointer;\n}\n\n .tb-home-widget-link {\n width: 100%;\n }\n\n .tb-home-widget-link:hover::after{\n color: inherit;\n }\n \n .tb-home-widget-link::after{\n content: 'arrow_forward';\n display: inline-block;\n transform: rotate(315deg);\n font-family: 'Material Icons';\n font-weight: normal;\n font-style: normal;\n font-size: 18px;\n color: rgba(0, 0, 0, 0.12);\n vertical-align: bottom;\n margin-left: 6px;\n}" }, From f667f0f23f796492050e336f1fee37e8495a69d0 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 10 Oct 2025 16:58:00 +0300 Subject: [PATCH 25/88] (wip) aggregation cf --- .../CalculatedFieldEntityActor.java | 3 + ...CalculatedFieldEntityMessageProcessor.java | 154 +++++++-- ...alculatedFieldManagerMessageProcessor.java | 320 +++++++++++++++++- .../CalculatedFieldRelatedEntityMsg.java | 49 +++ ...tractCalculatedFieldProcessingService.java | 97 ++++++ .../service/cf/CalculatedFieldCache.java | 3 + .../cf/CalculatedFieldProcessingService.java | 2 + .../cf/DefaultCalculatedFieldCache.java | 28 ++ ...faultCalculatedFieldProcessingService.java | 8 + .../DefaultCalculatedFieldQueueService.java | 39 ++- .../service/cf/ctx/state/ArgumentEntry.java | 10 +- .../cf/ctx/state/ArgumentEntryType.java | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 2 +- .../cf/ctx/state/CalculatedFieldCtx.java | 140 ++++++++ .../cf/ctx/state/CalculatedFieldState.java | 5 +- .../ctx/state/SingleValueArgumentEntry.java | 8 +- .../state/aggregation/AggArgumentEntry.java | 72 ++++ .../aggregation/AggSingleArgumentEntry.java | 83 +++++ ...ValuesAggregationCalculatedFieldState.java | 161 +++++++++ .../service/cf/ctx/state/aggregation/agg.json | 65 ++++ .../state/aggregation/function/AggEntry.java | 45 +++ .../function/AggFunctionFactory.java | 33 ++ .../aggregation/function/AvgAggEntry.java | 45 +++ .../aggregation/function/BaseAggEntry.java | 52 +++ .../aggregation/function/CountAggEntry.java | 41 +++ .../function/CountUniqueAggEntry.java | 45 +++ .../aggregation/function/MaxAggEntry.java | 40 +++ .../aggregation/function/MinAggEntry.java | 40 +++ .../aggregation/function/SumAggEntry.java | 42 +++ .../entitiy/EntityStateSourcingListener.java | 11 + .../queue/DefaultTbClusterService.java | 40 ++- .../processing/AbstractConsumerService.java | 18 +- .../utils/CalculatedFieldArgumentUtils.java | 2 + .../server/utils/CalculatedFieldUtils.java | 48 ++- ...tValuesAggregationCalculatedFieldTest.java | 314 +++++++++++++++++ .../server/controller/AbstractWebTest.java | 10 + .../server/cluster/TbClusterService.java | 8 + .../server/dao/relation/RelationService.java | 8 + .../common/data/cf/CalculatedFieldType.java | 3 +- .../CalculatedFieldConfiguration.java | 4 +- .../aggregation/AggFunction.java | 20 ++ .../aggregation/AggFunctionInput.java | 34 ++ .../configuration/aggregation/AggInput.java | 36 ++ .../aggregation/AggKeyInput.java | 34 ++ .../configuration/aggregation/AggMetric.java | 31 ++ .../configuration/aggregation/AggSource.java | 30 ++ .../aggregation/CfAggTrigger.java | 104 ++++++ ...gregationCalculatedFieldConfiguration.java | 52 +++ .../server/common/msg/MsgType.java | 2 + .../msg/plugin/ComponentLifecycleMsg.java | 6 +- .../server/common/util/ProtoUtils.java | 2 + common/proto/src/main/proto/queue.proto | 7 + .../script/api/tbel/TbelCfArg.java | 1 + .../tbel/TbelCfLatestValuesAggregation.java | 44 +++ .../dao/relation/BaseRelationService.java | 34 ++ .../server/dao/relation/RelationCacheKey.java | 7 +- .../server/dao/relation/RelationDao.java | 4 + .../dao/sql/relation/JpaRelationDao.java | 10 + .../dao/sql/relation/RelationRepository.java | 39 ++- 59 files changed, 2527 insertions(+), 70 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java create mode 100644 application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunction.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunctionInput.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggKeyInput.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggMetric.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java create mode 100644 common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java index cababd4b6d..ed4131c114 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -73,6 +73,9 @@ public class CalculatedFieldEntityActor extends AbstractCalculatedFieldActor { case CF_ENTITY_DELETE_MSG: processor.process((CalculatedFieldEntityDeleteMsg) msg); break; + case CF_RELATED_ENTITY_MSG: + processor.process((CalculatedFieldRelatedEntityMsg) msg); + break; case CF_ENTITY_TELEMETRY_MSG: processor.process((EntityCalculatedFieldTelemetryMsg) msg); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index ccbdcc6b33..98b5eba4c8 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -52,6 +52,9 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -66,7 +69,9 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createStateByType; @@ -189,7 +194,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } - public void process(CalculatedFieldEntityDeleteMsg msg) { + public void process(CalculatedFieldEntityDeleteMsg msg) throws CalculatedFieldException { log.debug("[{}] Processing CF entity delete msg.", msg.getEntityId()); if (this.entityId.equals(msg.getEntityId())) { if (states.isEmpty()) { @@ -200,16 +205,74 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM actorCtx.stop(actorCtx.getSelf()); } } else { - var cfId = new CalculatedFieldId(msg.getEntityId().getId()); - var state = removeState(cfId); - if (state != null) { - cfStateService.deleteState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); + EntityId msgEntityId = msg.getEntityId(); + if (msgEntityId instanceof CalculatedFieldId cfId) { + var state = removeState(cfId); + if (state != null) { + cfStateService.deleteState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); + } else { + msg.getCallback().onSuccess(); + } } else { - msg.getCallback().onSuccess(); + if (states.isEmpty()) { + msg.getCallback().onSuccess(); + } + for (Map.Entry entry : states.entrySet()) { + LatestValuesAggregationCalculatedFieldState state = (LatestValuesAggregationCalculatedFieldState) entry.getValue(); + state.getArguments().forEach((argName, argEntry) -> { + AggArgumentEntry aggArgEntry = (AggArgumentEntry) argEntry; + aggArgEntry.getAggInputs().remove(msgEntityId); + }); + state.getInputs().remove(msgEntityId); + state.setLastMetricsEvalTs(-1); + processStateIfReady(state, Collections.emptyMap(), state.getCtx(), Collections.emptyList(), null, null, msg.getCallback()); + } } } } + public void process(CalculatedFieldRelatedEntityMsg msg) throws CalculatedFieldException { + log.debug("[{}] Processing CF related entity msg.", msg.getEntityId()); + CalculatedFieldCtx cfCtx = msg.getCalculatedField(); + var state = states.get(cfCtx.getCfId()); + Map fetchedArguments = fetchAggArguments(msg.getCalculatedField(), msg.getEntityId()); + try { + if (state == null) { + state = createState(cfCtx); + } else { + state.setCtx(cfCtx, actorCtx); + } + if (state.isSizeOk()) { + if (state instanceof LatestValuesAggregationCalculatedFieldState latestValuesState) { + latestValuesState.setLastMetricsEvalTs(-1); + } + state.update(fetchedArguments, cfCtx); + state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, cfCtx.getCfId(), entityId), cfCtx.getMaxStateSize()); + states.put(cfCtx.getCfId(), state); + processStateIfReady(state, fetchedArguments, cfCtx, Collections.singletonList(cfCtx.getCfId()), null, null, msg.getCallback()); + } else { + throw new RuntimeException(cfCtx.getSizeExceedsLimitMessage()); + } + } catch (Exception e) { + log.debug("[{}][{}] Failed to initialize CF state", entityId, cfCtx.getCfId(), e); + if (e instanceof CalculatedFieldException cfe) { + throw cfe; + } + throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(entityId).cause(e).build(); + } + } + + @SneakyThrows + private Map fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId) { + ListenableFuture> argumentsFuture = cfService.fetchAggArguments(ctx, entityId); + // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. + // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. + // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, + // but this will significantly complicate the code. + return argumentsFuture.get(1, TimeUnit.MINUTES); + } + + public void process(EntityCalculatedFieldTelemetryMsg msg) throws CalculatedFieldException { log.trace("[{}] Processing CF telemetry msg: {}", msg.getEntityId(), msg); var proto = msg.getProto(); @@ -462,59 +525,78 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { - return mapToArguments(ctx.getMainEntityArguments(), data); + return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getAggregationInputs(), data); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { - return mapToArguments(ctx.getLinkedAndDynamicArgs(entityId), data); + return mapToArguments(entityId, ctx.getLinkedAndDynamicArgs(entityId), ctx.getAggregationInputs(), data); } - private Map mapToArguments(Map argNames, List data) { - if (argNames.isEmpty()) { - return Collections.emptyMap(); - } + private Map mapToArguments(EntityId originator, Map argNames, Map aggArgNames, List data) { Map arguments = new HashMap<>(); - for (TsKvProto item : data) { - ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); - String argName = argNames.get(key); - if (argName != null) { - arguments.put(argName, new SingleValueArgumentEntry(item)); + if (!aggArgNames.isEmpty()) { + for (Map.Entry entry : aggArgNames.entrySet()) { + for (TsKvProto item : data) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); + if (key.equals(entry.getValue())) { + arguments.put(entry.getKey(), new AggSingleArgumentEntry(originator, item)); + } + } } - key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null); - argName = argNames.get(key); - if (argName != null) { - arguments.put(argName, new SingleValueArgumentEntry(item)); + } + if (!argNames.isEmpty()) { + for (TsKvProto item : data) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); + String argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } + key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null); + argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } } } return arguments; } private Map mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List attrDataList) { - return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), scope, attrDataList); + return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), ctx.getAggregationInputs(), scope, attrDataList); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { var argNames = ctx.getLinkedAndDynamicArgs(entityId); - if (argNames.isEmpty()) { - return Collections.emptyMap(); - } List geofencingArgumentNames = ctx.getLinkedEntityAndCurrentOwnerGeofencingArgumentNames(); - return mapToArguments(entityId, argNames, geofencingArgumentNames, scope, attrDataList); + Map aggregationInputs = ctx.getAggregationInputs(); + return mapToArguments(entityId, argNames, geofencingArgumentNames, aggregationInputs, scope, attrDataList); } - private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, AttributeScopeProto scope, List attrDataList) { + private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, Map aggArgNames, AttributeScopeProto scope, List attrDataList) { Map arguments = new HashMap<>(); - for (AttributeValueProto item : attrDataList) { - ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); - String argName = argNames.get(key); - if (argName == null) { - continue; + if (!argNames.isEmpty()) { + for (AttributeValueProto item : attrDataList) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); + String argName = argNames.get(key); + if (argName == null) { + continue; + } + if (geofencingArgNames.contains(argName)) { + arguments.put(argName, new GeofencingArgumentEntry(entityId, item)); + continue; + } + arguments.put(argName, new SingleValueArgumentEntry(item)); } - if (geofencingArgNames.contains(argName)) { - arguments.put(argName, new GeofencingArgumentEntry(entityId, item)); - continue; + } + if (!aggArgNames.isEmpty()) { + for (AttributeValueProto item : attrDataList) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); + for (Map.Entry entry : aggArgNames.entrySet()) { + if (key.equals(entry.getValue())) { + arguments.put(entry.getKey(), new AggSingleArgumentEntry(entityId, item)); + } + } } - arguments.put(argName, new SingleValueArgumentEntry(item)); } return arguments; } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 4675821a5b..8a717c3cdb 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -25,18 +25,27 @@ import org.thingsboard.server.actors.calculatedField.EntityInitCalculatedFieldMs import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; +import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.msg.CalculatedFieldStatePartitionRestoreMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldCacheInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; @@ -44,10 +53,13 @@ import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldStateService; @@ -64,6 +76,8 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ScheduledFuture; @@ -82,6 +96,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final Map> entityIdCalculatedFields = new HashMap<>(); private final Map> entityIdCalculatedFieldLinks = new HashMap<>(); private final Map> ownerEntities = new HashMap<>(); + private final Map cfTriggers = new HashMap<>(); private ScheduledFuture cfsReevaluationTask; private final CalculatedFieldProcessingService cfExecService; @@ -90,6 +105,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final DeviceService deviceService; private final AssetService assetService; private final CustomerService customerService; + private final RelationService relationService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; private final TenantEntityProfileCache entityProfileCache; @@ -107,6 +123,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware this.deviceService = systemContext.getDeviceService(); this.assetService = systemContext.getAssetService(); this.customerService = systemContext.getCustomerService(); + this.relationService = systemContext.getRelationService(); this.assetProfileCache = systemContext.getAssetProfileCache(); this.deviceProfileCache = systemContext.getDeviceProfileCache(); this.entityProfileCache = new TenantEntityProfileCache(); @@ -129,6 +146,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware cfsReevaluationTask.cancel(true); cfsReevaluationTask = null; } + cfTriggers.clear(); ctx.stop(ctx.getSelf()); } @@ -233,6 +251,20 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware entityProfileCache.add(profileId, entityId); } updateEntityOwner(entityId); + + MultipleTbCallback callbackFor2 = new MultipleTbCallback(2, callback); + + // process aggregation cfs(in any) + List cfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(entityId, profileId); + if (!cfsRelatedToEntity.isEmpty()) { + MultipleTbCallback multiCallback = new MultipleTbCallback(cfsRelatedToEntity.size(), callbackFor2); + cfsRelatedToEntity.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> initRelatedEntity(id, entityId, ctx, cb)); + }); + } else { + callbackFor2.onSuccess(); + } + if (!isMyPartition(entityId, callback)) { return; } @@ -240,11 +272,11 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var profileIdFields = getCalculatedFieldsByEntityId(profileId); var fieldsCount = entityIdFields.size() + profileIdFields.size(); if (fieldsCount > 0) { - MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callbackFor2); entityIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); profileIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); } else { - callback.onSuccess(); + callbackFor2.onSuccess(); } } @@ -254,33 +286,180 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (!isMyPartition(msg.getEntityId(), callback)) { return; } + MultipleTbCallback callbackFor2 = new MultipleTbCallback(2, callback); + + // process aggregation cfs(in any) + List oldCfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getOldProfileId()); + List newCfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getProfileId()); + var fieldsWithRelatedEntityCount = oldCfsRelatedToEntity.size() + newCfsRelatedToEntity.size(); + if (fieldsWithRelatedEntityCount > 0) { + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsWithRelatedEntityCount, callbackFor2); + var entityId = msg.getEntityId(); + oldCfsRelatedToEntity.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> deleteRelatedEntity(id, entityId, cb)); + }); + newCfsRelatedToEntity.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> initRelatedEntity(id, entityId, ctx, cb)); + }); + } else { + callbackFor2.onSuccess(); + } + var oldProfileCfs = getCalculatedFieldsByEntityId(msg.getOldProfileId()); var newProfileCfs = getCalculatedFieldsByEntityId(msg.getProfileId()); var fieldsCount = oldProfileCfs.size() + newProfileCfs.size(); if (fieldsCount > 0) { - MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callbackFor2); var entityId = msg.getEntityId(); oldProfileCfs.forEach(ctx -> deleteCfForEntity(entityId, ctx.getCfId(), multiCallback)); newProfileCfs.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); } else { - callback.onSuccess(); + callbackFor2.onSuccess(); } } else if (msg.isOwnerChanged()) { onEntityOwnerChanged(msg, callback); + } else if (msg.isRelationChanged()) { + onRelationUpdated(msg, callback); } else { callback.onSuccess(); } } private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { - switch (msg.getEntityId().getEntityType()) { - case DEVICE, ASSET -> entityProfileCache.removeEntityId(msg.getEntityId()); - case CUSTOMER -> ownerEntities.remove(msg.getEntityId()); + if (msg.isRelationChanged()) { + onRelationDeleted(msg, callback); + } else { + switch (msg.getEntityId().getEntityType()) { + case DEVICE, ASSET -> entityProfileCache.removeEntityId(msg.getEntityId()); + case CUSTOMER -> ownerEntities.remove(msg.getEntityId()); + } + ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); + + getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getProfileId()).forEach(ctx -> { + applyToTargetCfEntityActors(ctx, callback, (id, cb) -> deleteRelatedEntity(id, msg.getEntityId(), cb)); + }); + if (isMyPartition(msg.getEntityId(), callback)) { + log.debug("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); + getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); + } } - ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); - if (isMyPartition(msg.getEntityId(), callback)) { - log.debug("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); - getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); + } + + private void onRelationUpdated(ComponentLifecycleMsg msg, TbCallback callback) { + try { + MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); + EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); + + EntityId toId = entityRelation.getTo(); + EntityId fromId = entityRelation.getFrom(); + String relationType = entityRelation.getType(); + EntityId toIdProfile = getProfileId(tenantId, toId); + EntityId fromIdProfile = getProfileId(tenantId, fromId); + + List toIdMatches = new ArrayList<>(); + List cfsByToId = getCalculatedFieldsByEntityId(toId); + List cfsByToProfileId = getCalculatedFieldsByEntityId(toIdProfile); + List cfsByToIdOrItsProfileId = new ArrayList<>(); + cfsByToIdOrItsProfileId.addAll(cfsByToId); + cfsByToIdOrItsProfileId.addAll(cfsByToProfileId); + + cfsByToIdOrItsProfileId.forEach(cf -> { + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + AggSource source = configuration.getSource(); + RelationPathLevel relation = source.getRelation(); + if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(fromIdProfile)) { + toIdMatches.add(cf); + } + }); + + MultipleTbCallback toCfsCallback = new MultipleTbCallback(toIdMatches.size(), callbackForToAndFrom); + toIdMatches.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, toCfsCallback, (entityId, cb) -> initRelatedEntity(entityId, fromId, ctx, cb)); + }); + + List fromIdMatches = new ArrayList<>(); + List cfsByFromId = getCalculatedFieldsByEntityId(fromId); + List cfsByFromProfileId = getCalculatedFieldsByEntityId(fromIdProfile); + List cfsByFromIdOrItsProfileId = new ArrayList<>(); + cfsByFromIdOrItsProfileId.addAll(cfsByFromId); + cfsByFromIdOrItsProfileId.addAll(cfsByFromProfileId); + + cfsByFromIdOrItsProfileId.forEach(cf -> { + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + AggSource source = configuration.getSource(); + RelationPathLevel relation = source.getRelation(); + if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(toIdProfile)) { + fromIdMatches.add(cf); + } + }); + + MultipleTbCallback fromCfsCallback = new MultipleTbCallback(fromIdMatches.size(), callbackForToAndFrom); + fromIdMatches.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, fromCfsCallback, (entityId, cb) -> initRelatedEntity(entityId, toId, ctx, cb)); + }); + + + } catch (Exception e) { + callback.onSuccess(); + } + } + + private void onRelationDeleted(ComponentLifecycleMsg msg, TbCallback callback) { + try { + MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); + EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); + + EntityId toId = entityRelation.getTo(); + EntityId fromId = entityRelation.getFrom(); + String relationType = entityRelation.getType(); + EntityId toIdProfile = getProfileId(tenantId, toId); + EntityId fromIdProfile = getProfileId(tenantId, fromId); + + List toIdMatches = new ArrayList<>(); + List cfsByToId = getCalculatedFieldsByEntityId(toId); + List cfsByToProfileId = getCalculatedFieldsByEntityId(toIdProfile); + List cfsByToIdOrItsProfileId = new ArrayList<>(); + cfsByToIdOrItsProfileId.addAll(cfsByToId); + cfsByToIdOrItsProfileId.addAll(cfsByToProfileId); + + cfsByToIdOrItsProfileId.forEach(cf -> { + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + AggSource source = configuration.getSource(); + RelationPathLevel relation = source.getRelation(); + if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(fromIdProfile)) { + toIdMatches.add(cf); + } + }); + + MultipleTbCallback toCfsCallback = new MultipleTbCallback(toIdMatches.size(), callbackForToAndFrom); + toIdMatches.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, toCfsCallback, (entityId, cb) -> deleteRelatedEntity(entityId, fromId, cb)); + }); + + List fromIdMatches = new ArrayList<>(); + List cfsByFromId = getCalculatedFieldsByEntityId(fromId); + List cfsByFromProfileId = getCalculatedFieldsByEntityId(fromIdProfile); + List cfsByFromIdOrItsProfileId = new ArrayList<>(); + cfsByFromIdOrItsProfileId.addAll(cfsByFromId); + cfsByFromIdOrItsProfileId.addAll(cfsByFromProfileId); + + cfsByFromIdOrItsProfileId.forEach(cf -> { + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + AggSource source = configuration.getSource(); + RelationPathLevel relation = source.getRelation(); + if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(toIdProfile)) { + fromIdMatches.add(cf); + } + }); + + MultipleTbCallback fromCfsCallback = new MultipleTbCallback(fromIdMatches.size(), callbackForToAndFrom); + fromIdMatches.forEach(ctx -> { + applyToTargetCfEntityActors(ctx, fromCfsCallback, (entityId, cb) -> deleteRelatedEntity(entityId, toId, cb)); + }); + + + } catch (Exception e) { + callback.onSuccess(); } } @@ -302,6 +481,9 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } calculatedFields.put(cf.getId(), cfCtx); + if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + } // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); @@ -333,6 +515,9 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(newCfCtx).eventEntity(newCfCtx.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(newCf.getId(), newCfCtx); + if (newCf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + cfTriggers.put(newCf.getId(), aggConfig.buildTrigger()); + } List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new CopyOnWriteArrayList<>(); boolean found = false; @@ -417,6 +602,106 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } else { callback.onSuccess(); } + // process all aggregation cfs (if any); + List aggregationCalculatedFields = filterAggregationCfs(msg); + if (!aggregationCalculatedFields.isEmpty()) { + cfExecService.pushMsgToLinks(msg, aggregationCalculatedFields, callback); + } else { + callback.onSuccess(); + } + } + + private List filterAggregationCfs(CalculatedFieldTelemetryMsg msg) { + EntityId entityId = msg.getEntityId(); + + List aggregationCalculatedFields = cfTriggers.entrySet().stream() + .filter(entry -> aggMatches(entry.getValue(), msg.getProto())) + .map(Entry::getKey) + .map(calculatedFields::get) + .filter(Objects::nonNull) + .toList(); + + List filteredByRelationCfs = new ArrayList<>(); + for (CalculatedFieldCtx cf : aggregationCalculatedFields) { + EntityId cfEntityId = cf.getEntityId(); + if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + RelationPathLevel relation = aggConfig.getSource().getRelation(); + EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) + ? cfEntityId + : getProfileId(tenantId, cfEntityId); + EntityId targetEntity = switch (relation.direction()) { + case FROM -> + relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).getFrom(); + case TO -> + relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0).getTo(); + }; + if (targetEntity != null) { + filteredByRelationCfs.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), targetEntity)); + } + } + } + return filteredByRelationCfs; + } + + private List getCalculatedFieldsRelatedToEntity(EntityId entityId, EntityId profileId) { + List aggCFsUsedProfile = cfTriggers.entrySet().stream() + .filter(entry -> entry.getValue().matchesProfile(profileId)) + .map(Entry::getKey) + .map(calculatedFields::get) + .filter(Objects::nonNull) + .toList(); + + List filteredByRelationCfs = new ArrayList<>(); + for (CalculatedFieldCtx cf : aggCFsUsedProfile) { + CalculatedFieldEntityCtxId calculatedFieldEntityCtxId = filterCfByRelationWithEntity(entityId, cf); + if (calculatedFieldEntityCtxId != null) { + filteredByRelationCfs.add(cf); + } + } + return filteredByRelationCfs; + } + + private CalculatedFieldEntityCtxId filterCfByRelationWithEntity(EntityId entityId, CalculatedFieldCtx cf) { + EntityId cfEntityId = cf.getEntityId(); + if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + RelationPathLevel relation = aggConfig.getSource().getRelation(); + EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) + ? cfEntityId + : getProfileId(tenantId, cfEntityId); + EntityId targetEntity = switch (relation.direction()) { + case FROM -> { + EntityRelation entityRelation = relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); + yield entityRelation == null ? null : entityRelation.getFrom(); + } + case TO -> { + EntityRelation entityRelation = relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0); + yield entityRelation == null ? null : entityRelation.getTo(); + } + }; + if (targetEntity != null) { + return new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), targetEntity); + } + } + return null; + } + + private boolean aggMatches(CfAggTrigger cfAggTrigger, CalculatedFieldTelemetryMsgProto proto) { + if (!proto.getTsDataList().isEmpty()) { + List updatedTelemetry = proto.getTsDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return cfAggTrigger.matchesTimeSeries(updatedTelemetry); + } else if (!proto.getAttrDataList().isEmpty()) { + AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); + List updatedTelemetry = proto.getAttrDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return cfAggTrigger.matchesAttributes(updatedTelemetry, scope); + } else if (!proto.getRemovedTsKeysList().isEmpty()) { + return cfAggTrigger.matchesTimeSeriesKeys(proto.getRemovedTsKeysList()); + } else { + return cfAggTrigger.matchesAttributesKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); + } } public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsg msg) { @@ -536,6 +821,16 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware getOrCreateActor(entityId).tell(msg); } + private void deleteRelatedEntity(EntityId entityId, EntityId relatedEntityId, TbCallback callback) { + log.debug("Pushing delete related entity msg to specific actor [{}]", relatedEntityId); + getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, relatedEntityId, callback)); + } + + private void initRelatedEntity(EntityId entityId, EntityId relatedEntityId, CalculatedFieldCtx cf, TbCallback callback) { + log.debug("Pushing init related entity msg to specific actor [{}]", relatedEntityId); + getOrCreateActor(entityId).tell(new CalculatedFieldRelatedEntityMsg(tenantId, relatedEntityId, cf, callback)); + } + private void deleteCfForEntity(EntityId entityId, CalculatedFieldId cfId, TbCallback callback) { log.debug("Pushing delete CF msg to specific actor [{}]", entityId); getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, cfId, callback)); @@ -614,6 +909,9 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(cf.getId(), cfCtx); + if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + } // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java new file mode 100644 index 0000000000..d3bdd9cc5c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +@Data +public class CalculatedFieldRelatedEntityMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldCtx calculatedField; + private final TbCallback callback; + + public CalculatedFieldRelatedEntityMsg(TenantId tenantId, + EntityId entityId, + CalculatedFieldCtx calculatedField, + TbCallback callback) { + this.tenantId = tenantId; + this.entityId = entityId; + this.calculatedField = calculatedField; + this.callback = callback; + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_RELATED_ENTITY_MSG; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index d17148a502..304b747f26 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -26,7 +26,10 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; @@ -36,6 +39,8 @@ import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.relation.RelationService; @@ -44,7 +49,9 @@ import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -98,6 +105,7 @@ public abstract class AbstractCalculatedFieldProcessingService { } yield futures; } + case LATEST_VALUES_AGGREGATION -> fetchAggregationArgumentFutures(ctx, entityId); }; return Futures.whenAllComplete(argFutures.values()) .call(() -> resolveArgumentFutures(argFutures), @@ -114,6 +122,19 @@ public abstract class AbstractCalculatedFieldProcessingService { return resolveOwnerArgument(tenantId, entityId); } + private List resolveRelatedEntities(TenantId tenantId, EntityId entityId, AggSource aggSource) { + RelationPathLevel relation = aggSource.getRelation(); + return switch (relation.direction()) { + case FROM -> aggSource.getEntityProfiles().stream() + .map(profile -> relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), profile)) + .flatMap(Collection::stream) + .map(EntityRelation::getTo) + .toList(); + case TO -> + aggSource.getEntityProfiles().stream().map(profile -> relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), profile).getFrom()).toList(); + }; + } + protected Map resolveArgumentFutures(Map> argFutures) { return argFutures.entrySet().stream() .collect(Collectors.toMap( @@ -176,6 +197,50 @@ public abstract class AbstractCalculatedFieldProcessingService { return ownerService.getOwner(tenantId, entityId); } + private Map> fetchAggregationArgumentFutures(CalculatedFieldCtx ctx, EntityId entityId) { + LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + + List entityIds = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getSource()); + + Map> futures = new HashMap<>(); + aggConfig.getInputs().forEach((key, refKey) -> { + Argument argument = new Argument(); + argument.setRefEntityKey(refKey); + futures.put(key, fetchAggArgumentEntry(ctx.getTenantId(), entityIds, argument, System.currentTimeMillis())); + }); + return futures; + } + + public ListenableFuture fetchAggArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) { + List>> futures = aggEntities.stream() + .map(entityId -> fetchSingleAggArgumentEntry(tenantId, entityId, argument, startTs)) + .toList(); + + ListenableFuture>> allFutures = Futures.allAsList(futures); + + return Futures.transform(allFutures, + entries -> ArgumentEntry.createAggArgument( + entries.stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) + ), + MoreExecutors.directExecutor()); + } + + protected ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { + CalculatedFieldConfiguration configuration = ctx.getCalculatedField().getConfiguration(); + LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) configuration; + Map> futures = new HashMap<>(); + aggConfig.getInputs().forEach((key, refKey) -> { + Argument argument = new Argument(); + argument.setRefEntityKey(refKey); + + ListenableFuture argumentEntryListenableFuture = fetchAggArgumentEntry(ctx.getTenantId(), List.of(entityId), argument, System.currentTimeMillis()); + futures.put(key, argumentEntryListenableFuture); + }); + return Futures.whenAllComplete(futures.values()) + .call(() -> resolveArgumentFutures(futures), + MoreExecutors.directExecutor()); + } + private ListenableFuture fetchGeofencingKvEntry(TenantId tenantId, List geofencingEntities, Argument argument) { if (argument.getRefEntityKey().getType() != ArgumentType.ATTRIBUTE) { throw new IllegalStateException("Unsupported argument key type: " + argument.getRefEntityKey().getType()); @@ -244,6 +309,38 @@ public abstract class AbstractCalculatedFieldProcessingService { }, calculatedFieldCallbackExecutor)); } + protected ListenableFuture> fetchSingleAggArgumentEntry(TenantId tenantId, EntityId entityId, Argument argument, long startTs) { + return switch (argument.getRefEntityKey().getType()) { + case TS_ROLLING -> throw new IllegalStateException("TS_ROLLING is not supported for aggregation"); + case ATTRIBUTE -> fetchAttributeAggEntry(tenantId, entityId, argument, startTs); + case TS_LATEST -> fetchTsLatestAggEntry(tenantId, entityId, argument, startTs); + }; + } + + private ListenableFuture> fetchAttributeAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultLastUpdateTs) { + log.trace("[{}][{}] Fetching attribute for key {}", tenantId, entityId, argument.getRefEntityKey()); + var attributeOptFuture = attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()); + return Futures.transform(attributeOptFuture, attrOpt -> { + log.debug("[{}][{}] Fetched attribute for key {}: {}", tenantId, entityId, argument.getRefEntityKey(), attrOpt); + AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, 0L)); + AggSingleArgumentEntry entry = new AggSingleArgumentEntry(entityId, attributeKvEntry); + return Map.entry(entityId, entry); + }, calculatedFieldCallbackExecutor); + } + + protected ListenableFuture> fetchTsLatestAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultTs) { + String key = argument.getRefEntityKey().getKey(); + log.trace("[{}][{}] Fetching latest timeseries {}", tenantId, entityId, key); + return Futures.transform( + timeseriesService.findLatest(tenantId, entityId, key), + result -> { + log.debug("[{}][{}] Fetched latest timeseries {}: {}", tenantId, entityId, key, result); + Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))); + AggSingleArgumentEntry entry = new AggSingleArgumentEntry(entityId, tsKvEntry.get()); + return Map.entry(entityId, entry); + }, calculatedFieldCallbackExecutor); + } + private ReadTsKvQuery buildTsRollingQuery(TenantId tenantId, Argument argument, long startTs, long endTs) { long maxDataPoints = apiLimitService.getLimit( tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index cc77913f4b..e32ca42f9c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -38,6 +39,8 @@ public interface CalculatedFieldCache { List getCalculatedFieldCtxsByEntityId(EntityId entityId); + List getCalculatedFieldCtxsByTrigger(EntityId profileId, Predicate cfAggFilter); + boolean hasCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter); void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java index a9139572b8..4b3e994f23 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -33,6 +33,8 @@ public interface CalculatedFieldProcessingService { ListenableFuture> fetchArguments(CalculatedFieldCtx ctx, EntityId entityId); + ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId); + Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId); Map fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map arguments); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 36210d7302..ae8238fb8a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; @@ -41,6 +43,8 @@ import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -48,6 +52,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Predicate; +import java.util.stream.Collectors; @Service @Slf4j @@ -68,6 +73,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); + private final ConcurrentMap cfTriggers = new ConcurrentHashMap<>(); private final ConcurrentMap> ownerEntities = new ConcurrentHashMap<>(); @@ -81,6 +87,9 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { cfs.forEach(cf -> { if (cf != null) { calculatedFields.putIfAbsent(cf.getId(), cf); + if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + } } }); calculatedFields.values().forEach(cf -> { @@ -146,6 +155,16 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { .toList(); } + @Override + public List getCalculatedFieldCtxsByTrigger(EntityId profileId, Predicate cfAggFilter) { + return cfTriggers.entrySet().stream() + .filter(entry -> entry.getValue().matches(profileId, cfAggFilter)) + .map(Map.Entry::getKey) + .map(this::getCalculatedFieldCtx) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + @Override public boolean hasCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter) { List entityCfs = getCalculatedFieldCtxsByEntityId(entityId); @@ -155,6 +174,10 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } } + return hasCalculatedFieldsByProfile(tenantId, entityId, filter); + } + + public boolean hasCalculatedFieldsByProfile(TenantId tenantId, EntityId entityId, Predicate filter) { EntityId profileId = getProfileId(tenantId, entityId); if (profileId != null) { List profileCfs = getCalculatedFieldCtxsByEntityId(profileId); @@ -183,6 +206,9 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new CopyOnWriteArrayList<>()).add(calculatedField); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); + if (configuration instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + cfTriggers.put(calculatedField.getId(), aggConfig.buildTrigger()); + } calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); configuration.getReferencedEntities().stream() @@ -214,6 +240,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); entityIdCalculatedFieldLinks.forEach((entityId, calculatedFieldLinks) -> calculatedFieldLinks.removeIf(link -> link.getCalculatedFieldId().equals(calculatedFieldId))); log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); + cfTriggers.remove(calculatedFieldId); + log.debug("[{}] evict calculated field from cached triggers: {}", calculatedFieldId, oldCalculatedField); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index 9b2964a736..b7bd4d87fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.service.cf; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; @@ -54,6 +56,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; @@ -87,6 +90,11 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF return super.fetchArguments(ctx, entityId, System.currentTimeMillis()); } + @Override + public ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId) { + return super.fetchAggArguments(ctx, entityId, System.currentTimeMillis()); + } + @Override public Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId) { // only scheduledSupported CF instances supports dynamic arguments scheduled updates diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index a3e049d1e2..a75df1fb40 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -27,6 +27,8 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -35,7 +37,10 @@ import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -71,6 +76,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS private final CalculatedFieldCache calculatedFieldCache; private final TbClusterService clusterService; + private final RelationService relationService; @Override public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback) { @@ -81,6 +87,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matches(entries), cf -> cf.linkMatches(entityId, entries), cf -> cf.dynamicSourceMatches(request.getEntries()), + cfTrigger -> cfTrigger.matchesTimeSeries(entries), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -99,6 +106,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matches(entries, scope), cf -> cf.linkMatches(entityId, entries, scope), cf -> cf.dynamicSourceMatches(request.getEntries(), request.getScope()), + cfTrigger -> cfTrigger.matchesAttributes(entries, scope), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -116,6 +124,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matchesKeys(result, scope), cf -> cf.linkMatchesAttrKeys(entityId, result, scope), cf -> cf.matchesDynamicSourceKeys(result, request.getScope()), + cfTrigger -> cfTrigger.matchesAttributesKeys(result, scope), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -127,6 +136,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matchesKeys(result), cf -> cf.linkMatchesTsKeys(entityId, result), cf -> cf.matchesDynamicSourceKeys(result), + cfTrigger -> cfTrigger.matchesTimeSeriesKeys(result), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -134,11 +144,12 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS Predicate mainEntityFilter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter, + Predicate cfAggKeysFilter, Supplier msg, FutureCallback callback) { if (EntityType.TENANT.equals(entityId.getEntityType())) { tenantId = (TenantId) entityId; } - boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter, dynamicSourceFilter); + boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter, dynamicSourceFilter, cfAggKeysFilter); if (send) { ToCalculatedFieldMsg calculatedFieldMsg = msg.get(); clusterService.pushMsgToCalculatedFields(tenantId, entityId, calculatedFieldMsg, wrap(callback)); @@ -149,7 +160,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } } - private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter) { + private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter, Predicate cfAggKeysFilter) { if (!CalculatedField.SUPPORTED_REFERENCED_ENTITIES.contains(entityId.getEntityType())) { return false; } @@ -176,6 +187,26 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } } + List cfCtxs = calculatedFieldCache.getCalculatedFieldCtxsByTrigger(calculatedFieldCache.getProfileId(tenantId, entityId), cfAggKeysFilter); + for (CalculatedFieldCtx cfCtx : cfCtxs) { + EntityId cfEntityId = cfCtx.getEntityId(); + if (cfCtx.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + RelationPathLevel relation = aggConfig.getSource().getRelation(); + EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) + ? cfEntityId + : calculatedFieldCache.getProfileId(tenantId, cfEntityId); + EntityRelation entityRelation = switch (relation.direction()) { + case FROM -> + relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); + case TO -> + relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0); + }; + if (entityRelation != null) { + return true; + } + } + } + return false; } @@ -266,6 +297,10 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS return telemetryMsg; } + private boolean isProfileEntity(EntityType entityType) { + return EntityType.DEVICE_PROFILE.equals(entityType) || EntityType.ASSET_PROFILE.equals(entityType); + } + private static TbQueueCallback wrap(FutureCallback callback) { if (callback != null) { return new FutureCallbackWrapper(callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 2d43883131..f22b10a9c1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -22,6 +22,8 @@ import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import java.util.List; @@ -35,7 +37,9 @@ import java.util.Map; @JsonSubTypes({ @JsonSubTypes.Type(value = SingleValueArgumentEntry.class, name = "SINGLE_VALUE"), @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING"), - @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING") + @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING"), + @JsonSubTypes.Type(value = AggArgumentEntry.class, name = "AGGREGATE_LATEST"), + @JsonSubTypes.Type(value = AggSingleArgumentEntry.class, name = "AGGREGATE_LATEST_SINGLE") }) public interface ArgumentEntry { @@ -66,4 +70,8 @@ public interface ArgumentEntry { return new GeofencingArgumentEntry(entityIdkvEntryMap); } + static ArgumentEntry createAggArgument(Map entityIdkvEntryMap) { + return new AggArgumentEntry(entityIdkvEntryMap, false); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java index 876bfa2a3f..30b92a78d3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java @@ -16,5 +16,5 @@ package org.thingsboard.server.service.cf.ctx.state; public enum ArgumentEntryType { - SINGLE_VALUE, TS_ROLLING, GEOFENCING + SINGLE_VALUE, TS_ROLLING, GEOFENCING, AGGREGATE_LATEST, AGGREGATE_LATEST_SINGLE } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index e442964280..f75711a107 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -123,7 +123,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, protected void validateNewEntry(String key, ArgumentEntry newEntry) {} - private void updateLastUpdateTimestamp(ArgumentEntry entry) { + protected void updateLastUpdateTimestamp(ArgumentEntry entry) { long newTs = this.latestTimestamp; if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { newTs = singleValueArgumentEntry.getTs(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index e08abb8b50..963ec3f06f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -42,6 +42,8 @@ import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.ScheduledUpdateSupportedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -81,6 +83,8 @@ public class CalculatedFieldCtx { private final Map mainEntityArguments; private final Map> linkedEntityArguments; private final Map dynamicEntityArguments; + private final List aggInputs; + private final Map aggregationInputs; private final List argNames; private Output output; private String expression; @@ -122,6 +126,8 @@ public class CalculatedFieldCtx { this.argNames = new ArrayList<>(); this.mainEntityGeofencingArgumentNames = new ArrayList<>(); this.linkedEntityAndCurrentOwnerGeofencingArgumentNames = new ArrayList<>(); + this.aggInputs = new ArrayList<>(); + this.aggregationInputs = new HashMap<>(); this.output = calculatedField.getConfiguration().getOutput(); if (calculatedField.getConfiguration() instanceof ArgumentsBasedCalculatedFieldConfiguration argBasedConfig) { this.arguments.putAll(argBasedConfig.getArguments()); @@ -165,6 +171,12 @@ public class CalculatedFieldCtx { this.scheduledUpdateIntervalMillis = scheduledConfig.isScheduledUpdateEnabled() ? TimeUnit.SECONDS.toMillis(scheduledConfig.getScheduledUpdateInterval()) : -1L; } this.requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); + if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + aggInputs.addAll(aggConfig.getInputs().values()); + aggregationInputs.putAll(aggConfig.getInputs()); + this.argNames.addAll(aggConfig.getInputs().keySet()); + this.scheduledUpdateIntervalMillis = aggConfig.getDeduplicationIntervalMillis(); + } this.systemContext = systemContext; this.tbelInvokeService = systemContext.getTbelInvokeService(); this.relationService = systemContext.getRelationService(); @@ -199,6 +211,19 @@ public class CalculatedFieldCtx { }); initialized = true; } + case LATEST_VALUES_AGGREGATION -> { + LatestValuesAggregationCalculatedFieldConfiguration configuration = (LatestValuesAggregationCalculatedFieldConfiguration) calculatedField.getConfiguration(); + configuration.getMetrics().forEach((key, metric) -> { + if (metric.getInput() instanceof AggFunctionInput functionInput) { + initTbelExpression(functionInput.getFunction()); + } + String filter = metric.getFilter(); + if (filter != null && !filter.isEmpty()) { + initTbelExpression(filter); + } + }); + initialized = true; + } } } @@ -242,6 +267,24 @@ public class CalculatedFieldCtx { return expression.executeScriptAsync(args.toArray()); } + public ListenableFuture evaluateTbelExpression(String expression, Map entries, long latestTimestamp) { + Map arguments = new LinkedHashMap<>(); + List args = new ArrayList<>(argNames.size() + 1); + args.add(new Object()); // first element is a ctx, but we will set it later; + for (String argName : argNames) { + var arg = entries.get(argName).toTbelCfArg(); + arguments.put(argName, arg); + if (arg instanceof TbelCfSingleValueArg svArg) { + args.add(svArg.getValue()); + } else { + args.add(arg); + } + } + args.set(0, new TbelCfCtx(arguments, latestTimestamp)); + + return tbelExpressions.get(expression).executeScriptAsync(args.toArray()); + } + public ScheduledFuture scheduleReevaluation(long delayMs, TbActorRef actorCtx) { log.debug("[{}] Scheduling CF reevaluation in {} ms", cfId, delayMs); // TODO: use single lazy-loaded instance of CalculatedFieldReevaluateMsg @@ -451,6 +494,25 @@ public class CalculatedFieldCtx { } } + public boolean aggMatches(CalculatedFieldTelemetryMsgProto proto) { + if (!proto.getTsDataList().isEmpty()) { + List updatedTelemetry = proto.getTsDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return matchesAggTimeSeries(updatedTelemetry); + } else if (!proto.getAttrDataList().isEmpty()) { + AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); + List updatedTelemetry = proto.getAttrDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return matchesAggAttributes(updatedTelemetry, scope); + } else if (!proto.getRemovedTsKeysList().isEmpty()) { + return matchesAggKeys(proto.getRemovedTsKeysList()); + } else { + return matchesAggAttributesKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); + } + } + public boolean linkMatches(EntityId entityId, CalculatedFieldTelemetryMsgProto proto) { if (!proto.getTsDataList().isEmpty()) { List updatedTelemetry = proto.getTsDataList().stream() @@ -482,6 +544,67 @@ public class CalculatedFieldCtx { return argNames; } + public boolean matchesAggKeys(List values) { + if (aggInputs.isEmpty() || values.isEmpty()) { + return false; + } + + for (String key : values) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); + if (aggInputs.contains(latestKey)) { + return true; + } + } + + return false; + } + + public boolean matchesAggTimeSeries(List values) { + if (aggInputs.isEmpty() || values.isEmpty()) { + return false; + } + + for (TsKvEntry tsKvEntry : values) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKvEntry.getKey(), ArgumentType.TS_LATEST, null); + if (aggInputs.contains(latestKey)) { + return true; + } + } + + return false; + } + + public boolean matchesAggAttributesKeys(List keys, AttributeScope scope) { + if (keys == null || keys.isEmpty()) { + return false; + } + + for (String key : keys) { + ReferencedEntityKey attrKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); + if (aggInputs.contains(attrKey)) { + return true; + } + } + + return false; + } + + public boolean matchesAggAttributes(List keys, AttributeScope scope) { + if (keys == null || keys.isEmpty()) { + return false; + } + + for (AttributeKvEntry attributeKvEntry : keys) { + ReferencedEntityKey attrKey = new ReferencedEntityKey(attributeKvEntry.getKey(), ArgumentType.ATTRIBUTE, scope); + if (aggInputs.contains(attrKey)) { + return true; + } + } + + return false; + } + + public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); } @@ -499,6 +622,11 @@ public class CalculatedFieldCtx { if (scheduledUpdateIntervalMillis != other.scheduledUpdateIntervalMillis) { return true; } + if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig + && other.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig + && !thisConfig.getMetrics().equals(otherConfig.getMetrics())) { + return true; + } return false; } @@ -517,6 +645,9 @@ public class CalculatedFieldCtx { if (hasGeofencingZoneGroupConfigurationChanges(other)) { return true; } + if (hasLatestValuesAggregationConfigurationChanges(other)) { + return true; + } return false; } @@ -528,6 +659,15 @@ public class CalculatedFieldCtx { return false; } + private boolean hasLatestValuesAggregationConfigurationChanges(CalculatedFieldCtx other) { + if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig + && other.calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig) { + return !thisConfig.getInputs().equals(otherConfig.getInputs()) || !thisConfig.getSource().equals(otherConfig.getSource()); + } + return false; + } + + public boolean hasRelationQueryDynamicArguments() { return relationQueryDynamicArguments && scheduledUpdateIntervalMillis != -1; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index cc7188e7a5..bd40ab7a05 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -32,6 +32,7 @@ import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculat import java.io.Closeable; import java.util.Map; +import java.util.concurrent.ExecutionException; import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArgumentProto; @@ -53,6 +54,8 @@ public interface CalculatedFieldState extends Closeable { long getLatestTimestamp(); + CalculatedFieldCtx getCtx(); + void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx); void init(); @@ -61,7 +64,7 @@ public interface CalculatedFieldState extends Closeable { void reset(); - ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx); + ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception; @JsonIgnore boolean isReady(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 5c1ed32e1d..288b486e83 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -37,11 +37,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; @AllArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { - private long ts; - private BasicKvEntry kvEntryValue; - private Long version; + protected long ts; + protected BasicKvEntry kvEntryValue; + protected Long version; - private boolean forceResetPrevious; + protected boolean forceResetPrevious; public static final Long DEFAULT_VERSION = -1L; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java new file mode 100644 index 0000000000..138053793f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; + +import java.util.Map; + +@Data +@AllArgsConstructor +public class AggArgumentEntry implements ArgumentEntry { + + private final Map aggInputs; + + private boolean forceResetPrevious; + + @Override + public ArgumentEntryType getType() { + return ArgumentEntryType.AGGREGATE_LATEST; + } + + @Override + public Object getValue() { + return aggInputs; + } + + @Override + public boolean updateEntry(ArgumentEntry entry) { + if (entry instanceof AggArgumentEntry aggArgumentEntry) { + aggInputs.putAll(aggArgumentEntry.aggInputs); + return true; + } else if (entry instanceof AggSingleArgumentEntry aggSingleArgumentEntry) { + if (aggSingleArgumentEntry.isDeleted()) { + aggInputs.remove(aggSingleArgumentEntry.getEntityId()); + } else { + aggInputs.put(aggSingleArgumentEntry.getEntityId(), aggSingleArgumentEntry); + } + return true; + } else { + throw new IllegalArgumentException("Unsupported argument entry type for aggregation argument entry: " + entry.getType()); + } + } + + @Override + public boolean isEmpty() { + return aggInputs.isEmpty(); + } + + @Override + public TbelCfArg toTbelCfArg() { + return null; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java new file mode 100644 index 0000000000..6b81d5380c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java @@ -0,0 +1,83 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class AggSingleArgumentEntry extends SingleValueArgumentEntry { + + private EntityId entityId; + private boolean deleted; + + public AggSingleArgumentEntry(EntityId entityId, TsKvProto entry) { + super(entry); + this.entityId = entityId; + } + + public AggSingleArgumentEntry(EntityId entityId, AttributeValueProto entry) { + super(entry); + this.entityId = entityId; + } + + public AggSingleArgumentEntry(EntityId entityId, KvEntry entry) { + super(entry); + this.entityId = entityId; + } + + public AggSingleArgumentEntry(EntityId entityId, long ts, BasicKvEntry kvEntryValue, Long version) { + super(ts, kvEntryValue, version); + this.entityId = entityId; + } + + @Override + public boolean updateEntry(ArgumentEntry entry) { + if (entry instanceof AggSingleArgumentEntry singleValueEntry) { + if (singleValueEntry.getTs() <= ts) { + return false; + } + + Long newVersion = singleValueEntry.getVersion(); + if (newVersion == null || this.version == null || newVersion > this.version) { + this.ts = singleValueEntry.getTs(); + this.version = newVersion; + this.kvEntryValue = singleValueEntry.getKvEntryValue(); + this.entityId = singleValueEntry.getEntityId(); + return true; + } + } else { + throw new IllegalArgumentException("Unsupported argument entry type for single value argument entry: " + entry.getType()); + } + return false; + } + + @Override + public ArgumentEntryType getType() { + return ArgumentEntryType.AGGREGATE_LATEST_SINGLE; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java new file mode 100644 index 0000000000..d25cde020c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java @@ -0,0 +1,161 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.actors.TbActorRef; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.BaseCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.aggregation.function.AggEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.function.AggFunctionFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +@Slf4j +@Data +public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedFieldState { + + private long lastArgsRefreshTs = -1; + private long lastMetricsEvalTs = -1; + private long deduplicationInterval = -1; + private Map metrics; + + private final Map> inputs = new HashMap<>(); + + public LatestValuesAggregationCalculatedFieldState(EntityId entityId) { + super(entityId); + } + + @Override + public void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx) { + super.setCtx(ctx, actorCtx); + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + metrics = configuration.getMetrics(); + deduplicationInterval = configuration.getDeduplicationIntervalMillis(); + } + + @Override + public void reset() { // must reset everything dependent on arguments + super.reset(); + lastArgsRefreshTs = -1; + lastMetricsEvalTs = -1; + metrics = null; + } + + @Override + public void init() { + super.init(); +// long scheduledUpdateIntervalMillis = ctx.getScheduledUpdateIntervalMillis(); +// ctx.scheduleReevaluation(scheduledUpdateIntervalMillis, actorCtx); + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.LATEST_VALUES_AGGREGATION; + } + + @Override + public Map update(Map argumentValues, CalculatedFieldCtx ctx) { + Map updatedArguments = super.update(argumentValues, ctx); + lastArgsRefreshTs = System.currentTimeMillis(); + for (Map.Entry argEntry : arguments.entrySet()) { + String key = argEntry.getKey(); + AggArgumentEntry aggArgumentEntry = (AggArgumentEntry) argEntry.getValue(); + Map aggInputs = aggArgumentEntry.getAggInputs(); + aggInputs.forEach((entityId, argumentEntry) -> { + inputs.computeIfAbsent(entityId, k -> new HashMap<>()).put(key, argumentEntry); + }); + } + return updatedArguments; + } + + @Override + public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception { + boolean intervalPassed = lastMetricsEvalTs <= System.currentTimeMillis() - deduplicationInterval; + boolean argsUpdatedDuringInterval = lastArgsRefreshTs > lastMetricsEvalTs; + if (intervalPassed && argsUpdatedDuringInterval) { + ObjectNode aggResult = JacksonUtil.newObjectNode(); + for (Entry entry : metrics.entrySet()) { + String metricKey = entry.getKey(); + AggMetric metric = entry.getValue(); + + AggEntry aggMetric = AggFunctionFactory.createAggFunction(metric.getFunction()); + + for (Map entityInputs : inputs.values()) { + if (applyAggregation(metric.getFilter(), entityInputs)) { + Object arg = resolveAggregationInput(metric.getInput(), entityInputs); + if (arg != null) { + aggMetric.update(arg); + } + } + } + + aggMetric.result().ifPresent(result -> { + aggResult.set(metricKey, JacksonUtil.valueToTree(result)); + }); + } + Output output = ctx.getOutput(); + lastMetricsEvalTs = System.currentTimeMillis(); + ctx.scheduleReevaluation(deduplicationInterval, actorCtx); + return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() + .type(output.getType()) + .scope(output.getScope()) + .result(aggResult) + .build()); + } else { + return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() + .result(null) + .build()); + } + } + + private boolean applyAggregation(String filter, Map entityInputs) throws Exception { + if (filter == null || filter.isEmpty()) { + return true; + } else { + Object filterResult = ctx.evaluateTbelExpression(filter, entityInputs, getLatestTimestamp()).get(); + return filterResult instanceof Boolean booleanResult && booleanResult; + } + } + + private Object resolveAggregationInput(AggInput aggInput, Map entityInputs) throws Exception { + if (aggInput instanceof AggFunctionInput functionInput) { + return ctx.evaluateTbelExpression(functionInput.getFunction(), entityInputs, getLatestTimestamp()).get(); + } else { + String inputKey = ((AggKeyInput) aggInput).getKey(); + return entityInputs.get(inputKey).getValue(); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json new file mode 100644 index 0000000000..d402f23c6e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json @@ -0,0 +1,65 @@ +{ + "type": "LATEST_VALUES_AGGREGATION", + "name": "Occupied spaces", + "debugSettings": { + "failuresEnabled": true, + "allEnabled": true, + "allEnabledUntil": 1769907492297 + }, + "entityId": { + "entityType": "ASSET", + "id": "cc830710-a4cf-11f0-87cb-2d6683c4fccf" + }, + "configuration": { + "type": "LATEST_VALUES_AGGREGATION", + "source": { + "relation": { + "direction": "FROM", + "relationType": "Contains" + }, + "entityProfiles": [ + { + "entityType": "DEVICE_PROFILE", + "id": "d7a05580-a4cf-11f0-87cb-2d6683c4fccf" + } + ] + }, + "inputs": { + "oc": { + "key": "occupied", + "type": "TS_LATEST" + } + }, + "deduplicationIntervalMillis": 10000, + "metrics": { + "totalSpaces": { + "function": "COUNT", + "input": { + "type": "function", + "function" : "return 1;" + } + }, + "occupiedSpaces": { + "function": "COUNT", + "filter": "return oc == true", + "input": { + "type": "key", + "key" : "oc" + } + }, + "freeSpaces": { + "function": "COUNT", + "filter": "return oc == false", + "input": { + "type": "key", + "key" : "oc" + } + } + }, + "output": { + "type": "TIME_SERIES", + "decimals": 2 + }, + "useLatestTsFromInputs": "true" + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java new file mode 100644 index 0000000000..4239e94fec --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +import java.util.Optional; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = AvgAggEntry.class, name = "AVG"), + @JsonSubTypes.Type(value = CountAggEntry.class, name = "COUNT"), + @JsonSubTypes.Type(value = CountUniqueAggEntry.class, name = "COUNT_UNIQUE"), + @JsonSubTypes.Type(value = MaxAggEntry.class, name = "MAX"), + @JsonSubTypes.Type(value = MinAggEntry.class, name = "MIN"), + @JsonSubTypes.Type(value = SumAggEntry.class, name = "SUM") +}) +public interface AggEntry { + + AggFunction getType(); + + void update(Object value); + + Optional result(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java new file mode 100644 index 0000000000..5ccc355b1f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +public class AggFunctionFactory { + + public static AggEntry createAggFunction(AggFunction function) { + return switch (function) { + case MIN -> new MinAggEntry(); + case MAX -> new MaxAggEntry(); + case SUM -> new SumAggEntry(); + case AVG -> new AvgAggEntry(); + case COUNT -> new CountAggEntry(); + case COUNT_UNIQUE -> new CountUniqueAggEntry(); + }; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java new file mode 100644 index 0000000000..ad1f2ee8a8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +import java.math.BigDecimal; +import java.math.RoundingMode; + +public class AvgAggEntry extends BaseAggEntry { + + private BigDecimal sum = BigDecimal.ZERO; + private long count = 0L; + + @Override + protected void doUpdate(double value) { + if (value != 0.0) { + sum = sum.add(BigDecimal.valueOf(value)); + } + this.count++; + } + + @Override + protected double prepareResult() { + return sum.divide(BigDecimal.valueOf(count), 2, RoundingMode.HALF_UP).doubleValue(); + } + + @Override + public AggFunction getType() { + return AggFunction.AVG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java new file mode 100644 index 0000000000..0c12bd13c0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import java.util.Optional; + +public abstract class BaseAggEntry implements AggEntry { + + private boolean hasResult = false; + + @Override + public void update(Object value) { + doUpdate(extractDoubleValue(value)); + hasResult = true; + } + + @Override + public Optional result() { + if (hasResult) { + hasResult = false; + return Optional.of(prepareResult()); + } else { + return Optional.empty(); + } + } + + protected abstract void doUpdate(double value); + + protected abstract double prepareResult(); + + protected double extractDoubleValue(Object value) { + try { + return Double.parseDouble(value.toString()); + } catch (Exception e) { + throw new NumberFormatException("Cannot parse value " + value.toString()); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java new file mode 100644 index 0000000000..469048d62d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +import java.util.Optional; + +public class CountAggEntry implements AggEntry { + + private long count = 0L; + + @Override + public void update(Object value) { + count++; + } + + @Override + public Optional result() { + return Optional.of(count); + } + + @Override + public AggFunction getType() { + return AggFunction.COUNT; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java new file mode 100644 index 0000000000..b8b0b92470 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; + +public class CountUniqueAggEntry implements AggEntry { + + private Set items; + + @Override + public void update(Object value) { + if (value != null) { + items.add(JacksonUtil.toString(value)); + } + } + + @Override + public Optional result() { + return Optional.of(items.size()); + } + + @Override + public AggFunction getType() { + return AggFunction.COUNT_UNIQUE; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java new file mode 100644 index 0000000000..6e4235b72f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +public class MaxAggEntry extends BaseAggEntry { + + private double max = -Double.MAX_VALUE; + + @Override + protected void doUpdate(double value) { + if (value > max) { + max = value; + } + } + + @Override + protected double prepareResult() { + return max; + } + + @Override + public AggFunction getType() { + return AggFunction.MAX; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java new file mode 100644 index 0000000000..eeacc3a7d9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +public class MinAggEntry extends BaseAggEntry { + + private double min = Double.MAX_VALUE; + + @Override + protected void doUpdate(double value) { + if (value < min) { + min = value; + } + } + + @Override + protected double prepareResult() { + return min; + } + + @Override + public AggFunction getType() { + return AggFunction.MIN; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java new file mode 100644 index 0000000000..b90817d784 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state.aggregation.function; + +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; + +import java.math.BigDecimal; + +public class SumAggEntry extends BaseAggEntry { + + private BigDecimal sum = BigDecimal.ZERO; + + @Override + protected void doUpdate(double value) { + if (value != 0.0) { + sum = sum.add(BigDecimal.valueOf(value)); + } + } + + @Override + protected double prepareResult() { + return sum.doubleValue(); + } + + @Override + public AggFunction getType() { + return AggFunction.SUM; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index b9fd38f1e2..f762cf5be5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; @@ -61,6 +62,7 @@ import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.edge.EdgeSynchronizationManager; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; +import org.thingsboard.server.dao.eventsourcing.RelationActionEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.gen.transport.TransportProtos.EntityActionEventProto; @@ -271,6 +273,15 @@ public class EntityStateSourcingListener { } } + @TransactionalEventListener(fallbackExecution = true) + public void handleEvent(RelationActionEvent relationEvent) { + if (relationEvent.getActionType() == ActionType.RELATION_ADD_OR_UPDATE) { + tbClusterService.onRelationUpdated(relationEvent.getTenantId(), relationEvent.getRelation(), TbQueueCallback.EMPTY); + } else if (relationEvent.getActionType() == ActionType.RELATION_DELETED) { + tbClusterService.onRelationDeleted(relationEvent.getTenantId(), relationEvent.getRelation(), TbQueueCallback.EMPTY); + } + } + private void onTenantUpdate(Tenant tenant, ComponentLifecycleEvent lifecycleEvent) { tbClusterService.onTenantChange(tenant, null); tbClusterService.broadcastEntityStateChangeEvent(tenant.getId(), tenant.getId(), lifecycleEvent); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index f1494658a8..22faeaf61b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -56,6 +56,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; @@ -369,6 +370,17 @@ public class DefaultTbClusterService implements TbClusterService { broadcast(new ComponentLifecycleMsg(tenantId, entityId, state)); } + @Override + public void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, EntityId profileId, ComponentLifecycleEvent state) { + log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); + broadcast(ComponentLifecycleMsg.builder() + .tenantId(tenantId) + .entityId(entityId) + .profileId(profileId) + .event(state) + .build()); + } + @Override public void onDeviceProfileChange(DeviceProfile deviceProfile, DeviceProfile oldDeviceProfile, TbQueueCallback callback) { boolean isFirmwareChanged = false; @@ -420,13 +432,13 @@ public class DefaultTbClusterService implements TbClusterService { gatewayNotificationsService.onDeviceDeleted(device); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); - broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); + broadcastEntityStateChangeEvent(tenantId, deviceId, device.getDeviceProfileId(), ComponentLifecycleEvent.DELETED); } @Override public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { AssetId assetId = asset.getId(); - broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); + broadcastEntityStateChangeEvent(tenantId, assetId, asset.getAssetProfileId(), ComponentLifecycleEvent.DELETED); } @Override @@ -723,6 +735,30 @@ public class DefaultTbClusterService implements TbClusterService { broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), ComponentLifecycleEvent.DELETED); } + @Override + public void onRelationUpdated(TenantId tenantId, EntityRelation entityRelation, TbQueueCallback callback) { + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(tenantId) + .entityId(entityRelation.getFrom()) + .relationChanged(true) + .event(ComponentLifecycleEvent.UPDATED) + .info(JacksonUtil.valueToTree(entityRelation)) + .build(); + broadcast(msg); + } + + @Override + public void onRelationDeleted(TenantId tenantId, EntityRelation entityRelation, TbQueueCallback callback) { + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(tenantId) + .entityId(entityRelation.getFrom()) + .relationChanged(true) + .event(ComponentLifecycleEvent.DELETED) + .info(JacksonUtil.valueToTree(entityRelation)) + .build(); + broadcast(msg); + } + @Override public void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId originatorEdgeId) { if (!edgesEnabled) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 7382ef1c4d..761b69024f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -184,24 +184,34 @@ public abstract class AbstractConsumerService new ScriptCalculatedFieldState(entityId); case GEOFENCING -> new GeofencingCalculatedFieldState(entityId); case ALARM -> new AlarmCalculatedFieldState(entityId); + case LATEST_VALUES_AGGREGATION -> new LatestValuesAggregationCalculatedFieldState(entityId); }; } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 4319f72448..d4ee7bb2e3 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.util.KvProtoUtil; import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos.AggSingleArgumentEntryProto; import org.thingsboard.server.gen.transport.TransportProtos.AlarmRuleStateProto; import org.thingsboard.server.gen.transport.TransportProtos.AlarmStateProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; @@ -45,12 +46,16 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingZoneState; +import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.TreeMap; @@ -92,7 +97,10 @@ public class CalculatedFieldUtils { .setType(state.getType().name()); state.getArguments().forEach((argName, argEntry) -> { - if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + if (argEntry instanceof AggArgumentEntry aggArgumentEntry) { + aggArgumentEntry.getAggInputs() + .forEach((entityId, entry) -> builder.addAggArguments(toAggSingleArgumentProto(argName, entityId, entry))); + } else if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { builder.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { builder.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry)); @@ -130,6 +138,17 @@ public class CalculatedFieldUtils { return ruleState; } + public static AggSingleArgumentEntryProto toAggSingleArgumentProto(String argName, EntityId entityId, ArgumentEntry argumentEntry) { + AggSingleArgumentEntryProto.Builder builder = AggSingleArgumentEntryProto.newBuilder() + .setEntityId(ProtoUtils.toProto(entityId)); + + if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + builder.setValue(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); + } + + return builder.build(); + } + public static SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() .setArgName(argName); @@ -187,6 +206,7 @@ public class CalculatedFieldUtils { case SCRIPT -> new ScriptCalculatedFieldState(id.entityId()); case GEOFENCING -> new GeofencingCalculatedFieldState(id.entityId()); case ALARM -> new AlarmCalculatedFieldState(id.entityId()); + case LATEST_VALUES_AGGREGATION -> new LatestValuesAggregationCalculatedFieldState(id.entityId()); }; proto.getSingleValueArgumentsList().forEach(argProto -> @@ -212,11 +232,37 @@ public class CalculatedFieldUtils { alarmState.setClearRuleState(fromAlarmRuleStateProto(alarmStateProto.getClearRuleState(), alarmState)); } } + case LATEST_VALUES_AGGREGATION -> { + LatestValuesAggregationCalculatedFieldState aggState = (LatestValuesAggregationCalculatedFieldState) state; + Map> arguments = new HashMap<>(); + proto.getAggArgumentsList().forEach(argProto -> { + AggSingleArgumentEntry entry = fromAggSingleValueArgumentProto(argProto); + arguments.computeIfAbsent(argProto.getValue().getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); + }); + arguments.forEach((argName, entityInputs) -> { + aggState.getArguments().put(argName, new AggArgumentEntry(entityInputs, false)); + }); + } } return state; } + public static AggSingleArgumentEntry fromAggSingleValueArgumentProto(AggSingleArgumentEntryProto proto) { + if (!proto.hasValue()) { + return new AggSingleArgumentEntry(); + } + EntityId entityId = ProtoUtils.fromProto(proto.getEntityId()); + SingleValueArgumentProto singleValueArgument = proto.getValue(); + TsValueProto tsValueProto = singleValueArgument.getValue(); + return new AggSingleArgumentEntry( + entityId, + tsValueProto.getTs(), + (BasicKvEntry) KvProtoUtil.fromTsValueProto(singleValueArgument.getArgName(), tsValueProto), + singleValueArgument.getVersion() + ); + } + public static SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { if (!proto.hasValue()) { return new SingleValueArgumentEntry(); diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java new file mode 100644 index 0000000000..1a5fc2e3d3 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -0,0 +1,314 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cf; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; +import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration; +import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; +import org.thingsboard.server.common.data.device.data.DeviceData; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationPathLevel; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.thingsboard.server.cf.CalculatedFieldIntegrationTest.POLL_INTERVAL; + +@DaoSqlTest +public class LatestValuesAggregationCalculatedFieldTest extends AbstractControllerTest { + + private Tenant savedTenant; + + private DeviceProfile deviceProfile; + private Device device1; + private String accessToken1 = "1234567890111"; + private Device device2; + private String accessToken2 = "1234567890222"; + + private AssetProfile assetProfile; + private Asset asset; + + private CalculatedField calculatedField; + + private long deduplicationInterval = 10000; + + @Before + public void beforeTest() throws Exception { + loginSysAdmin(); + + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + savedTenant = saveTenant(tenant); + assertThat(savedTenant).isNotNull(); + + User tenantAdmin = new User(); + tenantAdmin.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin.setTenantId(savedTenant.getId()); + tenantAdmin.setEmail("tenant@thingsboard.org"); + tenantAdmin.setFirstName("John"); + tenantAdmin.setLastName("Doe"); + + createUserAndLogin(tenantAdmin, "testPassword"); + + deviceProfile = doPost("/api/deviceProfile", createDeviceProfile("Device Profile"), DeviceProfile.class); + device1 = createDevice("Device 1", deviceProfile.getId(), accessToken1); + device2 = createDevice("Device 2", deviceProfile.getId(), accessToken2); + + postTelemetry(device1.getId(), "{\"occupied\":true}"); + postTelemetry(device2.getId(), "{\"occupied\":false}"); + + assetProfile = doPost("/api/assetProfile", createAssetProfile("Asset Profile"), AssetProfile.class); + asset = createAsset("Asset", assetProfile.getId()); + + createEntityRelation(asset.getId(), device1.getId(), "Contains"); + createEntityRelation(asset.getId(), device2.getId(), "Contains"); + + calculatedField = createOccupancyCF(asset.getId(), List.of(deviceProfile.getId())); + + checkInitialCalculation(); + } + + @After + public void afterTest() throws Exception { + loginSysAdmin(); + + deleteTenant(savedTenant.getId()); + } + + @Test + public void testUpdateTelemetry_checkMetricsCalculation() throws Exception { + postTelemetry(device1.getId(), "{\"occupied\":false}"); + + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + }); + } + + @Test + public void testUpdateTelemetry_checkMetricsCalculationNotExecutedUntilDeduplicationInterval() throws Exception { + postTelemetry(device1.getId(), "{\"occupied\":false}"); + + await().alias("update telemetry -> no changes").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(this::checkInitialCalculationValues); + + postTelemetry(device2.getId(), "{\"occupied\":false}"); + + await().alias("create CF and perform initial calculation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + }); + } + + @Test + public void testChangeProfile_checkMetricsCalculation() throws Exception { + DeviceProfile deviceProfile2 = doPost("/api/deviceProfile", createDeviceProfile("Device Profile 2"), DeviceProfile.class); + device1.setDeviceProfileId(deviceProfile2.getId()); + device1 = doPost("/api/device?accessToken=" + accessToken1, device1, Device.class); + + postTelemetry(device1.getId(), "{\"occupied\":false}"); + + await().alias("change profile and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); + }); + } + + @Test + public void testAddEntityToProfile_checkMetricsCalculation() throws Exception { + Device device3 = createDevice("Device 3", deviceProfile.getId(), "1234567890333"); + + postTelemetry(device3.getId(), "{\"occupied\":true}"); + + await().alias("add entity to profile and no calculation (there is no relation between device and asset)").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(this::checkInitialCalculationValues); + + createEntityRelation(asset.getId(), device3.getId(), "Contains"); + + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("3"); + }); + } + + @Test + public void testDeleteRelation_checkMetricsCalculation() throws Exception { + deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); + + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); + }); + } + + private void checkInitialCalculation() { + await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(this::checkInitialCalculationValues); + } + + private void checkInitialCalculationValues() throws Exception { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + } + + private CalculatedField createOccupancyCF(EntityId entityId, List profiles) { + Map aggMetrics = new HashMap<>(); + + AggMetric freeSpaces = new AggMetric(); + freeSpaces.setFunction(AggFunction.COUNT); + freeSpaces.setFilter("return oc == false"); + freeSpaces.setInput(new AggKeyInput("oc")); + aggMetrics.put("freeSpaces", freeSpaces); + + AggMetric occupiedSpaces = new AggMetric(); + occupiedSpaces.setFunction(AggFunction.COUNT); + occupiedSpaces.setFilter("return oc == true"); + occupiedSpaces.setInput(new AggKeyInput("oc")); + aggMetrics.put("occupiedSpaces", occupiedSpaces); + + AggMetric totalSpaces = new AggMetric(); + totalSpaces.setFunction(AggFunction.COUNT); + totalSpaces.setInput(new AggFunctionInput("return 1;")); + aggMetrics.put("totalSpaces", totalSpaces); + + Output output = new Output(); + output.setType(OutputType.TIME_SERIES); + + return createAggCf("Occupied spaces", entityId, + buildSource(EntitySearchDirection.FROM, "Contains", profiles), + Map.of("oc", new ReferencedEntityKey("occupied", ArgumentType.TS_LATEST, null)), + aggMetrics, + output); + } + + private AggSource buildSource(EntitySearchDirection direction, String relationType, List profiles) { + AggSource source = new AggSource(); + source.setRelation(new RelationPathLevel(direction, relationType)); + source.setEntityProfiles(profiles); + return source; + } + + private CalculatedField createAggCf(String name, + EntityId entityId, + AggSource aggSource, + Map inputs, + Map metrics, + Output output) { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setName(name); + calculatedField.setEntityId(entityId); + calculatedField.setType(CalculatedFieldType.LATEST_VALUES_AGGREGATION); + + LatestValuesAggregationCalculatedFieldConfiguration configuration = new LatestValuesAggregationCalculatedFieldConfiguration(); + configuration.setSource(aggSource); + configuration.setInputs(inputs); + configuration.setDeduplicationIntervalMillis(deduplicationInterval); + configuration.setMetrics(metrics); + configuration.setOutput(output); + + calculatedField.setConfiguration(configuration); + calculatedField.setDebugSettings(DebugSettings.all()); + return saveCalculatedField(calculatedField); + } + + private Device createDevice(String name, DeviceProfileId deviceProfileId, String accessToken) { + Device device = new Device(); + device.setName(name); + device.setDeviceProfileId(deviceProfileId); + DeviceData deviceData = new DeviceData(); + deviceData.setTransportConfiguration(new DefaultDeviceTransportConfiguration()); + deviceData.setConfiguration(new DefaultDeviceConfiguration()); + device.setDeviceData(deviceData); + return doPost("/api/device?accessToken=" + accessToken, device, Device.class); + } + + private Asset createAsset(String name, AssetProfileId assetProfileId) { + Asset asset = new Asset(); + asset.setName(name); + asset.setAssetProfileId(assetProfileId); + return doPost("/api/asset", asset, Asset.class); + } + + private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { + return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 2050e1075c..6c6b9c44ec 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -1063,6 +1063,16 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { doPost("/api/relation", relation); } + protected void deleteEntityRelation(EntityRelation entityRelation) throws Exception { + String url = String.format("/api/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", + entityRelation.getFrom().getId(), + entityRelation.getFrom().getEntityType(), + entityRelation.getType(), + entityRelation.getTo().getId(), + entityRelation.getTo().getEntityType()); + doDelete(url); + } + protected List findRelationsByTo(EntityId entityId) throws Exception { String url = String.format("/api/relations?toId=%s&toType=%s", entityId.getId(), entityId.getEntityType().name()); MvcResult mvcResult = doGet(url).andReturn(); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 701c22587b..6c8acbfb83 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; @@ -30,6 +31,7 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; @@ -87,6 +89,8 @@ public interface TbClusterService extends TbQueueClusterService { void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); + void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, EntityId profileId, ComponentLifecycleEvent state); + void broadcast(ComponentLifecycleMsg componentLifecycleMsg); void onDeviceProfileChange(DeviceProfile deviceProfile, DeviceProfile oldDeviceProfile, TbQueueCallback callback); @@ -137,4 +141,8 @@ public interface TbClusterService extends TbQueueClusterService { void onCalculatedFieldDeleted(CalculatedField calculatedField, TbQueueCallback callback); + void onRelationUpdated(TenantId tenantId, EntityRelation entityRelation, TbQueueCallback callback); + + void onRelationDeleted(TenantId tenantId, EntityRelation entityRelation, TbQueueCallback callback); + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java index a0bc9a72e6..5b3290c110 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java @@ -86,6 +86,14 @@ public interface RelationService { ListenableFuture> findByRelationPathQueryAsync(TenantId tenantId, EntityRelationPathQuery relationPathQuery); + List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId profileId); + + EntityRelation findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId profileId); + + void evictRelationsByProfile(TenantId tenantId, EntityId profileId); + + void evictRelationsByEntityAndProfile(TenantId tenantId, EntityId entityId, EntityId profileId); + // TODO: This method may be useful for some validations in the future // ListenableFuture checkRecursiveRelation(EntityId from, EntityId to); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java index 7052ab70e9..397d31650a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -24,7 +24,8 @@ public enum CalculatedFieldType { SIMPLE, SCRIPT, GEOFENCING, - ALARM; + ALARM, + LATEST_VALUES_AGGREGATION; public static final Set all = Collections.unmodifiableSet(EnumSet.allOf(CalculatedFieldType.class)); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index d3622a2dcf..4a4e3380f5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -40,7 +41,8 @@ import java.util.stream.Collectors; @Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), @Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT"), @Type(value = GeofencingCalculatedFieldConfiguration.class, name = "GEOFENCING"), - @Type(value = AlarmCalculatedFieldConfiguration.class, name = "ALARM") + @Type(value = AlarmCalculatedFieldConfiguration.class, name = "ALARM"), + @Type(value = LatestValuesAggregationCalculatedFieldConfiguration.class, name = "LATEST_VALUES_AGGREGATION") }) @JsonIgnoreProperties(ignoreUnknown = true) public interface CalculatedFieldConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunction.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunction.java new file mode 100644 index 0000000000..cd0e3f66d4 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunction.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +public enum AggFunction { + MIN, MAX, SUM, AVG, COUNT, COUNT_UNIQUE +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunctionInput.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunctionInput.java new file mode 100644 index 0000000000..f6df80952e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggFunctionInput.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class AggFunctionInput implements AggInput { + + private String function; + + @Override + public String getType() { + return "function"; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java new file mode 100644 index 0000000000..fac988d5d9 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = AggKeyInput.class, name = "key"), + @JsonSubTypes.Type(value = AggFunctionInput.class, name = "function") +}) +@JsonIgnoreProperties(ignoreUnknown = true) +public interface AggInput { + + String getType(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggKeyInput.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggKeyInput.java new file mode 100644 index 0000000000..1a00a18a9d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggKeyInput.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class AggKeyInput implements AggInput { + + private String key; + + @Override + public String getType() { + return "key"; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggMetric.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggMetric.java new file mode 100644 index 0000000000..ebd612b1e0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggMetric.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Data; + +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class AggMetric { + + private AggFunction function; + private String filter; + private AggInput input; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java new file mode 100644 index 0000000000..84f6f92eb3 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.relation.RelationPathLevel; + +import java.util.List; + +@Data +public class AggSource { + + private RelationPathLevel relation; + private List entityProfiles; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java new file mode 100644 index 0000000000..393697877e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java @@ -0,0 +1,104 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; +import java.util.function.Predicate; + +@Data +@Builder +public class CfAggTrigger { + + private List entityProfiles; + private List inputs; + + public boolean matches(EntityId profileId, Predicate cfAggTrigger) { + if (matchesProfile(profileId)) { + return cfAggTrigger.test(this); + } + return false; + } + + public boolean matchesProfile(EntityId profileId) { + return entityProfiles.contains(profileId); + } + + public boolean matchesTimeSeries(List telemetry) { + if (telemetry == null || telemetry.isEmpty()) { + return false; + } + for (TsKvEntry tsKvEntry : telemetry) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKvEntry.getKey(), ArgumentType.TS_LATEST, null); + if (inputs.contains(latestKey)) { + return true; + } + } + return false; + } + + public boolean matchesAttributes(List attributes, AttributeScope scope) { + if (attributes == null || attributes.isEmpty()) { + return false; + } + for (AttributeKvEntry attributeKvEntry : attributes) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(attributeKvEntry.getKey(), ArgumentType.ATTRIBUTE, scope); + if (inputs.contains(latestKey)) { + return true; + } + } + return false; + } + + public boolean matchesTimeSeriesKeys(List telemetry) { + if (telemetry == null || telemetry.isEmpty()) { + return false; + } + + for (String key : telemetry) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); + if (inputs.contains(latestKey)) { + return true; + } + } + + return false; + } + + public boolean matchesAttributesKeys(List attributes, AttributeScope scope) { + if (attributes == null || attributes.isEmpty()) { + return false; + } + + for (String key : attributes) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); + if (inputs.contains(latestKey)) { + return true; + } + } + + return false; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..e3db50fda0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration.aggregation; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; + +import java.util.List; +import java.util.Map; + +@Data +public class LatestValuesAggregationCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + private AggSource source; + private Map inputs; + private long deduplicationIntervalMillis; + private Map metrics; + private Output output; + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.LATEST_VALUES_AGGREGATION; + } + + @Override + public void validate() { + } + + public CfAggTrigger buildTrigger() { + return CfAggTrigger.builder() + .inputs(List.copyOf(inputs.values())) + .entityProfiles(source.getEntityProfiles()) + .build(); + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index c13b0200c7..f82de35819 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -152,6 +152,8 @@ public enum MsgType { CF_ENTITY_INIT_CF_MSG, CF_ENTITY_DELETE_MSG, + CF_RELATED_ENTITY_MSG, + CF_ARGUMENT_RESET_MSG, // Sent to reset argument; CF_REEVALUATE_MSG; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index 23b9fe08e3..38df529c96 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -47,14 +47,15 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { private final EntityId oldProfileId; private final EntityId profileId; private final boolean ownerChanged; + private final boolean relationChanged; private final JsonNode info; public ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event) { - this(tenantId, entityId, event, null, null, null, null, false, null); + this(tenantId, entityId, event, null, null, null, null, false, false, null); } @Builder - private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId, boolean ownerChanged, JsonNode info) { + private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId, boolean ownerChanged, boolean relationChanged, JsonNode info) { this.tenantId = tenantId; this.entityId = entityId; this.event = event; @@ -63,6 +64,7 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { this.oldProfileId = oldProfileId; this.profileId = profileId; this.ownerChanged = ownerChanged; + this.relationChanged = relationChanged; this.info = info; } diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 26a64c7f8a..83fb158efd 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -130,6 +130,7 @@ public class ProtoUtils { builder.setOldProfileIdLSB(msg.getOldProfileId().getId().getLeastSignificantBits()); } builder.setOwnerChanged(msg.isOwnerChanged()); + builder.setRelationChanged(msg.isRelationChanged()); if (msg.getName() != null) { builder.setName(msg.getName()); } @@ -167,6 +168,7 @@ public class ProtoUtils { builder.oldProfileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB()))); } builder.ownerChanged(proto.getOwnerChanged()); + builder.relationChanged(proto.getRelationChanged()); if (proto.hasInfo()) { builder.info(JacksonUtil.toJsonNode(proto.getInfo())); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 4d99608a6d..764f4c3ec9 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -915,6 +915,11 @@ message GeofencingArgumentProto { repeated GeofencingZoneProto zones = 2; } +message AggSingleArgumentEntryProto { + EntityIdProto entityId = 1; + SingleValueArgumentProto value = 2; +} + message CalculatedFieldStateProto { CalculatedFieldEntityCtxIdProto id = 1; string type = 2; @@ -922,6 +927,7 @@ message CalculatedFieldStateProto { repeated TsRollingArgumentProto rollingValueArguments = 4; repeated GeofencingArgumentProto geofencingArguments = 5; AlarmStateProto alarmState = 6; + repeated AggSingleArgumentEntryProto aggArguments = 7; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. @@ -1292,6 +1298,7 @@ message ComponentLifecycleMsgProto { int64 profileIdLSB = 12; optional string info = 13; bool ownerChanged = 100; + bool relationChanged = 15; } message EdgeEventMsgProto { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java index 73a2183564..bf2123a26b 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java @@ -28,6 +28,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = TbelCfSingleValueArg.class, name = "SINGLE_VALUE"), @JsonSubTypes.Type(value = TbelCfTsRollingArg.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = TbelCfTsGeofencingArg.class, name = "GEOFENCING_CF_ARGUMENT_VALUE"), + @JsonSubTypes.Type(value = TbelCfLatestValuesAggregation.class, name = "LATEST_VALUES_AGGREGATION") }) public interface TbelCfArg extends TbelCfObject { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java new file mode 100644 index 0000000000..1b5fa394d2 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class TbelCfLatestValuesAggregation implements TbelCfArg { + + private final Object value; + + @JsonCreator + public TbelCfLatestValuesAggregation( + @JsonProperty("value") Object value + ) { + this.value = value; + } + + + @Override + public String getType() { + return "LATEST_VALUES_AGGREGATION"; + } + + @Override + public long memorySize() { + return 32; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 18d1806fe4..df79ca145a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -514,6 +514,40 @@ public class BaseRelationService implements RelationService { return executor.submit(() -> relationDao.findByRelationPathQuery(tenantId, relationPathQuery)); } + @Override + public List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId profileId) { + RelationCacheKey cacheKey = RelationCacheKey.builder().from(from).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.FROM).entityProfile(profileId).build(); + return cache.getAndPutInTransaction(cacheKey, + () -> relationDao.findByFromAndTypeAndProfile(tenantId, from, relationType, RelationTypeGroup.COMMON, profileId), + RelationCacheValue::getRelations, + relations -> RelationCacheValue.builder().relations(relations).build(), false); + } + + @Override + public EntityRelation findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId profileId) { + RelationCacheKey cacheKey = RelationCacheKey.builder().to(to).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.TO).entityProfile(profileId).build(); + return cache.getAndPutInTransaction(cacheKey, + () -> relationDao.findByToAndTypeAndProfile(tenantId, to, relationType, RelationTypeGroup.COMMON, profileId), + RelationCacheValue::getRelation, + relation -> RelationCacheValue.builder().relation(relation).build(), false); + } + + @Override + public void evictRelationsByProfile(TenantId tenantId, EntityId profileId) { + RelationCacheKey key = RelationCacheKey.builder().entityProfile(profileId).build(); + cache.evict(List.of(key)); + log.debug("Processed evict relations by key: {}", key); + } + + @Override + public void evictRelationsByEntityAndProfile(TenantId tenantId, EntityId entityId, EntityId profileId) { + List keys = new ArrayList<>(2); + keys.add(RelationCacheKey.builder().from(entityId).entityProfile(profileId).build()); + keys.add(RelationCacheKey.builder().to(entityId).entityProfile(profileId).build()); + cache.evict(keys); + log.debug("Processed evict relations by keys: {}", keys); + } + private void validate(EntityRelationPathQuery relationPathQuery) { validateId((UUIDBased) relationPathQuery.rootEntityId(), id -> "Invalid root entity id: " + id); List levels = relationPathQuery.levels(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java index d6f0525c9d..344af0a6f3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java @@ -40,9 +40,14 @@ public class RelationCacheKey implements Serializable { private final String type; private final RelationTypeGroup typeGroup; private final EntitySearchDirection direction; + private final EntityId entityProfile; public RelationCacheKey(EntityId from, EntityId to, String type, RelationTypeGroup typeGroup) { - this(from, to, type, typeGroup, null); + this(from, to, type, typeGroup, null, null); + } + + public RelationCacheKey(EntityId from, EntityId to, String type, RelationTypeGroup typeGroup, EntitySearchDirection direction) { + this(from, to, type, typeGroup, direction, null); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index ad53164ad7..f529396965 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -36,8 +36,12 @@ public interface RelationDao { List findAllByFromAndType(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup); + List findByFromAndTypeAndProfile(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup, EntityId profileId); + List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup); + EntityRelation findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId); + List findAllByTo(TenantId tenantId, EntityId to); List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index b2871313ed..c859b71580 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -103,6 +103,11 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } + @Override + public List findByFromAndTypeAndProfile(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup, EntityId profileId) { + return DaoUtil.convertDataList(relationRepository.findByFromAndProfile(from.getId(), from.getEntityType().name(), typeGroup.name(), relationType, profileId.getId())); + } + @Override public List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup) { return DaoUtil.convertDataList( @@ -112,6 +117,11 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } + @Override + public EntityRelation findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId) { + return DaoUtil.getData(relationRepository.findByToAndProfile(to.getId(), to.getEntityType().name(), typeGroup.name(), relationType, profileId.getId())); + } + @Override public List findAllByTo(TenantId tenantId, EntityId to) { return DaoUtil.convertDataList( diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index b4e8a21372..9294236526 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.dao.sql.relation; -import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; @@ -28,6 +27,7 @@ import org.thingsboard.server.dao.model.sql.RelationCompositeKey; import org.thingsboard.server.dao.model.sql.RelationEntity; import java.util.List; +import java.util.Optional; import java.util.UUID; public interface RelationRepository @@ -96,4 +96,41 @@ public interface RelationRepository @Param("toId") UUID toId, @Param("toType") String toType, @Param("batchSize") int batchSize); + + @Query(value = """ + SELECT r.from_id, r.from_type, r.relation_type_group, r.relation_type, r.to_id, r.to_type, r.additional_info, r.version + FROM relation r + LEFT JOIN device d ON r.to_id = d.id AND r.to_type = 'DEVICE' + LEFT JOIN asset a ON r.to_id = a.id AND r.to_type = 'ASSET' + WHERE r.from_id = :fromId + AND r.from_type = :fromType + AND r.relation_type = :relationType + AND r.relation_type_group = :relationTypeGroup + AND ((d.device_profile_id = :profileId) OR (a.asset_profile_id = :profileId)) + AND (d.id IS NOT NULL OR a.id IS NOT NULL) + """, nativeQuery = true) + List findByFromAndProfile(@Param("fromId") UUID fromId, + @Param("fromType") String fromType, + @Param("relationTypeGroup") String relationTypeGroup, + @Param("relationType") String relationType, + @Param("profileId") UUID profileId); + + @Query(value = """ + SELECT r.from_id, r.from_type, r.relation_type_group, r.relation_type, r.to_id, r.to_type, r.additional_info, r.version + FROM relation r + LEFT JOIN device d ON r.from_id = d.id AND r.from_type = 'DEVICE' + LEFT JOIN asset a ON r.from_id = a.id AND r.from_type = 'ASSET' + WHERE r.to_id = :toId + AND r.to_type = :toType + AND r.relation_type = :relationType + AND r.relation_type_group = :relationTypeGroup + AND ((d.device_profile_id = :profileId) OR (a.asset_profile_id = :profileId)) + AND (d.id IS NOT NULL OR a.id IS NOT NULL) + LIMIT 1 + """, nativeQuery = true) + Optional findByToAndProfile(@Param("toId") UUID toId, + @Param("toType") String toType, + @Param("relationTypeGroup") String relationTypeGroup, + @Param("relationType") String relationType, + @Param("profileId") UUID profileId); } From 74b101f4b644108bd929597759e70568bc7d00da Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Fri, 10 Oct 2025 17:13:40 +0300 Subject: [PATCH 26/88] lwm2m: bootstrap new: update tests --- .../lwm2m/client/LwM2MTestClient.java | 55 ++++++++++++------- ...LwM2MIntegrationBS3SectionTriggerTest.java | 29 ++++++++++ ...nBSLwm2mOnlyNoneTriggerOneSectionTest.java | 37 +++++++++++++ .../NoSecLwM2MIntegrationBSNoTriggerTest.java | 1 - ...ntegrationBSOnlyTriggerOneSectionTest.java | 29 ++++++++++ .../NoSecLwM2MIntegrationBSTriggerTest.java | 32 +---------- ...LwM2MBootstrapConfigStoreTaskProvider.java | 13 +++-- 7 files changed, 140 insertions(+), 56 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBS3SectionTriggerTest.java create mode 100644 application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSLwm2mOnlyNoneTriggerOneSectionTest.java create mode 100644 application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSOnlyTriggerOneSectionTest.java diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java index 7c5e2973ac..4d6265c393 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java @@ -68,6 +68,7 @@ import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandle import org.thingsboard.server.transport.lwm2m.utils.LwM2mValueConverterImpl; import java.io.IOException; +import java.io.InputStream; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.HashMap; @@ -77,6 +78,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.eclipse.californium.scandium.config.DtlsConfig.DTLS_CONNECTION_ID_LENGTH; import static org.eclipse.californium.scandium.config.DtlsConfig.DTLS_RECOMMENDED_CIPHER_SUITES_ONLY; @@ -146,25 +148,9 @@ public class LwM2MTestClient { this.defaultLwM2mUplinkMsgHandlerTest = defaultLwM2mUplinkMsgHandler; this.clientContext = clientContext; - List models = ObjectLoader.loadAllDefault(); - for (String resourceName : lwm2mClientResources) { - models.addAll(ObjectLoader.loadDdfFile(LwM2MTestClient.class.getClassLoader().getResourceAsStream("lwm2m/" + resourceName), resourceName)); - } - if (this.modelResources != null) { - List modelsRes = new ArrayList<>(); - for (String resourceName : this.modelResources) { - modelsRes.addAll(ObjectLoader.loadDdfFile(LwM2MTestClient.class.getClassLoader().getResourceAsStream("lwm2m/" + resourceName), resourceName)); - } - Set idsToRemove = new HashSet<>(); - for (ObjectModel model : modelsRes) { - idsToRemove.add(model.id); - } - models.removeIf(model -> idsToRemove.contains(model.id)); - models.addAll(modelsRes); - } + ObjectsInitializer initializer = createFreshInitializer(); + - LwM2mModel model = new StaticModel(models); - ObjectsInitializer initializer = new ObjectsInitializer(model); if (securityBs != null && security != null) { // SECURITIES securityBs.setId(0); @@ -179,9 +165,9 @@ public class LwM2MTestClient { initializer.setInstancesForObject(SERVER, instances); } else if (securityBs != null) { // SECURITY - securityBs.setId(0); initializer.setClassForObject(SERVER, Server.class); initializer.setInstancesForObject(SECURITY, securityBs); + log.warn("Security section: securityBsId [{}] ", securityBs.getId()); } else { // SECURITY security.setId(0); @@ -478,4 +464,35 @@ public class LwM2MTestClient { LwM2mClient lwM2MClient = this.clientContext.getClientByEndpoint(endpoint); Mockito.doAnswer(invocationOnMock -> null).when(defaultLwM2mUplinkMsgHandlerTest).initAttributes(lwM2MClient, true); } + + private ObjectsInitializer createFreshInitializer() { + List models = new ArrayList<>(ObjectLoader.loadAllDefault()); + for (String resourceName : lwm2mClientResources) { + try (InputStream in = LwM2MTestClient.class.getClassLoader() + .getResourceAsStream("lwm2m/" + resourceName)) { + models.addAll(ObjectLoader.loadDdfFile(in, resourceName)); + } catch (IOException | InvalidDDFFileException e) { + log.warn("Failed to load resource {}", resourceName, e); + } + } + if (this.modelResources != null) { + List modelsRes = new ArrayList<>(); + for (String resourceName : this.modelResources) { + try (InputStream in = LwM2MTestClient.class.getClassLoader() + .getResourceAsStream("lwm2m/" + resourceName)) { + modelsRes.addAll(ObjectLoader.loadDdfFile(in, resourceName)); + } catch (IOException | InvalidDDFFileException e) { + log.warn("Failed to load resource {}", resourceName, e); + } + } + Set idsToRemove = modelsRes.stream() + .map(m -> m.id) + .collect(Collectors.toSet()); + models.removeIf(m -> idsToRemove.contains(m.id)); + models.addAll(modelsRes); + } + LwM2mModel model = new StaticModel(models); + return new ObjectsInitializer(model); + } } + diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBS3SectionTriggerTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBS3SectionTriggerTest.java new file mode 100644 index 0000000000..af8284484d --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBS3SectionTriggerTest.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.security.sql; + +import org.junit.Test; +import org.thingsboard.server.transport.lwm2m.security.AbstractSecurityLwM2MIntegrationTest; +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.LWM2M_ONLY; +public class NoSecLwM2MIntegrationBS3SectionTriggerTest extends AbstractSecurityLwM2MIntegrationTest { + + @Test + public void testWithNoSecConnectLwm2mSuccessBootstrapRequestTrigger_3_ConnectBsSuccess_UpdateLwm2mSection_3_AndLm2m_1_ConnectLwm2mSuccess() throws Exception { + String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "Trigger_3" + LWM2M_ONLY.name(); + String awaitAlias = "await on client state (NoSecBS Trigger Lwm2m section)"; + basicTestConnectionBootstrapRequestTriggerBefore(clientEndpoint, awaitAlias, LWM2M_ONLY, 3); + } +} diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSLwm2mOnlyNoneTriggerOneSectionTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSLwm2mOnlyNoneTriggerOneSectionTest.java new file mode 100644 index 0000000000..4fceba60f3 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSLwm2mOnlyNoneTriggerOneSectionTest.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.security.sql; + +import org.junit.Test; +import org.thingsboard.server.transport.lwm2m.security.AbstractSecurityLwM2MIntegrationTest; +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.LWM2M_ONLY; +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.NONE; +public class NoSecLwM2MIntegrationBSLwm2mOnlyNoneTriggerOneSectionTest extends AbstractSecurityLwM2MIntegrationTest { + + @Test + public void testWithNoSecConnectLwm2mSuccessBootstrapRequestTriggerConnectBsSuccess_UpdateLwm2mSectionAndLm2m_ConnectLwm2mSuccess() throws Exception { + String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "Trigger" + LWM2M_ONLY.name(); + String awaitAlias = "await on client state (NoSecBS Trigger Lwm2m section)"; + basicTestConnectionBootstrapRequestTriggerBefore(clientEndpoint, awaitAlias, LWM2M_ONLY, 1); + } + + @Test + public void testWithNoSecConnectLwm2mSuccessBootstrapRequestTriggerConnectBsSuccess_UpdateNoneSectionAndLm2m_ConnectLwm2mSuccess() throws Exception { + String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "Trigger" + NONE.name(); + String awaitAlias = "await on client state (NoSecBS Trigger None section)"; + basicTestConnectionBootstrapRequestTriggerBefore(clientEndpoint, awaitAlias, NONE, 1); + } +} diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSNoTriggerTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSNoTriggerTest.java index 2622cac18c..edfe805cf8 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSNoTriggerTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSNoTriggerTest.java @@ -17,7 +17,6 @@ package org.thingsboard.server.transport.lwm2m.security.sql; import org.junit.Test; import org.thingsboard.server.transport.lwm2m.security.AbstractSecurityLwM2MIntegrationTest; - import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_REGISTRATION_SUCCESS; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.BOTH; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.LWM2M_ONLY; diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSOnlyTriggerOneSectionTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSOnlyTriggerOneSectionTest.java new file mode 100644 index 0000000000..5510cdf614 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSOnlyTriggerOneSectionTest.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.security.sql; + +import org.junit.Test; +import org.thingsboard.server.transport.lwm2m.security.AbstractSecurityLwM2MIntegrationTest; +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.BOOTSTRAP_ONLY; +public class NoSecLwM2MIntegrationBSOnlyTriggerOneSectionTest extends AbstractSecurityLwM2MIntegrationTest { + + @Test + public void testWithNoSecConnectLwm2mSuccessBootstrapRequestTriggerConnectBsSuccess_UpdateBootstrapSectionAndLm2m_ConnectLwm2mSuccess() throws Exception { + String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "Trigger" + BOOTSTRAP_ONLY.name(); + String awaitAlias = "await on client state (NoSecBS Trigger Bootstrap section)"; + basicTestConnectionBootstrapRequestTriggerBefore(clientEndpoint, awaitAlias, BOOTSTRAP_ONLY, 1); + } +} diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSTriggerTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSTriggerTest.java index af5571864f..b007d16bde 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSTriggerTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSTriggerTest.java @@ -17,45 +17,15 @@ package org.thingsboard.server.transport.lwm2m.security.sql; import org.junit.Test; import org.thingsboard.server.transport.lwm2m.security.AbstractSecurityLwM2MIntegrationTest; - -import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.BOOTSTRAP_ONLY; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.BOTH; -import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.LWM2M_ONLY; -import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.NONE; public class NoSecLwM2MIntegrationBSTriggerTest extends AbstractSecurityLwM2MIntegrationTest { - @Test - public void testWithNoSecConnectLwm2mSuccessBootstrapRequestTriggerConnectBsSuccess_UpdateBootstrapSectionAndLm2m_ConnectLwm2mSuccess() throws Exception { - String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "Trigger" + BOOTSTRAP_ONLY.name(); - String awaitAlias = "await on client state (NoSecBS Trigger Bootstrap section)"; - basicTestConnectionBootstrapRequestTriggerBefore(clientEndpoint, awaitAlias, BOOTSTRAP_ONLY, 1); - } - @Test public void testWithNoSecConnectLwm2mSuccessBootstrapRequestTriggerConnectBsSuccess_UpdateTwoSectionAndLm2m_ConnectLwm2mSuccess() throws Exception { String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "Trigger" + BOTH.name(); String awaitAlias = "await on client state (NoSecBS Trigger Two section)"; basicTestConnectionBootstrapRequestTriggerBefore(clientEndpoint, awaitAlias, BOTH, 1); } - - @Test - public void testWithNoSecConnectLwm2mSuccessBootstrapRequestTriggerConnectBsSuccess_UpdateLwm2mSectionAndLm2m_ConnectLwm2mSuccess() throws Exception { - String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "Trigger" + LWM2M_ONLY.name(); - String awaitAlias = "await on client state (NoSecBS Trigger Lwm2m section)"; - basicTestConnectionBootstrapRequestTriggerBefore(clientEndpoint, awaitAlias, LWM2M_ONLY, 1); - } - @Test - public void testWithNoSecConnectLwm2mSuccessBootstrapRequestTriggerConnectBsSuccess_UpdateLwm2mSection_3_AndLm2m_1_ConnectLwm2mSuccess() throws Exception { - String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "Trigger" + LWM2M_ONLY.name(); - String awaitAlias = "await on client state (NoSecBS Trigger Lwm2m section)"; - basicTestConnectionBootstrapRequestTriggerBefore(clientEndpoint, awaitAlias, LWM2M_ONLY, 3); - } - - @Test - public void testWithNoSecConnectLwm2mSuccessBootstrapRequestTriggerConnectBsSuccess_UpdateNoneSectionAndLm2m_ConnectLwm2mSuccess() throws Exception { - String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "Trigger" + NONE.name(); - String awaitAlias = "await on client state (NoSecBS Trigger None section)"; - basicTestConnectionBootstrapRequestTriggerBefore(clientEndpoint, awaitAlias, NONE, 1); - } } + diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java index 3223179486..8e5c1cf4bf 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java @@ -98,14 +98,16 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask if (previousResponse != null) { if (previousResponse.get(0) instanceof BootstrapDiscoverResponse discoverResponse) { if (discoverResponse.isSuccess()) { - this.initAfterBootstrapDiscover(discoverResponse); + this.initAfterBootstrapDiscover(discoverResponse); /// Short Server Ids - in old config findInstancesIdOldByServerId(discoverResponse, session.getEndpoint()); + log.warn( + "Bootstrap server instance successfully found in Security Object (0) in response {}. Continuing bootstrap session. Session: {}", + discoverResponse, session); } else { - log.error( - "Unable to find bootstrap server instance in Security Object (0) in response {}: unable to continue bootstrap session with autoIdForSecurityObject mode. {}", + log.warn( + "Unable to find bootstrap server instance in Security Object (0) in response {}. Continuing bootstrap session with autoIdForSecurityObject mode, ignoring information from discoverResponse. Session: {}", discoverResponse, session); - return null; } } // create requests from config @@ -200,7 +202,8 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask public List> toRequests(BootstrapConfig bootstrapConfigNew, ContentFormat contentFormat, String endpoint) { - Integer bootstrapSecurityInstanceId = this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(BOOTSTRAP_DEFAULT_SHORT_ID_0); + Integer bootstrapSecurityInstanceId = this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(BOOTSTRAP_DEFAULT_SHORT_ID_0) == null ? + 0 : this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(BOOTSTRAP_DEFAULT_SHORT_ID_0); List> requests = new ArrayList<>(); Set pathsDelete = new HashSet<>(); ConcurrentHashMap> requestsWrite = new ConcurrentHashMap<>(); From 873dcabb4794b107d348ca23212968bdbe44bf6a Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 13 Oct 2025 16:05:55 +0300 Subject: [PATCH 27/88] fixed file attaching for Github AI models --- .../service/ai/AiChatModelServiceImpl.java | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java b/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java index d6252f57a6..78c0b77f40 100644 --- a/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java +++ b/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java @@ -16,6 +16,11 @@ package org.thingsboard.server.service.ai; import com.google.common.util.concurrent.FluentFuture; +import dev.langchain4j.data.message.ChatMessage; +import dev.langchain4j.data.message.Content; +import dev.langchain4j.data.message.TextContent; +import dev.langchain4j.data.message.UserMessage; +import dev.langchain4j.model.ModelProvider; import dev.langchain4j.model.chat.ChatModel; import dev.langchain4j.model.chat.request.ChatRequest; import dev.langchain4j.model.chat.response.ChatResponse; @@ -24,6 +29,9 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.ai.model.chat.AiChatModelConfig; import org.thingsboard.server.common.data.ai.model.chat.Langchain4jChatModelConfigurer; +import java.util.List; +import java.util.stream.Collectors; + @Service @RequiredArgsConstructor class AiChatModelServiceImpl implements AiChatModelService { @@ -34,7 +42,46 @@ class AiChatModelServiceImpl implements AiChatModelService { @Override public > FluentFuture sendChatRequestAsync(AiChatModelConfig chatModelConfig, ChatRequest chatRequest) { ChatModel langChainChatModel = chatModelConfig.configure(chatModelConfigurer); + if (langChainChatModel.provider() == ModelProvider.GITHUB_MODELS) { + chatRequest = prepareGithubChatRequest(chatRequest); + } return aiRequestsExecutor.sendChatRequestAsync(langChainChatModel, chatRequest); } + private ChatRequest prepareGithubChatRequest(ChatRequest chatRequest) { + List messages = chatRequest.messages().stream() + .map(this::escapeIfUserMessage) + .collect(Collectors.toList()); + + return ChatRequest.builder() + .messages(messages) + .responseFormat(chatRequest.responseFormat()) + .build(); + } + + private ChatMessage escapeIfUserMessage(ChatMessage message) { + if (message instanceof UserMessage userMessage) { + List newContents = userMessage.contents().stream() + .map(this::escapeContent) + .collect(Collectors.toList()); + + return UserMessage.from(newContents); + } + return message; + } + + private Content escapeContent(Content content) { + if (content instanceof TextContent txt) { + return new TextContent(escapeWhitespace(txt.text())); + } + return content; + } + + private String escapeWhitespace(String text) { + return text + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t"); + } + } From 6bb7dce150246d0fa4625d235f2234dd5952b854 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 13 Oct 2025 16:08:12 +0300 Subject: [PATCH 28/88] refactoting --- .../thingsboard/server/service/ai/AiChatModelServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java b/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java index 78c0b77f40..29d4a148fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java +++ b/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java @@ -72,12 +72,12 @@ class AiChatModelServiceImpl implements AiChatModelService { private Content escapeContent(Content content) { if (content instanceof TextContent txt) { - return new TextContent(escapeWhitespace(txt.text())); + return new TextContent(escapeControlChars(txt.text())); } return content; } - private String escapeWhitespace(String text) { + private String escapeControlChars(String text) { return text .replace("\n", "\\n") .replace("\r", "\\r") From d14466459d70cbef19caa7aa1ea6c127583f6c04 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 13 Oct 2025 16:35:57 +0300 Subject: [PATCH 29/88] refactoring --- .../service/ai/AiChatModelServiceImpl.java | 17 ++++++----------- .../server/common/data/StringUtils.java | 7 +++++++ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java b/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java index 29d4a148fd..c1829cbf84 100644 --- a/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java +++ b/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java @@ -32,6 +32,8 @@ import org.thingsboard.server.common.data.ai.model.chat.Langchain4jChatModelConf import java.util.List; import java.util.stream.Collectors; +import static org.thingsboard.server.common.data.StringUtils.escapeControlChars; + @Service @RequiredArgsConstructor class AiChatModelServiceImpl implements AiChatModelService { @@ -50,7 +52,7 @@ class AiChatModelServiceImpl implements AiChatModelService { private ChatRequest prepareGithubChatRequest(ChatRequest chatRequest) { List messages = chatRequest.messages().stream() - .map(this::escapeIfUserMessage) + .map(this::prepareUserMessage) .collect(Collectors.toList()); return ChatRequest.builder() @@ -59,10 +61,10 @@ class AiChatModelServiceImpl implements AiChatModelService { .build(); } - private ChatMessage escapeIfUserMessage(ChatMessage message) { + private ChatMessage prepareUserMessage(ChatMessage message) { if (message instanceof UserMessage userMessage) { List newContents = userMessage.contents().stream() - .map(this::escapeContent) + .map(this::prepareContent) .collect(Collectors.toList()); return UserMessage.from(newContents); @@ -70,18 +72,11 @@ class AiChatModelServiceImpl implements AiChatModelService { return message; } - private Content escapeContent(Content content) { + private Content prepareContent(Content content) { if (content instanceof TextContent txt) { return new TextContent(escapeControlChars(txt.text())); } return content; } - private String escapeControlChars(String text) { - return text - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace("\t", "\\t"); - } - } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java index cbc881d72f..2b8b631027 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java @@ -275,4 +275,11 @@ public class StringUtils { return result; } + public static String escapeControlChars(String text) { + return text + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t"); + } + } From ea8ab484dfe8b20dba68c22a7351e38471ce4832 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Mon, 13 Oct 2025 17:08:21 +0300 Subject: [PATCH 30/88] lwm2m: bootstrap new: update tests-2 --- .../lwm2m/AbstractLwM2MIntegrationTest.java | 17 ++- .../lwm2m/client/LwM2MTestClient.java | 42 +++++- .../lwm2m/Lwm2mServerIdentifier.java | 137 ++++++++++++++++++ ...LwM2MBootstrapConfigStoreTaskProvider.java | 22 +-- .../store/LwM2MConfigurationChecker.java | 8 +- .../lwm2m/utils/LwM2MTransportUtil.java | 3 - .../validator/DeviceProfileDataValidator.java | 13 +- .../DeviceProfileDataValidatorTest.java | 10 +- .../lwm2m-device-config-server.component.ts | 5 +- .../assets/locale/locale.constant-en_US.json | 2 +- 10 files changed, 221 insertions(+), 38 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/device/credentials/lwm2m/Lwm2mServerIdentifier.java diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java index 50051c08d5..8a62795863 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java @@ -33,10 +33,12 @@ import org.eclipse.leshan.server.registration.Registration; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.jupiter.api.TestInstance; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.http.HttpStatus; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.TestPropertySource; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; @@ -120,11 +122,13 @@ import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfil import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.LwM2MProfileBootstrapConfigType.NONE; import static org.thingsboard.server.transport.lwm2m.ota.AbstractOtaLwM2MIntegrationTest.CLIENT_LWM2M_SETTINGS_19; -@TestPropertySource(properties = { - "transport.lwm2m.enabled=true", -}) @Slf4j @DaoSqlTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) +@TestPropertySource(properties = { + "transport.lwm2m.enabled=true" +}) public abstract class AbstractLwM2MIntegrationTest extends AbstractTransportIntegrationTest { @SpyBean @@ -317,9 +321,16 @@ public abstract class AbstractLwM2MIntegrationTest extends AbstractTransportInte @After public void after() throws Exception { this.clientDestroy(true); + if (executor != null && !executor.isShutdown()) { executor.shutdownNow(); + if (!executor.awaitTermination(2, TimeUnit.SECONDS)) { + log.warn("⚠️ Executor did not terminate cleanly, forcing GC"); + } } + Thread.sleep(300); + System.gc(); + log.info("✅ Test teardown completed: {}", this.getClass().getSimpleName()); } private void init() throws Exception { diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java index 4d6265c393..c368dbfe54 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java @@ -69,6 +69,7 @@ import org.thingsboard.server.transport.lwm2m.utils.LwM2mValueConverterImpl; import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.Field; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.HashMap; @@ -153,29 +154,30 @@ public class LwM2MTestClient { if (securityBs != null && security != null) { // SECURITIES - securityBs.setId(0); - security.setId(1); + forceNullSecurityId(securityBs); + forceNullSecurityId(security); LwM2mInstanceEnabler[] instances = new LwM2mInstanceEnabler[]{securityBs, security}; initializer.setInstancesForObject(SECURITY, instances); + log.warn("Security BS section: securityBsId [{}] Security Lwm2m section: securityLwm2mId [{}] ", securityBs.getId(), security.getId()); // SERVER Server lwm2mServer = new Server(shortServerId, TimeUnit.MINUTES.toSeconds(60)); - lwm2mServer.setId(0); instances = new LwM2mInstanceEnabler[]{lwm2mServer}; - initializer.setInstancesForObject(SERVER, instances); } else if (securityBs != null) { // SECURITY - initializer.setClassForObject(SERVER, Server.class); +; forceNullSecurityId(securityBs); initializer.setInstancesForObject(SECURITY, securityBs); - log.warn("Security section: securityBsId [{}] ", securityBs.getId()); + // SERVER + initializer.setClassForObject(SERVER, Server.class); + log.warn("Security BS section: securityBsId [{}] ", securityBs.getId()); } else { // SECURITY - security.setId(0); + forceNullSecurityId(security); initializer.setInstancesForObject(SECURITY, security); // SERVER Server lwm2mServer = new Server(shortServerId, TimeUnit.MINUTES.toSeconds(60)); - lwm2mServer.setId(0); initializer.setInstancesForObject(SERVER, lwm2mServer); + log.warn("Security Lwm2m section: securityLwm2mId [{}] Server Lwm2m section: securityLwm2mId [{}] ", security.getId(), lwm2mServer.getId()); } initializer.setInstancesForObject(DEVICE, lwM2MDevice = new SimpleLwM2MDevice(executor, value3_0_9)); @@ -494,5 +496,29 @@ public class LwM2MTestClient { LwM2mModel model = new StaticModel(models); return new ObjectsInitializer(model); } + + private void forceNullSecurityId(Security securityBs) { + if (securityBs == null) { + return; + } + try { + Field field = securityBs.getClass().getDeclaredField("id"); + field.setAccessible(true); + field.set(securityBs, null); + log.info("[forceNullSecurityId] Set id=null for {}", securityBs); + } catch (NoSuchFieldException e) { + try { + // Якщо поле в батьківському класі (наприклад SecurityObjectInstance) + Field field = securityBs.getClass().getSuperclass().getDeclaredField("id"); + field.setAccessible(true); + field.set(securityBs, null); + log.info("[forceNullSecurityId] Set id=null for {} (via superclass)", securityBs); + } catch (Exception ex) { + log.error("[forceNullSecurityId] Field 'id' not found for {}", securityBs.getClass(), ex); + } + } catch (Exception e) { + log.error("[forceNullSecurityId] Failed to set id=null for {}", securityBs.getClass(), e); + } + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/credentials/lwm2m/Lwm2mServerIdentifier.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/credentials/lwm2m/Lwm2mServerIdentifier.java new file mode 100644 index 0000000000..a9f81ab655 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/credentials/lwm2m/Lwm2mServerIdentifier.java @@ -0,0 +1,137 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.device.credentials.lwm2m; + +/** + * Enum representing predefined LwM2M Short Server Identifiers. + *

+ * See OMA Lightweight M2M Specification for details about the server identifier space. + */ +public enum Lwm2mServerIdentifier { + + /** + * Bootstrap Short Server ID (0). + * Reserved for the Bootstrap Server — used exclusively during the bootstrap phase. + */ + BOOTSTRAP(0, "Bootstrap Short Server ID", true), + + /** + * Primary LwM2M Server Short Server ID (1). + * Upper boundary for valid LwM2M Server Identifiers (1–65534). + */ + PRIMARY_LWM2M_SERVER(1, "LwM2M Server Short Server ID", false), + + /** + * Maximum valid LwM2M Server ID (65534). + * Upper boundary for valid LwM2M Server Identifiers (1–65534). + */ + LWM2M_SERVER_MAX(65534, "LwM2M Server Short Server ID", false), + + /** + * Not used for identifying an LwM2M Server (65535). + * Reserved sentinel value representing "no server associated" or "invalid ID". + * MUST NOT be assigned to any LwM2M Server according to OMA-TS-LightweightM2M-Core, §6.2.1. + * OMA LwM2M Core / v1.2: Server / Short Server ID): «MAX_ID 65535 is a reserved value and MUST NOT be used for identifying an Object» + */ + NOT_USED_IDENTIFYING_LWM2M_SERVER(65535, "Reserved sentinel value (no active server)", false); + + private final int id; + private final String description; + private final boolean isBootstrap; + + Lwm2mServerIdentifier(int id, String description, boolean isBootstrap) { + this.id = id; + this.description = description; + this.isBootstrap = isBootstrap; + } + + /** + * @return the integer value of this Short Server ID. + */ + public int getId() { + return id; + } + + /** + * @return a human-readable description of this Server ID. + */ + public String getDescription() { + return description; + } + + /** + * @return true if this ID represents a Bootstrap Server. + */ + public boolean isBootstrap() { + return isBootstrap; + } + + /** + * Checks whether a given numeric ID belongs to the Bootstrap Server (0). + * OMA Spec (LwM2M v1.0 / v1.1): + * Short Server ID Resource (Resource ID: 0) + * The Short Server ID identifies a Server Object Instance. + * The value 0 is reserved for the Bootstrap Server. + * A value between 1 and 65534 identifies a LwM2M Server. + * The value 65535 MUST NOT be used. + * @param id Short Server ID value. + * @return true if id == 0. + */ + public static boolean isBootstrap(int id) { + return id == BOOTSTRAP.id; + } + + /** + * Checks whether a given ID represents a valid LwM2M Server (1–65534). + * + * @param id Short Server ID value. + * @return true if the ID belongs to a standard LwM2M Server. + */ + public static boolean isLwm2mServer(int id) { + return id >= PRIMARY_LWM2M_SERVER.id && id <= LWM2M_SERVER_MAX.id; + } + + /** + * Checks whether the provided ID is within the valid LwM2M range [0–65535]. + * + * @param id ID to check. + * @return true if valid, false otherwise. + */ + public static boolean isValid(int id) { + return id >= 0 && id <= 65535; + } + + /** + * Returns a {@link Lwm2mServerIdentifier} instance matching the given ID. + * + * @param id numeric ID. + * @return corresponding enum constant. + * @throws IllegalArgumentException if no constant matches the given ID. + */ + public static Lwm2mServerIdentifier fromId(int id) { + for (Lwm2mServerIdentifier s : values()) { + if (s.id == id) { + return s; + } + } + throw new IllegalArgumentException("Unknown Lwm2mServerIdentifier: " + id); + } + + @Override + public String toString() { + return name() + "(" + id + ") - " + description; + } +} diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java index 8e5c1cf4bf..33d1fb207c 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java @@ -48,9 +48,9 @@ import static org.eclipse.leshan.core.LwM2mId.ACCESS_CONTROL; import static org.eclipse.leshan.core.LwM2mId.SECURITY; import static org.eclipse.leshan.core.LwM2mId.SERVER; import static org.eclipse.leshan.server.bootstrap.BootstrapUtil.toWriteRequest; -import static org.thingsboard.server.transport.lwm2m.utils.LwM2MTransportUtil.BOOTSTRAP_DEFAULT_SHORT_ID_0; -import static org.thingsboard.server.transport.lwm2m.utils.LwM2MTransportUtil.LWM2M_DEFAULT_SHORT_ID_1; -import static org.thingsboard.server.transport.lwm2m.utils.LwM2MTransportUtil.LWM2M_DEFAULT_SHORT_ID_65534; +import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.BOOTSTRAP; +import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.LWM2M_SERVER_MAX; +import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.PRIMARY_LWM2M_SERVER; @Slf4j public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTaskProvider { @@ -147,7 +147,7 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask log.error("Invalid lwm2mSecurityInstance [{}] by short server id [{}]", path.getObjectInstanceId(), lwm2mShortServerId); } } else { - this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().putIfAbsent(BOOTSTRAP_DEFAULT_SHORT_ID_0, path.getObjectInstanceId()); + this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().putIfAbsent(BOOTSTRAP.getId(), path.getObjectInstanceId()); } } else if (path.getObjectId() == 1) { if (link.getAttributes().get("ssid") != null) { @@ -192,7 +192,7 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask * SECURITY = 0; InstanceId = 0 * SERVER = 1; InstanceId = 0 * 2) Both - * - Short Server ID == 0 or 65535 bs) + * - Short Server ID == 0 bs) * SECURITY = 0; InstanceId = 0 * SERVER = 1; InstanceId = null * - Short Server ID == 1 - 65534 lwm2m) @@ -202,8 +202,8 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask public List> toRequests(BootstrapConfig bootstrapConfigNew, ContentFormat contentFormat, String endpoint) { - Integer bootstrapSecurityInstanceId = this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(BOOTSTRAP_DEFAULT_SHORT_ID_0) == null ? - 0 : this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(BOOTSTRAP_DEFAULT_SHORT_ID_0); + Integer bootstrapSecurityInstanceId = this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(BOOTSTRAP.getId()) == null ? + 0 : this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(BOOTSTRAP.getId()); List> requests = new ArrayList<>(); Set pathsDelete = new HashSet<>(); ConcurrentHashMap> requestsWrite = new ConcurrentHashMap<>(); @@ -214,7 +214,7 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask // bootstrap Security new - There can only be one instance of bootstrap at a time. /// bs: handle security only for (BootstrapConfig.ServerSecurity security : new TreeMap<>(bootstrapConfigNew.security).values()) { - if (security.bootstrapServer && security.serverId == BOOTSTRAP_DEFAULT_SHORT_ID_0) { + if (security.bootstrapServer && security.serverId == BOOTSTRAP.getId()) { // delete old bootstrap Security String path = "/" + SECURITY + "/" + bootstrapSecurityInstanceId; pathsDelete.add(path); @@ -231,7 +231,7 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask /// lwm2m server: handle security & server //max Lwm2m Security instance old id if new for (Integer shortId : this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().keySet()) { - if (shortId >= BOOTSTRAP_DEFAULT_SHORT_ID_0 && shortId <= LWM2M_DEFAULT_SHORT_ID_65534) { + if (shortId >= BOOTSTRAP.getId() && shortId <= LWM2M_SERVER_MAX.getId()) { lwm2mSecurityInstanceIdMax = this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(shortId) > lwm2mSecurityInstanceIdMax ? this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(shortId) : lwm2mSecurityInstanceIdMax; @@ -239,7 +239,7 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask } //max Lwm2m Server instance old id if new for (Integer shortId : this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().keySet()) { - if (shortId >= LWM2M_DEFAULT_SHORT_ID_1 && shortId <= LWM2M_DEFAULT_SHORT_ID_65534) { + if (shortId >= PRIMARY_LWM2M_SERVER.getId() && shortId <= LWM2M_SERVER_MAX.getId()) { lwm2mServerInstanceIdMax = this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().get(shortId) > lwm2mServerInstanceIdMax ? this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().get(shortId) : lwm2mServerInstanceIdMax; @@ -297,7 +297,7 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask } private boolean validateLwm2mShortServerId(int id){ - return id >= LWM2M_DEFAULT_SHORT_ID_1 && id <= LWM2M_DEFAULT_SHORT_ID_65534; + return id >= PRIMARY_LWM2M_SERVER.getId() && id <= LWM2M_SERVER_MAX.getId(); } @Override diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MConfigurationChecker.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MConfigurationChecker.java index aba29aed24..113c3451e0 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MConfigurationChecker.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MConfigurationChecker.java @@ -21,6 +21,10 @@ import org.eclipse.leshan.server.bootstrap.InvalidConfigurationException; import java.util.Map; +import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.BOOTSTRAP; +import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.NOT_USED_IDENTIFYING_LWM2M_SERVER; +import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.isLwm2mServer; + public class LwM2MConfigurationChecker extends ConfigurationChecker { @Override @@ -74,8 +78,8 @@ public class LwM2MConfigurationChecker extends ConfigurationChecker { * This Resource MUST be set when the Bootstrap-Server Resource has false value. * Specific ID:0 and ID:65535 values MUST NOT be used for identifying the LwM2M Server (Section 6.3 of the LwM2M version 1.0 specification). */ - if (!security.bootstrapServer && (srvCfg.shortId < 1 && srvCfg.shortId > 65534 )) { - throw new InvalidConfigurationException("Specific ID:0 and ID:65535 values MUST NOT be used for identifying the LwM2M Server"); + if (!security.bootstrapServer && !isLwm2mServer(srvCfg.shortId)) { + throw new InvalidConfigurationException("Specific ID:" + BOOTSTRAP.getId() + " and ID:" + NOT_USED_IDENTIFYING_LWM2M_SERVER.getId() + " values MUST NOT be used for identifying the LwM2M Server"); } } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/LwM2MTransportUtil.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/LwM2MTransportUtil.java index 7464361c24..d5ae8febe1 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/LwM2MTransportUtil.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/LwM2MTransportUtil.java @@ -81,9 +81,6 @@ public class LwM2MTransportUtil { public static final String LOG_LWM2M_INFO = "info"; public static final String LOG_LWM2M_ERROR = "error"; public static final String LOG_LWM2M_WARN = "warn"; - public static final int BOOTSTRAP_DEFAULT_SHORT_ID_0 = 0; - public static final int LWM2M_DEFAULT_SHORT_ID_1 = 1; - public static final int LWM2M_DEFAULT_SHORT_ID_65534 = 65534; public static LwM2mOtaConvert convertOtaUpdateValueToString(String pathIdVer, Object value, ResourceModel.Type currentType) { String path = fromVersionedIdToObjectId(pathIdVer); diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java index dfd0ee82bc..9b59b2eee6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java @@ -69,6 +69,11 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.BOOTSTRAP; +import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.LWM2M_SERVER_MAX; +import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.PRIMARY_LWM2M_SERVER; +import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.isLwm2mServer; + @Slf4j @Component public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator { @@ -339,12 +344,12 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator 65535) { - throw new DeviceCredentialsValidationException("Bootstrap Server ShortServerId must be in range [0 - 65535]!"); + if (serverConfig.getShortServerId() != BOOTSTRAP.getId()) { + throw new DeviceCredentialsValidationException("Bootstrap Server ShortServerId must be in range [" + BOOTSTRAP.getId() + "]!"); } } else { - if (serverConfig.getShortServerId() < 1 || serverConfig.getShortServerId() > 65534) { - throw new DeviceCredentialsValidationException("LwM2M Server ShortServerId must be in range [1 - 65534]!"); + if (!isLwm2mServer(serverConfig.getShortServerId())) { + throw new DeviceCredentialsValidationException("LwM2M Server ShortServerId must be in range [" + PRIMARY_LWM2M_SERVER.getId() + " - " + LWM2M_SERVER_MAX.getId() + "!"); } } } else { diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidatorTest.java index b69165be7c..d80b7a6519 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidatorTest.java @@ -48,6 +48,10 @@ import java.util.UUID; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.BDDMockito.willReturn; import static org.mockito.Mockito.verify; +import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.BOOTSTRAP; +import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.LWM2M_SERVER_MAX; +import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.NOT_USED_IDENTIFYING_LWM2M_SERVER; +import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.PRIMARY_LWM2M_SERVER; @SpringBootTest(classes = DeviceProfileDataValidator.class) class DeviceProfileDataValidatorTest { @@ -74,8 +78,8 @@ class DeviceProfileDataValidatorTest { " \"clientOnlyObserveAfterConnect\": 1\n" + " }"; - private static final String msgErrorLwm2mRange = "LwM2M Server ShortServerId must be in range [1 - 65534]!"; - private static final String msgErrorBsRange = "Bootstrap Server ShortServerId must be in range [0 - 65535]!"; + private static final String msgErrorLwm2mRange = "LwM2M Server ShortServerId must be in range [" + PRIMARY_LWM2M_SERVER.getId() + " - " + LWM2M_SERVER_MAX.getId() + "]!"; + private static final String msgErrorBsRange = "Bootstrap Server ShortServerId must be in range [" + BOOTSTRAP.getId() + " - " + NOT_USED_IDENTIFYING_LWM2M_SERVER.getId() + "]!"; private static final String msgErrorNotNull = " Server ShortServerId must not be null!"; private static final String host = "localhost"; private static final String hostBs = "localhost"; @@ -153,7 +157,7 @@ class DeviceProfileDataValidatorTest { @Test void testValidateDeviceProfile_Lwm2mShortServerId_More_65534_Error_BootstrapShortServerId_Ok() { - verifyValidationError(65535, 111, msgErrorLwm2mRange); + verifyValidationError(NOT_USED_IDENTIFYING_LWM2M_SERVER.getId(), 111, msgErrorLwm2mRange); } @Test diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts index 6756ea2874..71f4264f94 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts @@ -76,8 +76,7 @@ export class Lwm2mDeviceConfigServerComponent implements OnInit, ControlValueAcc readonly shortServerIdMin = 1; readonly shortServerIdMax = 65534; - readonly shortServerIdBsMin = 0; - readonly shortServerIdBsMax = 65535; + readonly shortServerIdBs = 0; @Input() @coerceBoolean() @@ -103,7 +102,7 @@ export class Lwm2mDeviceConfigServerComponent implements OnInit, ControlValueAcc serverPublicKey: [''], clientHoldOffTime: ['', [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], shortServerId: ['', this.isBootstrap - ? [Validators.required, Validators.pattern('^(' + this.shortServerIdBsMin+ '|' + this.shortServerIdBsMax + ')$' )] + ? [Validators.required, Validators.pattern('^(' + this.shortServerIdBs + ')$' )] : [Validators.required, Validators.pattern('[0-9]*'), Validators.min(this.shortServerIdMin), Validators.max(this.shortServerIdMax)] ], bootstrapServerAccountTimeout: ['', [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 05a1f5ceb3..912eeedc5f 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -2302,7 +2302,7 @@ "short-id-required": "Short server ID is required.", "short-id-range": "Short server ID should be in a range from {{ min }} to {{ max }}.", "short-id-pattern": "Short server ID must be a positive integer.", - "short-id-pattern-bs": "Short server ID must be only 0 or 65535", + "short-id-pattern-bs": "Short server ID must be only 0", "lifetime": "Client registration lifetime", "lifetime-required": "Client registration lifetime is required.", "lifetime-pattern": "Client registration lifetime must be a positive integer.", From d1656cfd00260e80e615069c9c17b4c7d347688c Mon Sep 17 00:00:00 2001 From: ArtemDzhereleiko Date: Mon, 13 Oct 2025 18:41:15 +0300 Subject: [PATCH 31/88] UI: Add sync with db to resources autocoplete --- .../home/components/rule-node/external/ai-config.component.html | 1 + 1 file changed, 1 insertion(+) diff --git a/ui-ngx/src/app/modules/home/components/rule-node/external/ai-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/external/ai-config.component.html index 5381fc770b..0c05132a56 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/external/ai-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/external/ai-config.component.html @@ -72,6 +72,7 @@ Date: Mon, 13 Oct 2025 20:04:58 +0300 Subject: [PATCH 32/88] lwm2m: bootstrap new: update tests-3 --- .../dao/service/validator/DeviceProfileDataValidator.java | 2 +- .../dao/service/validator/DeviceProfileDataValidatorTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java index 9b59b2eee6..2ea8c343d7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java @@ -349,7 +349,7 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator Date: Mon, 13 Oct 2025 23:11:14 +0300 Subject: [PATCH 33/88] lwm2m: bootstrap new: update tests-4 --- .../lwm2m/AbstractLwM2MIntegrationTest.java | 16 ++++++++-------- .../transport/lwm2m/client/LwM2MTestClient.java | 10 ++++++---- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java index 8a62795863..4828bbbdde 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java @@ -124,8 +124,8 @@ import static org.thingsboard.server.transport.lwm2m.ota.AbstractOtaLwM2MIntegra @Slf4j @DaoSqlTest -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) +//@TestInstance(TestInstance.Lifecycle.PER_CLASS) +//@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) @TestPropertySource(properties = { "transport.lwm2m.enabled=true" }) @@ -324,13 +324,13 @@ public abstract class AbstractLwM2MIntegrationTest extends AbstractTransportInte if (executor != null && !executor.isShutdown()) { executor.shutdownNow(); - if (!executor.awaitTermination(2, TimeUnit.SECONDS)) { - log.warn("⚠️ Executor did not terminate cleanly, forcing GC"); - } +// if (!executor.awaitTermination(2, TimeUnit.SECONDS)) { +// log.warn("⚠️ Executor did not terminate cleanly, forcing GC"); +// } } - Thread.sleep(300); - System.gc(); - log.info("✅ Test teardown completed: {}", this.getClass().getSimpleName()); +// Thread.sleep(300); +// System.gc(); +// log.info("✅ Test teardown completed: {}", this.getClass().getSimpleName()); } private void init() throws Exception { diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java index c368dbfe54..bf5a4aa10d 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java @@ -154,28 +154,30 @@ public class LwM2MTestClient { if (securityBs != null && security != null) { // SECURITIES - forceNullSecurityId(securityBs); - forceNullSecurityId(security); + securityBs.setId(0); + security.setId(1); LwM2mInstanceEnabler[] instances = new LwM2mInstanceEnabler[]{securityBs, security}; initializer.setInstancesForObject(SECURITY, instances); log.warn("Security BS section: securityBsId [{}] Security Lwm2m section: securityLwm2mId [{}] ", securityBs.getId(), security.getId()); // SERVER Server lwm2mServer = new Server(shortServerId, TimeUnit.MINUTES.toSeconds(60)); + lwm2mServer.setId(0); instances = new LwM2mInstanceEnabler[]{lwm2mServer}; initializer.setInstancesForObject(SERVER, instances); } else if (securityBs != null) { // SECURITY -; forceNullSecurityId(securityBs); +; securityBs.setId(0);; initializer.setInstancesForObject(SECURITY, securityBs); // SERVER initializer.setClassForObject(SERVER, Server.class); log.warn("Security BS section: securityBsId [{}] ", securityBs.getId()); } else { // SECURITY - forceNullSecurityId(security); + security.setId(0); initializer.setInstancesForObject(SECURITY, security); // SERVER Server lwm2mServer = new Server(shortServerId, TimeUnit.MINUTES.toSeconds(60)); + lwm2mServer.setId(0); initializer.setInstancesForObject(SERVER, lwm2mServer); log.warn("Security Lwm2m section: securityLwm2mId [{}] Server Lwm2m section: securityLwm2mId [{}] ", security.getId(), lwm2mServer.getId()); } From 2857b8c1cbd6eb1e36fda7a240a3a84275e3261c Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Tue, 14 Oct 2025 10:22:52 +0300 Subject: [PATCH 34/88] lwm2m: bootstrap new: update tests-5 --- .../lwm2m/AbstractLwM2MIntegrationTest.java | 12 +++--- .../lwm2m/client/LwM2MTestClient.java | 37 ++++++++++--------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java index 4828bbbdde..338cf166ae 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java @@ -324,13 +324,13 @@ public abstract class AbstractLwM2MIntegrationTest extends AbstractTransportInte if (executor != null && !executor.isShutdown()) { executor.shutdownNow(); -// if (!executor.awaitTermination(2, TimeUnit.SECONDS)) { -// log.warn("⚠️ Executor did not terminate cleanly, forcing GC"); -// } + if (!executor.awaitTermination(2, TimeUnit.SECONDS)) { + log.warn("Executor did not terminate cleanly, forcing GC"); + } } -// Thread.sleep(300); -// System.gc(); -// log.info("✅ Test teardown completed: {}", this.getClass().getSimpleName()); + Thread.sleep(300); + System.gc(); + log.warn("Test lwm2m after completed: {}", this.getClass().getSimpleName()); } private void init() throws Exception { diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java index bf5a4aa10d..3bb127d736 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java @@ -151,33 +151,33 @@ public class LwM2MTestClient { ObjectsInitializer initializer = createFreshInitializer(); - + forceNullSecurityId(securityBs); + forceNullSecurityId(security); if (securityBs != null && security != null) { // SECURITIES - securityBs.setId(0); - security.setId(1); +// securityBs.setId(0); +// security.setId(1); + LwM2mInstanceEnabler[] instances = new LwM2mInstanceEnabler[]{securityBs, security}; initializer.setInstancesForObject(SECURITY, instances); log.warn("Security BS section: securityBsId [{}] Security Lwm2m section: securityLwm2mId [{}] ", securityBs.getId(), security.getId()); // SERVER Server lwm2mServer = new Server(shortServerId, TimeUnit.MINUTES.toSeconds(60)); - lwm2mServer.setId(0); +// lwm2mServer.setId(0); instances = new LwM2mInstanceEnabler[]{lwm2mServer}; initializer.setInstancesForObject(SERVER, instances); } else if (securityBs != null) { // SECURITY -; securityBs.setId(0);; initializer.setInstancesForObject(SECURITY, securityBs); // SERVER initializer.setClassForObject(SERVER, Server.class); log.warn("Security BS section: securityBsId [{}] ", securityBs.getId()); } else { // SECURITY - security.setId(0); initializer.setInstancesForObject(SECURITY, security); // SERVER Server lwm2mServer = new Server(shortServerId, TimeUnit.MINUTES.toSeconds(60)); - lwm2mServer.setId(0); +// lwm2mServer.setId(0); initializer.setInstancesForObject(SERVER, lwm2mServer); log.warn("Security Lwm2m section: securityLwm2mId [{}] Server Lwm2m section: securityLwm2mId [{}] ", security.getId(), lwm2mServer.getId()); } @@ -470,7 +470,8 @@ public class LwM2MTestClient { } private ObjectsInitializer createFreshInitializer() { - List models = new ArrayList<>(ObjectLoader.loadAllDefault()); +// List models = new ArrayList<>(ObjectLoader.loadAllDefault()); + List models = ObjectLoader.loadAllDefault(); for (String resourceName : lwm2mClientResources) { try (InputStream in = LwM2MTestClient.class.getClassLoader() .getResourceAsStream("lwm2m/" + resourceName)) { @@ -499,27 +500,27 @@ public class LwM2MTestClient { return new ObjectsInitializer(model); } - private void forceNullSecurityId(Security securityBs) { - if (securityBs == null) { + private void forceNullSecurityId(Security security) { + if (security == null) { return; } try { - Field field = securityBs.getClass().getDeclaredField("id"); + Field field = security.getClass().getDeclaredField("id"); field.setAccessible(true); - field.set(securityBs, null); - log.info("[forceNullSecurityId] Set id=null for {}", securityBs); + field.set(security, null); + log.info("[forceNullSecurityId] Set id=null for {}", security); } catch (NoSuchFieldException e) { try { // Якщо поле в батьківському класі (наприклад SecurityObjectInstance) - Field field = securityBs.getClass().getSuperclass().getDeclaredField("id"); + Field field = security.getClass().getSuperclass().getDeclaredField("id"); field.setAccessible(true); - field.set(securityBs, null); - log.info("[forceNullSecurityId] Set id=null for {} (via superclass)", securityBs); + field.set(security, null); + log.info("[forceNullSecurityId] Set id=null for {} (via superclass)", security); } catch (Exception ex) { - log.error("[forceNullSecurityId] Field 'id' not found for {}", securityBs.getClass(), ex); + log.error("[forceNullSecurityId] Field 'id' not found for {}", security.getClass(), ex); } } catch (Exception e) { - log.error("[forceNullSecurityId] Failed to set id=null for {}", securityBs.getClass(), e); + log.error("[forceNullSecurityId] Failed to set id=null for {}", security.getClass(), e); } } } From ab7f9b00a283dcab0e0d55d96531ef0a083c2f27 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 14 Oct 2025 12:19:35 +0300 Subject: [PATCH 35/88] Update UI help links url to release-4.2.1 --- application/src/main/resources/thingsboard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 660e9e3a59..7da26fdc81 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -208,7 +208,7 @@ ui: # Help parameters help: # Base URL for UI help assets - base-url: "${UI_HELP_BASE_URL:https://raw.githubusercontent.com/thingsboard/thingsboard-ui-help/release-4.2}" + base-url: "${UI_HELP_BASE_URL:https://raw.githubusercontent.com/thingsboard/thingsboard-ui-help/release-4.2.1 }" # Database telemetry parameters database: From a585a3b9e69ec1c0cad03c96424db65301caa171 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 14 Oct 2025 13:03:45 +0300 Subject: [PATCH 36/88] minor refactoring --- ...CalculatedFieldEntityMessageProcessor.java | 4 +- ...alculatedFieldManagerMessageProcessor.java | 86 ++++------ ...tractCalculatedFieldProcessingService.java | 158 ++++++++++-------- .../cf/CalculatedFieldProcessingService.java | 2 +- ...faultCalculatedFieldProcessingService.java | 4 +- .../DefaultCalculatedFieldQueueService.java | 28 +++- .../state/aggregation/AggArgumentEntry.java | 3 +- .../service/cf/ctx/state/aggregation/agg.json | 4 +- .../processing/AbstractConsumerService.java | 2 - ...tValuesAggregationCalculatedFieldTest.java | 73 +++++++- .../server/dao/relation/RelationService.java | 11 +- .../aggregation/CfAggTrigger.java | 2 +- ...gregationCalculatedFieldConfiguration.java | 2 +- .../ProfileEntityRelationPathQuery.java | 21 +++ .../tbel/TbelCfLatestValuesAggregation.java | 2 +- .../dao/relation/BaseRelationService.java | 98 +++++++++-- .../server/dao/relation/RelationDao.java | 5 +- .../dao/sql/relation/JpaRelationDao.java | 102 ++++++++++- .../dao/sql/relation/RelationRepository.java | 3 +- 19 files changed, 436 insertions(+), 174 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 98b5eba4c8..9dfde4d821 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -69,9 +69,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createStateByType; @@ -264,7 +262,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM @SneakyThrows private Map fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId) { - ListenableFuture> argumentsFuture = cfService.fetchAggArguments(ctx, entityId); + ListenableFuture> argumentsFuture = cfService.fetchAggEntityArguments(ctx, entityId); // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 8a717c3cdb..3c1e21bcc8 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -24,8 +24,8 @@ import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; import org.thingsboard.server.actors.calculatedField.EntityInitCalculatedFieldMsg.StateAction; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; -import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ProfileEntityIdInfo; @@ -255,7 +255,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware MultipleTbCallback callbackFor2 = new MultipleTbCallback(2, callback); // process aggregation cfs(in any) - List cfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(entityId, profileId); + List cfsRelatedToEntity = getCfsWithRelationToEntity(entityId, profileId); if (!cfsRelatedToEntity.isEmpty()) { MultipleTbCallback multiCallback = new MultipleTbCallback(cfsRelatedToEntity.size(), callbackFor2); cfsRelatedToEntity.forEach(ctx -> { @@ -289,8 +289,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware MultipleTbCallback callbackFor2 = new MultipleTbCallback(2, callback); // process aggregation cfs(in any) - List oldCfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getOldProfileId()); - List newCfsRelatedToEntity = getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getProfileId()); + List oldCfsRelatedToEntity = getCfsWithRelationToEntity(msg.getEntityId(), msg.getOldProfileId()); + List newCfsRelatedToEntity = getCfsWithRelationToEntity(msg.getEntityId(), msg.getProfileId()); var fieldsWithRelatedEntityCount = oldCfsRelatedToEntity.size() + newCfsRelatedToEntity.size(); if (fieldsWithRelatedEntityCount > 0) { MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsWithRelatedEntityCount, callbackFor2); @@ -335,7 +335,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); - getCalculatedFieldsRelatedToEntity(msg.getEntityId(), msg.getProfileId()).forEach(ctx -> { + getCfsWithRelationToEntity(msg.getEntityId(), msg.getProfileId()).forEach(ctx -> { applyToTargetCfEntityActors(ctx, callback, (id, cb) -> deleteRelatedEntity(id, msg.getEntityId(), cb)); }); if (isMyPartition(msg.getEntityId(), callback)) { @@ -559,6 +559,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onCfDeleted(ComponentLifecycleMsg msg, TbCallback callback) { var cfId = new CalculatedFieldId(msg.getEntityId().getId()); var cfCtx = calculatedFields.remove(cfId); // fixme wtf? why isn't ctx closed properly? + cfTriggers.remove(cfId); if (cfCtx == null) { log.debug("[{}] CF was already deleted [{}]", tenantId, cfId); callback.onSuccess(); @@ -613,76 +614,55 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private List filterAggregationCfs(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); - - List aggregationCalculatedFields = cfTriggers.entrySet().stream() + return cfTriggers.entrySet().stream() .filter(entry -> aggMatches(entry.getValue(), msg.getProto())) .map(Entry::getKey) .map(calculatedFields::get) .filter(Objects::nonNull) + .flatMap(cf -> findRelationsForCf(entityId, cf).stream()) .toList(); - - List filteredByRelationCfs = new ArrayList<>(); - for (CalculatedFieldCtx cf : aggregationCalculatedFields) { - EntityId cfEntityId = cf.getEntityId(); - if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - RelationPathLevel relation = aggConfig.getSource().getRelation(); - EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) - ? cfEntityId - : getProfileId(tenantId, cfEntityId); - EntityId targetEntity = switch (relation.direction()) { - case FROM -> - relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).getFrom(); - case TO -> - relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0).getTo(); - }; - if (targetEntity != null) { - filteredByRelationCfs.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), targetEntity)); - } - } - } - return filteredByRelationCfs; } - private List getCalculatedFieldsRelatedToEntity(EntityId entityId, EntityId profileId) { - List aggCFsUsedProfile = cfTriggers.entrySet().stream() + private List getCfsWithRelationToEntity(EntityId entityId, EntityId profileId) { + return cfTriggers.entrySet().stream() .filter(entry -> entry.getValue().matchesProfile(profileId)) .map(Entry::getKey) .map(calculatedFields::get) .filter(Objects::nonNull) + .filter(cf -> !findRelationsForCf(entityId, cf).isEmpty()) .toList(); - - List filteredByRelationCfs = new ArrayList<>(); - for (CalculatedFieldCtx cf : aggCFsUsedProfile) { - CalculatedFieldEntityCtxId calculatedFieldEntityCtxId = filterCfByRelationWithEntity(entityId, cf); - if (calculatedFieldEntityCtxId != null) { - filteredByRelationCfs.add(cf); - } - } - return filteredByRelationCfs; } - private CalculatedFieldEntityCtxId filterCfByRelationWithEntity(EntityId entityId, CalculatedFieldCtx cf) { - EntityId cfEntityId = cf.getEntityId(); - if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - RelationPathLevel relation = aggConfig.getSource().getRelation(); - EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) + private List findRelationsForCf(EntityId entityId, CalculatedFieldCtx cf) { + List result = new ArrayList<>(); + if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration configuration) { + AggSource source = configuration.getSource(); + RelationPathLevel relation = source.getRelation(); + EntityId cfEntityId = cf.getEntityId(); + EntityId targetProfileId = isProfileEntity(cfEntityId.getEntityType()) ? cfEntityId : getProfileId(tenantId, cfEntityId); - EntityId targetEntity = switch (relation.direction()) { + switch (relation.direction()) { case FROM -> { - EntityRelation entityRelation = relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); - yield entityRelation == null ? null : entityRelation.getFrom(); + List relationsByTo = relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), targetProfileId); + if (relationsByTo != null && !relationsByTo.isEmpty()) { + EntityRelation entityRelation = relationsByTo.get(0); // only one supported + result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getFrom())); + } } case TO -> { - EntityRelation entityRelation = relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0); - yield entityRelation == null ? null : entityRelation.getTo(); + List relationsByFrom = relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), targetProfileId); + if (relationsByFrom != null && !relationsByFrom.isEmpty()) { + for (EntityRelation entityRelation : relationsByFrom) { + if (entityRelation.getTo().equals(cf.getEntityId())) { + result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getTo())); + } + } + } } - }; - if (targetEntity != null) { - return new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), targetEntity); } } - return null; + return result; } private boolean aggMatches(CfAggTrigger cfAggTrigger, CalculatedFieldTelemetryMsgProto proto) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 304b747f26..db15fc1dc8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -26,7 +26,6 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; @@ -40,6 +39,7 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.attributes.AttributesService; @@ -51,10 +51,11 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; -import java.util.Collection; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -105,7 +106,7 @@ public abstract class AbstractCalculatedFieldProcessingService { } yield futures; } - case LATEST_VALUES_AGGREGATION -> fetchAggregationArgumentFutures(ctx, entityId); + case LATEST_VALUES_AGGREGATION -> fetchAggArguments(ctx, entityId, ts); }; return Futures.whenAllComplete(argFutures.values()) .call(() -> resolveArgumentFutures(argFutures), @@ -122,17 +123,32 @@ public abstract class AbstractCalculatedFieldProcessingService { return resolveOwnerArgument(tenantId, entityId); } - private List resolveRelatedEntities(TenantId tenantId, EntityId entityId, AggSource aggSource) { + private ListenableFuture> resolveRelatedEntities(TenantId tenantId, EntityId entityId, AggSource aggSource) { RelationPathLevel relation = aggSource.getRelation(); - return switch (relation.direction()) { - case FROM -> aggSource.getEntityProfiles().stream() - .map(profile -> relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), profile)) - .flatMap(Collection::stream) - .map(EntityRelation::getTo) + + List>> relationListsFut = new ArrayList<>(); + if (aggSource.getEntityProfiles().isEmpty()) { + relationListsFut.add(relationService.findByProfileEntityRelationPathQueryAsync(tenantId, new ProfileEntityRelationPathQuery(entityId, relation, null))); + } else { + aggSource.getEntityProfiles().forEach(profile -> relationListsFut.add(relationService.findByProfileEntityRelationPathQueryAsync(tenantId, new ProfileEntityRelationPathQuery(entityId, relation, profile)))); + } + + return Futures.transform(Futures.allAsList(relationListsFut), relationLists -> { + if (relationLists == null) { + return new ArrayList<>(); + } + List allRelations = relationLists.stream() + .filter(Objects::nonNull) + .flatMap(List::stream) .toList(); - case TO -> - aggSource.getEntityProfiles().stream().map(profile -> relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), profile).getFrom()).toList(); - }; + + return switch (relation.direction()) { + case FROM -> allRelations.stream() + .map(EntityRelation::getTo) + .toList(); + case TO -> allRelations.isEmpty() ? List.of() : List.of(allRelations.get(0).getFrom()); + }; + }, calculatedFieldCallbackExecutor); } protected Map resolveArgumentFutures(Map> argFutures) { @@ -174,6 +190,34 @@ public abstract class AbstractCalculatedFieldProcessingService { return argFutures; } + protected Map> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { + LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + + ListenableFuture> relatedEntities = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getSource()); + + Map> futures = new HashMap<>(); + aggConfig.getInputs().forEach((key, refKey) -> { + Argument argument = new Argument(); + argument.setRefEntityKey(refKey); + futures.put(key, Futures.transformAsync(relatedEntities, entityIds -> fetchAggArgumentEntry(ctx.getTenantId(), entityIds, argument, System.currentTimeMillis()), MoreExecutors.directExecutor())); + }); + return futures; + } + + protected ListenableFuture> fetchEntityAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { + LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + Map> futures = new HashMap<>(); + aggConfig.getInputs().forEach((key, refKey) -> { + Argument argument = new Argument(); + argument.setRefEntityKey(refKey); + ListenableFuture argEntryFut = fetchSingleAggArgumentEntry(ctx.getTenantId(), entityId, argument, ts); + futures.put(key, argEntryFut); + }); + return Futures.whenAllComplete(futures.values()) + .call(() -> resolveArgumentFutures(futures), + MoreExecutors.directExecutor()); + } + private ListenableFuture> resolveGeofencingEntityIds(TenantId tenantId, EntityId entityId, Map.Entry entry) { Argument value = entry.getValue(); if (value.getRefEntityId() != null) { @@ -197,50 +241,6 @@ public abstract class AbstractCalculatedFieldProcessingService { return ownerService.getOwner(tenantId, entityId); } - private Map> fetchAggregationArgumentFutures(CalculatedFieldCtx ctx, EntityId entityId) { - LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); - - List entityIds = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getSource()); - - Map> futures = new HashMap<>(); - aggConfig.getInputs().forEach((key, refKey) -> { - Argument argument = new Argument(); - argument.setRefEntityKey(refKey); - futures.put(key, fetchAggArgumentEntry(ctx.getTenantId(), entityIds, argument, System.currentTimeMillis())); - }); - return futures; - } - - public ListenableFuture fetchAggArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) { - List>> futures = aggEntities.stream() - .map(entityId -> fetchSingleAggArgumentEntry(tenantId, entityId, argument, startTs)) - .toList(); - - ListenableFuture>> allFutures = Futures.allAsList(futures); - - return Futures.transform(allFutures, - entries -> ArgumentEntry.createAggArgument( - entries.stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) - ), - MoreExecutors.directExecutor()); - } - - protected ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { - CalculatedFieldConfiguration configuration = ctx.getCalculatedField().getConfiguration(); - LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) configuration; - Map> futures = new HashMap<>(); - aggConfig.getInputs().forEach((key, refKey) -> { - Argument argument = new Argument(); - argument.setRefEntityKey(refKey); - - ListenableFuture argumentEntryListenableFuture = fetchAggArgumentEntry(ctx.getTenantId(), List.of(entityId), argument, System.currentTimeMillis()); - futures.put(key, argumentEntryListenableFuture); - }); - return Futures.whenAllComplete(futures.values()) - .call(() -> resolveArgumentFutures(futures), - MoreExecutors.directExecutor()); - } - private ListenableFuture fetchGeofencingKvEntry(TenantId tenantId, List geofencingEntities, Argument argument) { if (argument.getRefEntityKey().getType() != ArgumentType.ATTRIBUTE) { throw new IllegalStateException("Unsupported argument key type: " + argument.getRefEntityKey().getType()); @@ -265,6 +265,22 @@ public abstract class AbstractCalculatedFieldProcessingService { .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))), MoreExecutors.directExecutor()); } + public ListenableFuture fetchAggArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) {List>> futures = aggEntities.stream() + .map(entityId -> { + ListenableFuture singleAggEntryFut = fetchSingleAggArgumentEntry(tenantId, entityId, argument, startTs); + return Futures.transform(singleAggEntryFut, singleAggEntry -> Map.entry(entityId, singleAggEntry), MoreExecutors.directExecutor()); + }) + .toList(); + + ListenableFuture>> allFutures = Futures.allAsList(futures); + + return Futures.transform(allFutures, + entries -> ArgumentEntry.createAggArgument( + entries.stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) + ), + MoreExecutors.directExecutor()); + } + protected ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId entityId, Argument argument, long startTs) { return switch (argument.getRefEntityKey().getType()) { case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument, startTs); @@ -309,7 +325,15 @@ public abstract class AbstractCalculatedFieldProcessingService { }, calculatedFieldCallbackExecutor)); } - protected ListenableFuture> fetchSingleAggArgumentEntry(TenantId tenantId, EntityId entityId, Argument argument, long startTs) { + private ReadTsKvQuery buildTsRollingQuery(TenantId tenantId, Argument argument, long startTs, long endTs) { + long maxDataPoints = apiLimitService.getLimit( + tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); + int argumentLimit = argument.getLimit(); + int limit = argumentLimit == 0 || argumentLimit > maxDataPoints ? (int) maxDataPoints : argumentLimit; + return new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, endTs, 0, limit, Aggregation.NONE); + } + + private ListenableFuture fetchSingleAggArgumentEntry(TenantId tenantId, EntityId entityId, Argument argument, long startTs) { return switch (argument.getRefEntityKey().getType()) { case TS_ROLLING -> throw new IllegalStateException("TS_ROLLING is not supported for aggregation"); case ATTRIBUTE -> fetchAttributeAggEntry(tenantId, entityId, argument, startTs); @@ -317,36 +341,26 @@ public abstract class AbstractCalculatedFieldProcessingService { }; } - private ListenableFuture> fetchAttributeAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultLastUpdateTs) { + private ListenableFuture fetchAttributeAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultLastUpdateTs) { log.trace("[{}][{}] Fetching attribute for key {}", tenantId, entityId, argument.getRefEntityKey()); var attributeOptFuture = attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()); return Futures.transform(attributeOptFuture, attrOpt -> { log.debug("[{}][{}] Fetched attribute for key {}: {}", tenantId, entityId, argument.getRefEntityKey(), attrOpt); AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, 0L)); - AggSingleArgumentEntry entry = new AggSingleArgumentEntry(entityId, attributeKvEntry); - return Map.entry(entityId, entry); + return new AggSingleArgumentEntry(entityId, attributeKvEntry); }, calculatedFieldCallbackExecutor); } - protected ListenableFuture> fetchTsLatestAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultTs) { + private ListenableFuture fetchTsLatestAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultTs) { String key = argument.getRefEntityKey().getKey(); log.trace("[{}][{}] Fetching latest timeseries {}", tenantId, entityId, key); return Futures.transform( timeseriesService.findLatest(tenantId, entityId, key), result -> { log.debug("[{}][{}] Fetched latest timeseries {}: {}", tenantId, entityId, key, result); - Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))); - AggSingleArgumentEntry entry = new AggSingleArgumentEntry(entityId, tsKvEntry.get()); - return Map.entry(entityId, entry); + Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(defaultTs, createDefaultKvEntry(argument), 0L))); + return new AggSingleArgumentEntry(entityId, tsKvEntry.get()); }, calculatedFieldCallbackExecutor); } - private ReadTsKvQuery buildTsRollingQuery(TenantId tenantId, Argument argument, long startTs, long endTs) { - long maxDataPoints = apiLimitService.getLimit( - tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); - int argumentLimit = argument.getLimit(); - int limit = argumentLimit == 0 || argumentLimit > maxDataPoints ? (int) maxDataPoints : argumentLimit; - return new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, endTs, 0, limit, Aggregation.NONE); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java index 4b3e994f23..52b3341151 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -33,7 +33,7 @@ public interface CalculatedFieldProcessingService { ListenableFuture> fetchArguments(CalculatedFieldCtx ctx, EntityId entityId); - ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId); + ListenableFuture> fetchAggEntityArguments(CalculatedFieldCtx ctx, EntityId entityId); Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index b7bd4d87fd..f9ed69a313 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -91,8 +91,8 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF } @Override - public ListenableFuture> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId) { - return super.fetchAggArguments(ctx, entityId, System.currentTimeMillis()); + public ListenableFuture> fetchAggEntityArguments(CalculatedFieldCtx ctx, EntityId entityId) { + return super.fetchEntityAggArguments(ctx, entityId, System.currentTimeMillis()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index a75df1fb40..44cb2aaee4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -27,6 +27,7 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -39,6 +40,7 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationPathLevel; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; @@ -191,19 +193,27 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS for (CalculatedFieldCtx cfCtx : cfCtxs) { EntityId cfEntityId = cfCtx.getEntityId(); if (cfCtx.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - RelationPathLevel relation = aggConfig.getSource().getRelation(); + AggSource source = aggConfig.getSource(); + RelationPathLevel relation = source.getRelation(); EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) ? cfEntityId : calculatedFieldCache.getProfileId(tenantId, cfEntityId); - EntityRelation entityRelation = switch (relation.direction()) { - case FROM -> - relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); - case TO -> - relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId).get(0); + switch (relation.direction()) { + case FROM -> { + List byToAndType = relationService.findByToAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); +// List byTo = relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); + if (!byToAndType.isEmpty()) { + return true; + } + } + case TO -> { + List byFromAndType = relationService.findByFromAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); +// List byFrom = relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); + if (!byFromAndType.isEmpty()) { + return true; + } + } }; - if (entityRelation != null) { - return true; - } } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java index 138053793f..12ae2c4638 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.cf.ctx.state.aggregation; import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.script.api.tbel.TbelCfLatestValuesAggregation; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; @@ -66,7 +67,7 @@ public class AggArgumentEntry implements ArgumentEntry { @Override public TbelCfArg toTbelCfArg() { - return null; + return new TbelCfLatestValuesAggregation(aggInputs.values()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json index d402f23c6e..b39092ca74 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json @@ -7,8 +7,8 @@ "allEnabledUntil": 1769907492297 }, "entityId": { - "entityType": "ASSET", - "id": "cc830710-a4cf-11f0-87cb-2d6683c4fccf" + "entityType": "ASSET_PROFILE", + "id": "bb8ddd40-a8bc-11f0-869b-e9d81fa6eaf1" }, "configuration": { "type": "LATEST_VALUES_AGGREGATION", diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 761b69024f..0b5fd3a2c0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -184,7 +184,6 @@ public abstract class AbstractConsumerService { + ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + + ObjectNode occupancy2 = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy2).isNotNull(); + assertThat(occupancy2.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy2.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + }); + + postTelemetry(device3.getId(), "{\"occupied\":true}"); + + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy2 = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy2).isNotNull(); + assertThat(occupancy2.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy2.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + }); + } + + @Test public void testDeleteRelation_checkMetricsCalculation() throws Exception { deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); @@ -215,6 +260,28 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll }); } +// @Test +// public void testCfWithoutTargetProfileSpecified_checkMetricsCalculation() throws Exception { +// Device device3 = createDevice("Device 3", "1234567890333"); +// postTelemetry(device3.getId(), "{\"occupied\":true}"); +// createEntityRelation(asset.getId(), device3.getId(), "Contains"); +// +// var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) calculatedField.getConfiguration(); +// configuration.getSource().setEntityProfiles(Collections.emptyList()); +// calculatedField.setConfiguration(configuration); +// saveCalculatedField(calculatedField); +// +// await().alias("update cf and perform aggregation for 3 devices").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) +// .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) +// .untilAsserted(() -> { +// ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); +// assertThat(occupancy).isNotNull(); +// assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); +// assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("2"); +// assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("3"); +// }); +// } + private void checkInitialCalculation() { await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) @@ -229,7 +296,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); } - private CalculatedField createOccupancyCF(EntityId entityId, List profiles) { + private CalculatedField createOccupancyCF(String name, EntityId entityId, List profiles) { Map aggMetrics = new HashMap<>(); AggMetric freeSpaces = new AggMetric(); @@ -252,7 +319,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll Output output = new Output(); output.setType(OutputType.TIME_SERIES); - return createAggCf("Occupied spaces", entityId, + return createAggCf(name, entityId, buildSource(EntitySearchDirection.FROM, "Contains", profiles), Map.of("oc", new ReferencedEntityKey("occupied", ArgumentType.TS_LATEST, null)), aggMetrics, diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java index 5b3290c110..20348e5922 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationInfo; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; +import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -86,11 +87,17 @@ public interface RelationService { ListenableFuture> findByRelationPathQueryAsync(TenantId tenantId, EntityRelationPathQuery relationPathQuery); + ListenableFuture> findByProfileEntityRelationPathQueryAsync(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery); + + List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery); + + ListenableFuture> findByFromAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId from, String relationType, EntityId targetProfileId); + List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId profileId); - EntityRelation findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId profileId); + ListenableFuture> findByToAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId to, String relationType, EntityId targetProfileId); - void evictRelationsByProfile(TenantId tenantId, EntityId profileId); + List findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId profileId); void evictRelationsByEntityAndProfile(TenantId tenantId, EntityId entityId, EntityId profileId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java index 393697877e..65d545bc6e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java @@ -42,7 +42,7 @@ public class CfAggTrigger { } public boolean matchesProfile(EntityId profileId) { - return entityProfiles.contains(profileId); + return entityProfiles.isEmpty() || entityProfiles.contains(profileId); } public boolean matchesTimeSeries(List telemetry) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java index e3db50fda0..2760e1855b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java @@ -45,7 +45,7 @@ public class LatestValuesAggregationCalculatedFieldConfiguration implements Calc public CfAggTrigger buildTrigger() { return CfAggTrigger.builder() .inputs(List.copyOf(inputs.values())) - .entityProfiles(source.getEntityProfiles()) + .entityProfiles(List.copyOf(source.getEntityProfiles())) .build(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java new file mode 100644 index 0000000000..32b338ff6f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.relation; + +import org.thingsboard.server.common.data.id.EntityId; + +public record ProfileEntityRelationPathQuery(EntityId rootEntityId, RelationPathLevel level, EntityId targetEntityProfileId) { +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java index 1b5fa394d2..4d1b42aa94 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java @@ -39,6 +39,6 @@ public class TbelCfLatestValuesAggregation implements TbelCfArg { @Override public long memorySize() { - return 32; + return OBJ_SIZE; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index df79ca145a..2d584ebebe 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -45,6 +45,7 @@ import org.thingsboard.server.common.data.relation.EntityRelationInfo; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.relation.RelationTypeGroup; @@ -515,32 +516,92 @@ public class BaseRelationService implements RelationService { } @Override - public List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId profileId) { - RelationCacheKey cacheKey = RelationCacheKey.builder().from(from).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.FROM).entityProfile(profileId).build(); - return cache.getAndPutInTransaction(cacheKey, - () -> relationDao.findByFromAndTypeAndProfile(tenantId, from, relationType, RelationTypeGroup.COMMON, profileId), - RelationCacheValue::getRelations, - relations -> RelationCacheValue.builder().relations(relations).build(), false); + public ListenableFuture> findByProfileEntityRelationPathQueryAsync(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery) { + log.trace("Executing findByProfileEntityRelationPathQueryAsync, tenantId [{}], relationPathQuery {}", tenantId, relationPathQuery); + validateId(tenantId, id -> "Invalid tenant id: " + id); + validate(relationPathQuery); + RelationPathLevel relationPathLevel = relationPathQuery.level(); + return switch (relationPathLevel.direction()) { + case FROM -> findByFromAndTypeAndEntityProfileAsync(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); + case TO -> findByToAndTypeAndEntityProfileAsync(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); + }; } @Override - public EntityRelation findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId profileId) { - RelationCacheKey cacheKey = RelationCacheKey.builder().to(to).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.TO).entityProfile(profileId).build(); - return cache.getAndPutInTransaction(cacheKey, - () -> relationDao.findByToAndTypeAndProfile(tenantId, to, relationType, RelationTypeGroup.COMMON, profileId), - RelationCacheValue::getRelation, - relation -> RelationCacheValue.builder().relation(relation).build(), false); + public List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery) { + log.trace("Executing findByProfileEntityRelationPathQuery, tenantId [{}], relationPathQuery {}", tenantId, relationPathQuery); + validateId(tenantId, id -> "Invalid tenant id: " + id); + validate(relationPathQuery); + return relationDao.findByProfileEntityRelationPathQuery(tenantId, relationPathQuery); +// RelationPathLevel relationPathLevel = relationPathQuery.level(); +// return switch (relationPathLevel.direction()) { +// case FROM -> findByFromAndTypeAndEntityProfile(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); +// case TO -> findByToAndTypeAndEntityProfile(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); +// }; + } + + @Override + public ListenableFuture> findByFromAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId from, String relationType, EntityId targetProfileId) { + log.trace("Executing findByFromAndTypeAndEntityProfileAsync [{}][{}][{}]", from, relationType, targetProfileId); + validate(from); + validateType(relationType); + if (targetProfileId == null) { + return findByFromAndTypeAsync(tenantId, from, relationType, RelationTypeGroup.COMMON); + } + return executor.submit(() -> findByFromAndTypeAndEntityProfile(tenantId, from, relationType, targetProfileId)); + } + + @Override + public List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId targetProfileId) { + if (targetProfileId == null) { + return findByFromAndType(tenantId, from, relationType, RelationTypeGroup.COMMON); + } +// RelationCacheKey cacheKey = RelationCacheKey.builder().from(from).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.FROM).entityProfile(targetProfileId).build(); +// return cache.getAndPutInTransaction(cacheKey, +// () -> relationDao.findByFromAndTypeAndProfile(tenantId, from, relationType, RelationTypeGroup.COMMON, targetProfileId), +// RelationCacheValue::getRelations, +// relations -> RelationCacheValue.builder().relations(relations).build(), false); + + return relationDao.findByFromAndTypeAndProfile(tenantId, from, relationType, RelationTypeGroup.COMMON, targetProfileId); } @Override - public void evictRelationsByProfile(TenantId tenantId, EntityId profileId) { - RelationCacheKey key = RelationCacheKey.builder().entityProfile(profileId).build(); - cache.evict(List.of(key)); - log.debug("Processed evict relations by key: {}", key); + public ListenableFuture> findByToAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId to, String relationType, EntityId targetProfileId) { + log.trace("Executing findByToAndTypeAndEntityProfileAsync [{}][{}][{}]", to, relationType, targetProfileId); + validate(to); + validateType(relationType); + if (targetProfileId == null) { + return findByToAndTypeAsync(tenantId, to, relationType, RelationTypeGroup.COMMON); + } + return executor.submit(() -> findByToAndTypeAndEntityProfile(tenantId, to, relationType, targetProfileId)); + } + + @Override + public List findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId targetProfileId) { + if (targetProfileId == null) { + return findByFromAndType(tenantId, to, relationType, RelationTypeGroup.COMMON); + } +// RelationCacheKey cacheKey = RelationCacheKey.builder().to(to).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.TO).entityProfile(targetProfileId).build(); +// return cache.getAndPutInTransaction(cacheKey, +// () -> relationDao.findByToAndTypeAndProfile(tenantId, to, relationType, RelationTypeGroup.COMMON, targetProfileId), +// RelationCacheValue::getRelations, +// relations -> RelationCacheValue.builder().relations(relations).build(), false); + + return relationDao.findByToAndTypeAndProfile(tenantId, to, relationType, RelationTypeGroup.COMMON, targetProfileId); } @Override public void evictRelationsByEntityAndProfile(TenantId tenantId, EntityId entityId, EntityId profileId) { + +// List keys = new ArrayList<>(5); +// keys.add(new RelationCacheKey(entityId, null, event.getType(), event.getTypeGroup())); +// keys.add(new RelationCacheKey(event.getFrom(), null, event.getType(), event.getTypeGroup(), EntitySearchDirection.FROM)); +// keys.add(new RelationCacheKey(event.getFrom(), null, null, event.getTypeGroup(), EntitySearchDirection.FROM)); +// keys.add(new RelationCacheKey(null, event.getTo(), event.getType(), event.getTypeGroup(), EntitySearchDirection.TO)); +// keys.add(new RelationCacheKey(null, event.getTo(), null, event.getTypeGroup(), EntitySearchDirection.TO)); +// cache.evict(keys); +// log.debug("Processed evict event: {}", event); + List keys = new ArrayList<>(2); keys.add(RelationCacheKey.builder().from(entityId).entityProfile(profileId).build()); keys.add(RelationCacheKey.builder().to(entityId).entityProfile(profileId).build()); @@ -548,6 +609,11 @@ public class BaseRelationService implements RelationService { log.debug("Processed evict relations by keys: {}", keys); } + private void validate(ProfileEntityRelationPathQuery relationPathQuery) { + validateId((UUIDBased) relationPathQuery.rootEntityId(), id -> "Invalid root entity id: " + id); + relationPathQuery.level().validate(); + } + private void validate(EntityRelationPathQuery relationPathQuery) { validateId((UUIDBased) relationPathQuery.rootEntityId(), id -> "Invalid root entity id: " + id); List levels = relationPathQuery.levels(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index f529396965..6318f1fc9d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -20,6 +20,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; +import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -40,7 +41,7 @@ public interface RelationDao { List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup); - EntityRelation findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId); + List findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId); List findAllByTo(TenantId tenantId, EntityId to); @@ -78,4 +79,6 @@ public interface RelationDao { List findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery relationPathQuery); + List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery query); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index c859b71580..afecca26c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -41,9 +42,12 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; +import static org.thingsboard.server.dao.model.ModelConstants.ASSET_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.DEVICE_TABLE_NAME; import static org.thingsboard.server.dao.model.ModelConstants.RELATION_FROM_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.RELATION_FROM_TYPE_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.RELATION_TABLE_NAME; @@ -118,8 +122,14 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple } @Override - public EntityRelation findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId) { - return DaoUtil.getData(relationRepository.findByToAndProfile(to.getId(), to.getEntityType().name(), typeGroup.name(), relationType, profileId.getId())); + public List findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId) { + return DaoUtil.convertDataList( + relationRepository.findByToAndProfile( + to.getId(), + to.getEntityType().name(), + typeGroup.name(), + relationType, + profileId.getId())); } @Override @@ -402,4 +412,92 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple return sb.toString(); } + @Override + public List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery query) { + String sql = buildProfileEntityRelationPathSql(query); + Object[] params = buildProfileEntityRelationPathParams(query); + + log.trace("[{}] profile entity relation path query: {}", tenantId, sql); + + return jdbcTemplate.queryForList(sql, params).stream() + .map(row -> { + var entityRelation = new EntityRelation(); + var fromId = (UUID) row.get(RELATION_FROM_ID_PROPERTY); + var fromType = (String) row.get(RELATION_FROM_TYPE_PROPERTY); + var toId = (UUID) row.get(RELATION_TO_ID_PROPERTY); + var toType = (String) row.get(RELATION_TO_TYPE_PROPERTY); + var grp = (String) row.get(RELATION_TYPE_GROUP_PROPERTY); + var type = (String) row.get(RELATION_TYPE_PROPERTY); + var version = (Long) row.get(VERSION_COLUMN); + + entityRelation.setFrom(EntityIdFactory.getByTypeAndUuid(fromType, fromId)); + entityRelation.setTo(EntityIdFactory.getByTypeAndUuid(toType, toId)); + entityRelation.setType(type); + entityRelation.setTypeGroup(RelationTypeGroup.valueOf(grp)); + entityRelation.setVersion(version); + return entityRelation; + }) + .collect(Collectors.toList()); + } + + private Object[] buildProfileEntityRelationPathParams(ProfileEntityRelationPathQuery query) { + final List params = new ArrayList<>(); + + params.add(query.rootEntityId().getId()); + params.add(query.rootEntityId().getEntityType().name()); + + params.add(query.level().relationType()); + + if (query.targetEntityProfileId() != null) { + params.add(query.targetEntityProfileId().getId()); + params.add(query.targetEntityProfileId().getId()); + } + + return params.toArray(); + } + + private static String buildProfileEntityRelationPathSql(ProfileEntityRelationPathQuery query) { + EntitySearchDirection direction = query.level().direction(); + + StringBuilder sb = new StringBuilder(); + + sb.append("\n") + .append("SELECT r.from_id, r.from_type, r.to_id, r.to_type,\n") + .append(" r.relation_type_group, r.relation_type, r.version\n") + .append("FROM ").append(RELATION_TABLE_NAME).append(" r\n"); + + sb.append("JOIN ").append(DEVICE_TABLE_NAME).append(" d ON "); + if (EntitySearchDirection.FROM == direction) { + sb.append("r.to_id = d.id AND r.to_type = 'DEVICE'").append("\n"); + } else { + sb.append("r.from_id = d.id AND r.from_type = 'DEVICE'").append("\n"); + } + + sb.append("JOIN ").append(ASSET_TABLE_NAME).append(" a ON "); + if (EntitySearchDirection.FROM == direction) { + sb.append("r.to_id = a.id AND r.to_type = 'ASSET'").append("\n"); + } else { + sb.append("r.from_id = a.id AND r.from_type = 'ASSET'").append("\n"); + } + + if (EntitySearchDirection.FROM == direction) { + sb.append("WHERE r.from_id = ?").append("\n") + .append("AND r.from_type = ?").append("\n"); + } else { + sb.append("WHERE r.to_id = ?").append("\n") + .append("AND r.to_type = ?").append("\n"); + } + + sb.append("AND r.relation_type = ?").append("\n") + .append("AND r.relation_type_group = '").append(RelationTypeGroup.COMMON).append("'\n"); + + if (query.targetEntityProfileId() != null) { + sb.append("AND ((d.device_profile_id = ?) OR (a.asset_profile_id = ?))").append("\n"); + } + + sb.append("AND (d.id IS NOT NULL OR a.id IS NOT NULL)"); + + return sb.toString(); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index 9294236526..0ebd5b6ceb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -126,9 +126,8 @@ public interface RelationRepository AND r.relation_type_group = :relationTypeGroup AND ((d.device_profile_id = :profileId) OR (a.asset_profile_id = :profileId)) AND (d.id IS NOT NULL OR a.id IS NOT NULL) - LIMIT 1 """, nativeQuery = true) - Optional findByToAndProfile(@Param("toId") UUID toId, + List findByToAndProfile(@Param("toId") UUID toId, @Param("toType") String toType, @Param("relationTypeGroup") String relationTypeGroup, @Param("relationType") String relationType, From 9ac6219882550ee6b138f6d10807096d313e7eda Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 14 Oct 2025 17:05:47 +0300 Subject: [PATCH 37/88] removed entityProfiles from config --- ...CalculatedFieldEntityMessageProcessor.java | 33 ++-- ...alculatedFieldManagerMessageProcessor.java | 143 ++++----------- ...tractCalculatedFieldProcessingService.java | 65 +++---- .../service/cf/CalculatedFieldCache.java | 3 +- .../cf/DefaultCalculatedFieldCache.java | 26 ++- .../DefaultCalculatedFieldQueueService.java | 31 +--- .../service/cf/ctx/state/ArgumentEntry.java | 4 + .../cf/ctx/state/CalculatedFieldCtx.java | 112 ++++-------- .../state/aggregation/function/new_agg.json | 60 +++++++ .../utils/CalculatedFieldArgumentUtils.java | 9 + ...tValuesAggregationCalculatedFieldTest.java | 167 ++++++++++-------- .../aggregation/CfAggTrigger.java | 104 ----------- ...gregationCalculatedFieldConfiguration.java | 19 +- 13 files changed, 297 insertions(+), 479 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 9dfde4d821..e253633727 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -171,8 +171,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM throw cfe; } throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); - } - } + } } public void process(CalculatedFieldArgumentResetMsg msg) throws CalculatedFieldException { log.debug("[{}] Processing CF argument reset msg.", entityId); @@ -523,22 +522,21 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { - return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getAggregationInputs(), data); + return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getRelatedEntityArguments(), data); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { - return mapToArguments(entityId, ctx.getLinkedAndDynamicArgs(entityId), ctx.getAggregationInputs(), data); + return mapToArguments(entityId, ctx.getLinkedAndDynamicArgs(entityId), ctx.getRelatedEntityArguments(), data); } - private Map mapToArguments(EntityId originator, Map argNames, Map aggArgNames, List data) { + private Map mapToArguments(EntityId originator, Map argNames, Map aggArgNames, List data) { Map arguments = new HashMap<>(); if (!aggArgNames.isEmpty()) { - for (Map.Entry entry : aggArgNames.entrySet()) { - for (TsKvProto item : data) { - ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); - if (key.equals(entry.getValue())) { - arguments.put(entry.getKey(), new AggSingleArgumentEntry(originator, item)); - } + for (TsKvProto item : data) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); + String argName = aggArgNames.get(key); + if (argName != null) { + arguments.put(argName, new AggSingleArgumentEntry(originator, item)); } } } @@ -560,17 +558,17 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List attrDataList) { - return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), ctx.getAggregationInputs(), scope, attrDataList); + return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), ctx.getRelatedEntityArguments(), scope, attrDataList); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { var argNames = ctx.getLinkedAndDynamicArgs(entityId); List geofencingArgumentNames = ctx.getLinkedEntityAndCurrentOwnerGeofencingArgumentNames(); - Map aggregationInputs = ctx.getAggregationInputs(); + Map aggregationInputs = ctx.getRelatedEntityArguments(); return mapToArguments(entityId, argNames, geofencingArgumentNames, aggregationInputs, scope, attrDataList); } - private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, Map aggArgNames, AttributeScopeProto scope, List attrDataList) { + private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, Map aggArgNames, AttributeScopeProto scope, List attrDataList) { Map arguments = new HashMap<>(); if (!argNames.isEmpty()) { for (AttributeValueProto item : attrDataList) { @@ -589,10 +587,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (!aggArgNames.isEmpty()) { for (AttributeValueProto item : attrDataList) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); - for (Map.Entry entry : aggArgNames.entrySet()) { - if (key.equals(entry.getValue())) { - arguments.put(entry.getKey(), new AggSingleArgumentEntry(entityId, item)); - } + String argName = aggArgNames.get(key); + if (argName != null) { + arguments.put(argName, new AggSingleArgumentEntry(entityId, item)); } } } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 3c1e21bcc8..620eee41ee 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -24,7 +24,6 @@ import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; import org.thingsboard.server.actors.calculatedField.EntityInitCalculatedFieldMsg.StateAction; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; @@ -32,20 +31,17 @@ import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; -import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationPathLevel; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.msg.CalculatedFieldStatePartitionRestoreMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldCacheInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; @@ -53,13 +49,11 @@ import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldStateService; @@ -76,8 +70,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ScheduledFuture; @@ -96,7 +88,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final Map> entityIdCalculatedFields = new HashMap<>(); private final Map> entityIdCalculatedFieldLinks = new HashMap<>(); private final Map> ownerEntities = new HashMap<>(); - private final Map cfTriggers = new HashMap<>(); + private final Map aggCalculatedFields = new HashMap<>(); private ScheduledFuture cfsReevaluationTask; private final CalculatedFieldProcessingService cfExecService; @@ -146,7 +138,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware cfsReevaluationTask.cancel(true); cfsReevaluationTask = null; } - cfTriggers.clear(); + aggCalculatedFields.clear(); ctx.stop(ctx.getSelf()); } @@ -252,19 +244,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } updateEntityOwner(entityId); - MultipleTbCallback callbackFor2 = new MultipleTbCallback(2, callback); - - // process aggregation cfs(in any) - List cfsRelatedToEntity = getCfsWithRelationToEntity(entityId, profileId); - if (!cfsRelatedToEntity.isEmpty()) { - MultipleTbCallback multiCallback = new MultipleTbCallback(cfsRelatedToEntity.size(), callbackFor2); - cfsRelatedToEntity.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> initRelatedEntity(id, entityId, ctx, cb)); - }); - } else { - callbackFor2.onSuccess(); - } - if (!isMyPartition(entityId, callback)) { return; } @@ -272,11 +251,11 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var profileIdFields = getCalculatedFieldsByEntityId(profileId); var fieldsCount = entityIdFields.size() + profileIdFields.size(); if (fieldsCount > 0) { - MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callbackFor2); + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); entityIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); profileIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); } else { - callbackFor2.onSuccess(); + callback.onSuccess(); } } @@ -286,35 +265,16 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (!isMyPartition(msg.getEntityId(), callback)) { return; } - MultipleTbCallback callbackFor2 = new MultipleTbCallback(2, callback); - - // process aggregation cfs(in any) - List oldCfsRelatedToEntity = getCfsWithRelationToEntity(msg.getEntityId(), msg.getOldProfileId()); - List newCfsRelatedToEntity = getCfsWithRelationToEntity(msg.getEntityId(), msg.getProfileId()); - var fieldsWithRelatedEntityCount = oldCfsRelatedToEntity.size() + newCfsRelatedToEntity.size(); - if (fieldsWithRelatedEntityCount > 0) { - MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsWithRelatedEntityCount, callbackFor2); - var entityId = msg.getEntityId(); - oldCfsRelatedToEntity.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> deleteRelatedEntity(id, entityId, cb)); - }); - newCfsRelatedToEntity.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, multiCallback, (id, cb) -> initRelatedEntity(id, entityId, ctx, cb)); - }); - } else { - callbackFor2.onSuccess(); - } - var oldProfileCfs = getCalculatedFieldsByEntityId(msg.getOldProfileId()); var newProfileCfs = getCalculatedFieldsByEntityId(msg.getProfileId()); var fieldsCount = oldProfileCfs.size() + newProfileCfs.size(); if (fieldsCount > 0) { - MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callbackFor2); + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); var entityId = msg.getEntityId(); oldProfileCfs.forEach(ctx -> deleteCfForEntity(entityId, ctx.getCfId(), multiCallback)); newProfileCfs.forEach(ctx -> initCfForEntity(entityId, ctx, StateAction.INIT, multiCallback)); } else { - callbackFor2.onSuccess(); + callback.onSuccess(); } } else if (msg.isOwnerChanged()) { onEntityOwnerChanged(msg, callback); @@ -334,10 +294,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware case CUSTOMER -> ownerEntities.remove(msg.getEntityId()); } ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); - - getCfsWithRelationToEntity(msg.getEntityId(), msg.getProfileId()).forEach(ctx -> { - applyToTargetCfEntityActors(ctx, callback, (id, cb) -> deleteRelatedEntity(id, msg.getEntityId(), cb)); - }); if (isMyPartition(msg.getEntityId(), callback)) { log.debug("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); @@ -365,9 +321,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware cfsByToIdOrItsProfileId.forEach(cf -> { var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - AggSource source = configuration.getSource(); - RelationPathLevel relation = source.getRelation(); - if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(fromIdProfile)) { + RelationPathLevel relation = configuration.getRelation(); + if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType())) { toIdMatches.add(cf); } }); @@ -386,9 +341,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware cfsByFromIdOrItsProfileId.forEach(cf -> { var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - AggSource source = configuration.getSource(); - RelationPathLevel relation = source.getRelation(); - if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(toIdProfile)) { + RelationPathLevel relation = configuration.getRelation(); + if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType())) { fromIdMatches.add(cf); } }); @@ -424,9 +378,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware cfsByToIdOrItsProfileId.forEach(cf -> { var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - AggSource source = configuration.getSource(); - RelationPathLevel relation = source.getRelation(); - if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(fromIdProfile)) { + RelationPathLevel relation = configuration.getRelation(); + if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType())) { toIdMatches.add(cf); } }); @@ -445,9 +398,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware cfsByFromIdOrItsProfileId.forEach(cf -> { var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - AggSource source = configuration.getSource(); - RelationPathLevel relation = source.getRelation(); - if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType()) && source.getEntityProfiles().contains(toIdProfile)) { + RelationPathLevel relation = configuration.getRelation(); + if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType())) { fromIdMatches.add(cf); } }); @@ -482,7 +434,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } calculatedFields.put(cf.getId(), cfCtx); if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + aggCalculatedFields.put(cf.getId(), cfCtx); } // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) @@ -515,8 +467,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(newCfCtx).eventEntity(newCfCtx.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(newCf.getId(), newCfCtx); - if (newCf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - cfTriggers.put(newCf.getId(), aggConfig.buildTrigger()); + if (newCf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + aggCalculatedFields.put(newCf.getId(), newCfCtx); } List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new CopyOnWriteArrayList<>(); @@ -559,7 +511,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onCfDeleted(ComponentLifecycleMsg msg, TbCallback callback) { var cfId = new CalculatedFieldId(msg.getEntityId().getId()); var cfCtx = calculatedFields.remove(cfId); // fixme wtf? why isn't ctx closed properly? - cfTriggers.remove(cfId); + aggCalculatedFields.remove(cfId); if (cfCtx == null) { log.debug("[{}] CF was already deleted [{}]", tenantId, cfId); callback.onSuccess(); @@ -614,21 +566,14 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private List filterAggregationCfs(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); - return cfTriggers.entrySet().stream() - .filter(entry -> aggMatches(entry.getValue(), msg.getProto())) - .map(Entry::getKey) - .map(calculatedFields::get) - .filter(Objects::nonNull) + return aggCalculatedFields.values().stream() + .filter(cf -> cf.relatedEntityMatches(msg.getProto())) .flatMap(cf -> findRelationsForCf(entityId, cf).stream()) .toList(); } - private List getCfsWithRelationToEntity(EntityId entityId, EntityId profileId) { - return cfTriggers.entrySet().stream() - .filter(entry -> entry.getValue().matchesProfile(profileId)) - .map(Entry::getKey) - .map(calculatedFields::get) - .filter(Objects::nonNull) + private List getCfsWithRelationToEntity(EntityId entityId) { + return aggCalculatedFields.values().stream() .filter(cf -> !findRelationsForCf(entityId, cf).isEmpty()) .toList(); } @@ -636,24 +581,19 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private List findRelationsForCf(EntityId entityId, CalculatedFieldCtx cf) { List result = new ArrayList<>(); if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration configuration) { - AggSource source = configuration.getSource(); - RelationPathLevel relation = source.getRelation(); - EntityId cfEntityId = cf.getEntityId(); - EntityId targetProfileId = isProfileEntity(cfEntityId.getEntityType()) - ? cfEntityId - : getProfileId(tenantId, cfEntityId); + RelationPathLevel relation = configuration.getRelation(); switch (relation.direction()) { case FROM -> { - List relationsByTo = relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), targetProfileId); - if (relationsByTo != null && !relationsByTo.isEmpty()) { - EntityRelation entityRelation = relationsByTo.get(0); // only one supported + List byToAndType = relationService.findByToAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); + if (byToAndType != null && !byToAndType.isEmpty()) { + EntityRelation entityRelation = byToAndType.get(0); // only one supported result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getFrom())); } } case TO -> { - List relationsByFrom = relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), targetProfileId); - if (relationsByFrom != null && !relationsByFrom.isEmpty()) { - for (EntityRelation entityRelation : relationsByFrom) { + List byFromAndType = relationService.findByFromAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); + if (byFromAndType != null && !byFromAndType.isEmpty()) { + for (EntityRelation entityRelation : byFromAndType) { if (entityRelation.getTo().equals(cf.getEntityId())) { result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getTo())); } @@ -665,25 +605,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return result; } - private boolean aggMatches(CfAggTrigger cfAggTrigger, CalculatedFieldTelemetryMsgProto proto) { - if (!proto.getTsDataList().isEmpty()) { - List updatedTelemetry = proto.getTsDataList().stream() - .map(ProtoUtils::fromProto) - .toList(); - return cfAggTrigger.matchesTimeSeries(updatedTelemetry); - } else if (!proto.getAttrDataList().isEmpty()) { - AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); - List updatedTelemetry = proto.getAttrDataList().stream() - .map(ProtoUtils::fromProto) - .toList(); - return cfAggTrigger.matchesAttributes(updatedTelemetry, scope); - } else if (!proto.getRemovedTsKeysList().isEmpty()) { - return cfAggTrigger.matchesTimeSeriesKeys(proto.getRemovedTsKeysList()); - } else { - return cfAggTrigger.matchesAttributesKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); - } - } - public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsg msg) { EntityId sourceEntityId = msg.getEntityId(); log.debug("Received linked telemetry msg from entity [{}]", sourceEntityId); @@ -889,8 +810,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(cf.getId(), cfCtx); - if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + aggCalculatedFields.put(cf.getId(), cfCtx); } // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index db15fc1dc8..c30a12a8f3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -27,7 +27,6 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; -import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -39,7 +38,7 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; +import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.attributes.AttributesService; @@ -49,13 +48,11 @@ import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -65,6 +62,7 @@ import static org.thingsboard.server.common.data.cf.configuration.geofencing.Ent import static org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates.ENTITY_ID_LONGITUDE_ARGUMENT_KEY; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultAttributeEntry; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultKvEntry; +import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformAggSingleArgument; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformSingleValueArgument; @Data @@ -123,30 +121,19 @@ public abstract class AbstractCalculatedFieldProcessingService { return resolveOwnerArgument(tenantId, entityId); } - private ListenableFuture> resolveRelatedEntities(TenantId tenantId, EntityId entityId, AggSource aggSource) { - RelationPathLevel relation = aggSource.getRelation(); + private ListenableFuture> resolveRelatedEntities(TenantId tenantId, EntityId entityId, RelationPathLevel relation) { + ListenableFuture> relationsFut = relationService.findByRelationPathQueryAsync(tenantId, new EntityRelationPathQuery(entityId, List.of(relation))); - List>> relationListsFut = new ArrayList<>(); - if (aggSource.getEntityProfiles().isEmpty()) { - relationListsFut.add(relationService.findByProfileEntityRelationPathQueryAsync(tenantId, new ProfileEntityRelationPathQuery(entityId, relation, null))); - } else { - aggSource.getEntityProfiles().forEach(profile -> relationListsFut.add(relationService.findByProfileEntityRelationPathQueryAsync(tenantId, new ProfileEntityRelationPathQuery(entityId, relation, profile)))); - } - - return Futures.transform(Futures.allAsList(relationListsFut), relationLists -> { - if (relationLists == null) { + return Futures.transform(relationsFut, relations -> { + if (relations == null) { return new ArrayList<>(); } - List allRelations = relationLists.stream() - .filter(Objects::nonNull) - .flatMap(List::stream) - .toList(); return switch (relation.direction()) { - case FROM -> allRelations.stream() + case FROM -> relations.stream() .map(EntityRelation::getTo) .toList(); - case TO -> allRelations.isEmpty() ? List.of() : List.of(allRelations.get(0).getFrom()); + case TO -> relations.isEmpty() ? List.of() : List.of(relations.get(0).getFrom()); }; }, calculatedFieldCallbackExecutor); } @@ -193,28 +180,26 @@ public abstract class AbstractCalculatedFieldProcessingService { protected Map> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); - ListenableFuture> relatedEntities = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getSource()); + ListenableFuture> relatedEntitiesFut = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getRelation()); - Map> futures = new HashMap<>(); - aggConfig.getInputs().forEach((key, refKey) -> { - Argument argument = new Argument(); - argument.setRefEntityKey(refKey); - futures.put(key, Futures.transformAsync(relatedEntities, entityIds -> fetchAggArgumentEntry(ctx.getTenantId(), entityIds, argument, System.currentTimeMillis()), MoreExecutors.directExecutor())); - }); - return futures; + return aggConfig.getArguments().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> Futures.transformAsync(relatedEntitiesFut, relatedEntities -> fetchAggArgumentEntry(ctx.getTenantId(), relatedEntities, entry.getValue(), ts), MoreExecutors.directExecutor()) + )); } protected ListenableFuture> fetchEntityAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); - Map> futures = new HashMap<>(); - aggConfig.getInputs().forEach((key, refKey) -> { - Argument argument = new Argument(); - argument.setRefEntityKey(refKey); - ListenableFuture argEntryFut = fetchSingleAggArgumentEntry(ctx.getTenantId(), entityId, argument, ts); - futures.put(key, argEntryFut); - }); - return Futures.whenAllComplete(futures.values()) - .call(() -> resolveArgumentFutures(futures), + + Map> argsFutures = aggConfig.getArguments().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> fetchSingleAggArgumentEntry(ctx.getTenantId(), entityId, entry.getValue(), ts) + )); + + return Futures.whenAllComplete(argsFutures.values()) + .call(() -> resolveArgumentFutures(argsFutures), MoreExecutors.directExecutor()); } @@ -347,7 +332,7 @@ public abstract class AbstractCalculatedFieldProcessingService { return Futures.transform(attributeOptFuture, attrOpt -> { log.debug("[{}][{}] Fetched attribute for key {}: {}", tenantId, entityId, argument.getRefEntityKey(), attrOpt); AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, 0L)); - return new AggSingleArgumentEntry(entityId, attributeKvEntry); + return transformAggSingleArgument(entityId, Optional.of(attributeKvEntry)); }, calculatedFieldCallbackExecutor); } @@ -359,7 +344,7 @@ public abstract class AbstractCalculatedFieldProcessingService { result -> { log.debug("[{}][{}] Fetched latest timeseries {}: {}", tenantId, entityId, key, result); Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(defaultTs, createDefaultKvEntry(argument), 0L))); - return new AggSingleArgumentEntry(entityId, tsKvEntry.get()); + return transformAggSingleArgument(entityId, tsKvEntry); }, calculatedFieldCallbackExecutor); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index e32ca42f9c..27e989de70 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -39,7 +38,7 @@ public interface CalculatedFieldCache { List getCalculatedFieldCtxsByEntityId(EntityId entityId); - List getCalculatedFieldCtxsByTrigger(EntityId profileId, Predicate cfAggFilter); + List getAggCalculatedFieldCtxsByFilter(Predicate relatedEntityFilter); boolean hasCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index ae8238fb8a..918d71e326 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -43,8 +42,6 @@ import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -52,7 +49,6 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Predicate; -import java.util.stream.Collectors; @Service @Slf4j @@ -73,7 +69,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); - private final ConcurrentMap cfTriggers = new ConcurrentHashMap<>(); + private final ConcurrentMap aggCalculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> ownerEntities = new ConcurrentHashMap<>(); @@ -87,8 +83,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { cfs.forEach(cf -> { if (cf != null) { calculatedFields.putIfAbsent(cf.getId(), cf); - if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - cfTriggers.put(cf.getId(), aggConfig.buildTrigger()); + if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + aggCalculatedFields.put(cf.getId(), cf); } } }); @@ -156,13 +152,11 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } @Override - public List getCalculatedFieldCtxsByTrigger(EntityId profileId, Predicate cfAggFilter) { - return cfTriggers.entrySet().stream() - .filter(entry -> entry.getValue().matches(profileId, cfAggFilter)) - .map(Map.Entry::getKey) + public List getAggCalculatedFieldCtxsByFilter(Predicate relatedEntityFilter) { + return aggCalculatedFields.keySet().stream() .map(this::getCalculatedFieldCtx) - .filter(Objects::nonNull) - .collect(Collectors.toList()); + .filter(relatedEntityFilter) + .toList(); } @Override @@ -206,8 +200,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new CopyOnWriteArrayList<>()).add(calculatedField); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); - if (configuration instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - cfTriggers.put(calculatedField.getId(), aggConfig.buildTrigger()); + if (configuration instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + aggCalculatedFields.put(calculatedField.getId(), calculatedField); } calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); @@ -240,7 +234,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); entityIdCalculatedFieldLinks.forEach((entityId, calculatedFieldLinks) -> calculatedFieldLinks.removeIf(link -> link.getCalculatedFieldId().equals(calculatedFieldId))); log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); - cfTriggers.remove(calculatedFieldId); + aggCalculatedFields.remove(calculatedFieldId); log.debug("[{}] evict calculated field from cached triggers: {}", calculatedFieldId, oldCalculatedField); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 44cb2aaee4..37c7b2ef6c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -27,8 +27,6 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; -import org.thingsboard.server.common.data.cf.configuration.aggregation.CfAggTrigger; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -89,7 +87,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matches(entries), cf -> cf.linkMatches(entityId, entries), cf -> cf.dynamicSourceMatches(request.getEntries()), - cfTrigger -> cfTrigger.matchesTimeSeries(entries), + cf -> cf.relatedEntityMatches(entries), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -108,7 +106,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matches(entries, scope), cf -> cf.linkMatches(entityId, entries, scope), cf -> cf.dynamicSourceMatches(request.getEntries(), request.getScope()), - cfTrigger -> cfTrigger.matchesAttributes(entries, scope), + cf -> cf.relatedEntityMatches(entries, scope), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -126,7 +124,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matchesKeys(result, scope), cf -> cf.linkMatchesAttrKeys(entityId, result, scope), cf -> cf.matchesDynamicSourceKeys(result, request.getScope()), - cfTrigger -> cfTrigger.matchesAttributesKeys(result, scope), + cf -> cf.matchesRelatedEntityKeys(result, scope), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -138,7 +136,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS cf -> cf.matchesKeys(result), cf -> cf.linkMatchesTsKeys(entityId, result), cf -> cf.matchesDynamicSourceKeys(result), - cfTrigger -> cfTrigger.matchesTimeSeriesKeys(result), + cf -> cf.matchesRelatedEntityKeys(result), () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @@ -146,12 +144,12 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS Predicate mainEntityFilter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter, - Predicate cfAggKeysFilter, + Predicate relatedEntityFilter, Supplier msg, FutureCallback callback) { if (EntityType.TENANT.equals(entityId.getEntityType())) { tenantId = (TenantId) entityId; } - boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter, dynamicSourceFilter, cfAggKeysFilter); + boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter, dynamicSourceFilter, relatedEntityFilter); if (send) { ToCalculatedFieldMsg calculatedFieldMsg = msg.get(); clusterService.pushMsgToCalculatedFields(tenantId, entityId, calculatedFieldMsg, wrap(callback)); @@ -162,7 +160,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } } - private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter, Predicate cfAggKeysFilter) { + private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter, Predicate dynamicSourceFilter, Predicate relatedEntityFilter) { if (!CalculatedField.SUPPORTED_REFERENCED_ENTITIES.contains(entityId.getEntityType())) { return false; } @@ -189,26 +187,19 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS } } - List cfCtxs = calculatedFieldCache.getCalculatedFieldCtxsByTrigger(calculatedFieldCache.getProfileId(tenantId, entityId), cfAggKeysFilter); + List cfCtxs = calculatedFieldCache.getAggCalculatedFieldCtxsByFilter(relatedEntityFilter); for (CalculatedFieldCtx cfCtx : cfCtxs) { - EntityId cfEntityId = cfCtx.getEntityId(); if (cfCtx.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - AggSource source = aggConfig.getSource(); - RelationPathLevel relation = source.getRelation(); - EntityId cfEntityProfileId = isProfileEntity(cfEntityId.getEntityType()) - ? cfEntityId - : calculatedFieldCache.getProfileId(tenantId, cfEntityId); + RelationPathLevel relation = aggConfig.getRelation(); switch (relation.direction()) { case FROM -> { List byToAndType = relationService.findByToAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); -// List byTo = relationService.findByToAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); if (!byToAndType.isEmpty()) { return true; } } case TO -> { List byFromAndType = relationService.findByFromAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); -// List byFrom = relationService.findByFromAndTypeAndEntityProfile(tenantId, entityId, relation.relationType(), cfEntityProfileId); if (!byFromAndType.isEmpty()) { return true; } @@ -307,10 +298,6 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS return telemetryMsg; } - private boolean isProfileEntity(EntityType entityType) { - return EntityType.DEVICE_PROFILE.equals(entityType) || EntityType.ASSET_PROFILE.equals(entityType); - } - private static TbQueueCallback wrap(FutureCallback callback) { if (callback != null) { return new FutureCallbackWrapper(callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index f22b10a9c1..4e3d00ee62 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -74,4 +74,8 @@ public interface ArgumentEntry { return new AggArgumentEntry(entityIdkvEntryMap, false); } + static ArgumentEntry createAggSingleArgument(EntityId entityId, KvEntry kvEntry) { + return new AggSingleArgumentEntry(entityId, kvEntry); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 963ec3f06f..f8b9f5b665 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -83,8 +83,7 @@ public class CalculatedFieldCtx { private final Map mainEntityArguments; private final Map> linkedEntityArguments; private final Map dynamicEntityArguments; - private final List aggInputs; - private final Map aggregationInputs; + private final Map relatedEntityArguments; private final List argNames; private Output output; private String expression; @@ -123,11 +122,10 @@ public class CalculatedFieldCtx { this.mainEntityArguments = new HashMap<>(); this.linkedEntityArguments = new HashMap<>(); this.dynamicEntityArguments = new HashMap<>(); + this.relatedEntityArguments = new HashMap<>(); this.argNames = new ArrayList<>(); this.mainEntityGeofencingArgumentNames = new ArrayList<>(); this.linkedEntityAndCurrentOwnerGeofencingArgumentNames = new ArrayList<>(); - this.aggInputs = new ArrayList<>(); - this.aggregationInputs = new HashMap<>(); this.output = calculatedField.getConfiguration().getOutput(); if (calculatedField.getConfiguration() instanceof ArgumentsBasedCalculatedFieldConfiguration argBasedConfig) { this.arguments.putAll(argBasedConfig.getArguments()); @@ -135,6 +133,10 @@ public class CalculatedFieldCtx { var refId = entry.getValue().getRefEntityId(); var refKey = entry.getValue().getRefEntityKey(); if (refId == null) { + if (CalculatedFieldType.LATEST_VALUES_AGGREGATION.equals(cfType)) { + relatedEntityArguments.put(refKey, entry.getKey()); + continue; + } if (entry.getValue().hasRelationQuerySource()) { relationQueryDynamicArguments = true; continue; @@ -172,9 +174,6 @@ public class CalculatedFieldCtx { } this.requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - aggInputs.addAll(aggConfig.getInputs().values()); - aggregationInputs.putAll(aggConfig.getInputs()); - this.argNames.addAll(aggConfig.getInputs().keySet()); this.scheduledUpdateIntervalMillis = aggConfig.getDeduplicationIntervalMillis(); } this.systemContext = systemContext; @@ -475,41 +474,57 @@ public class CalculatedFieldCtx { return map != null && matchesTimeSeriesKeys(map, keys); } - public boolean dynamicSourceMatches(CalculatedFieldTelemetryMsgProto proto) { + public boolean relatedEntityMatches(List values) { + return matchesTimeSeries(relatedEntityArguments, values); + } + + public boolean relatedEntityMatches(List values, AttributeScope scope) { + return matchesAttributes(relatedEntityArguments, values, scope); + } + + public boolean matchesRelatedEntityKeys(List keys, AttributeScope scope) { + return matchesAttributesKeys(relatedEntityArguments, keys, scope); + } + + public boolean matchesRelatedEntityKeys(List keys) { + return matchesTimeSeriesKeys(relatedEntityArguments, keys); + } + + public boolean relatedEntityMatches(CalculatedFieldTelemetryMsgProto proto) { if (!proto.getTsDataList().isEmpty()) { List updatedTelemetry = proto.getTsDataList().stream() .map(ProtoUtils::fromProto) .toList(); - return dynamicSourceMatches(updatedTelemetry); + return relatedEntityMatches(updatedTelemetry); } else if (!proto.getAttrDataList().isEmpty()) { AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); List updatedTelemetry = proto.getAttrDataList().stream() .map(ProtoUtils::fromProto) .toList(); - return dynamicSourceMatches(updatedTelemetry, scope); + return relatedEntityMatches(updatedTelemetry, scope); } else if (!proto.getRemovedTsKeysList().isEmpty()) { - return matchesDynamicSourceKeys(proto.getRemovedTsKeysList()); + return matchesRelatedEntityKeys(proto.getRemovedTsKeysList()); } else { - return matchesDynamicSourceKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); + return matchesRelatedEntityKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); } } - public boolean aggMatches(CalculatedFieldTelemetryMsgProto proto) { + public boolean dynamicSourceMatches(CalculatedFieldTelemetryMsgProto proto) { if (!proto.getTsDataList().isEmpty()) { List updatedTelemetry = proto.getTsDataList().stream() .map(ProtoUtils::fromProto) .toList(); - return matchesAggTimeSeries(updatedTelemetry); + return dynamicSourceMatches(updatedTelemetry); } else if (!proto.getAttrDataList().isEmpty()) { AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); List updatedTelemetry = proto.getAttrDataList().stream() .map(ProtoUtils::fromProto) .toList(); - return matchesAggAttributes(updatedTelemetry, scope); + return dynamicSourceMatches(updatedTelemetry, scope); } else if (!proto.getRemovedTsKeysList().isEmpty()) { - return matchesAggKeys(proto.getRemovedTsKeysList()); + return matchesDynamicSourceKeys(proto.getRemovedTsKeysList()); } else { - return matchesAggAttributesKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); + return matchesDynamicSourceKeys(proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); } } @@ -544,67 +559,6 @@ public class CalculatedFieldCtx { return argNames; } - public boolean matchesAggKeys(List values) { - if (aggInputs.isEmpty() || values.isEmpty()) { - return false; - } - - for (String key : values) { - ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); - if (aggInputs.contains(latestKey)) { - return true; - } - } - - return false; - } - - public boolean matchesAggTimeSeries(List values) { - if (aggInputs.isEmpty() || values.isEmpty()) { - return false; - } - - for (TsKvEntry tsKvEntry : values) { - ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKvEntry.getKey(), ArgumentType.TS_LATEST, null); - if (aggInputs.contains(latestKey)) { - return true; - } - } - - return false; - } - - public boolean matchesAggAttributesKeys(List keys, AttributeScope scope) { - if (keys == null || keys.isEmpty()) { - return false; - } - - for (String key : keys) { - ReferencedEntityKey attrKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); - if (aggInputs.contains(attrKey)) { - return true; - } - } - - return false; - } - - public boolean matchesAggAttributes(List keys, AttributeScope scope) { - if (keys == null || keys.isEmpty()) { - return false; - } - - for (AttributeKvEntry attributeKvEntry : keys) { - ReferencedEntityKey attrKey = new ReferencedEntityKey(attributeKvEntry.getKey(), ArgumentType.ATTRIBUTE, scope); - if (aggInputs.contains(attrKey)) { - return true; - } - } - - return false; - } - - public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); } @@ -662,7 +616,7 @@ public class CalculatedFieldCtx { private boolean hasLatestValuesAggregationConfigurationChanges(CalculatedFieldCtx other) { if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig && other.calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig) { - return !thisConfig.getInputs().equals(otherConfig.getInputs()) || !thisConfig.getSource().equals(otherConfig.getSource()); + return !thisConfig.getArguments().equals(otherConfig.getArguments()) || !thisConfig.getRelation().equals(otherConfig.getRelation()); } return false; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json new file mode 100644 index 0000000000..32cd053d41 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json @@ -0,0 +1,60 @@ +{ + "type": "LATEST_VALUES_AGGREGATION", + "name": "Occupied spaces", + "debugSettings": { + "failuresEnabled": true, + "allEnabled": true, + "allEnabledUntil": 1769907492297 + }, + "entityId": { + "entityType": "ASSET_PROFILE", + "id": "2b759c60-a8f4-11f0-be29-7fa922118588" + }, + "configuration": { + "type": "LATEST_VALUES_AGGREGATION", + "relation": { + "direction": "FROM", + "relationType": "Contains" + }, + "arguments": { + "oc": { + "refEntityKey": { + "key": "occupied", + "type": "TS_LATEST" + }, + "defaultValue": "false" + } + }, + "deduplicationIntervalMillis": 20000, + "metrics": { + "totalSpaces": { + "function": "COUNT", + "input": { + "type": "function", + "function" : "return 1;" + } + }, + "occupiedSpaces": { + "function": "COUNT", + "filter": "return oc == true", + "input": { + "type": "key", + "key" : "oc" + } + }, + "freeSpaces": { + "function": "COUNT", + "filter": "return oc == false", + "input": { + "type": "key", + "key" : "oc" + } + } + }, + "output": { + "type": "TIME_SERIES", + "decimals": 2 + }, + "useLatestTsFromInputs": "true" + } +} diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index 6f6fabbea2..6485508602 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -34,6 +34,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -54,6 +55,14 @@ public class CalculatedFieldArgumentUtils { } } + public static ArgumentEntry transformAggSingleArgument(EntityId entityId, Optional kvEntry) { + if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { + return ArgumentEntry.createAggSingleArgument(entityId, kvEntry.get()); + } else { + return new AggSingleArgumentEntry(); + } + } + public static KvEntry createDefaultKvEntry(Argument argument) { String key = argument.getRefEntityKey().getKey(); String defaultValue = argument.getDefaultValue(); diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java index 2ed3554666..cd9c1c578b 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -27,8 +27,8 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; @@ -36,7 +36,6 @@ import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFuncti import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; -import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration; @@ -53,9 +52,7 @@ import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.service.DaoSqlTest; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -77,8 +74,6 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll private AssetProfile assetProfile; private Asset asset; - private CalculatedField calculatedField; - private long deduplicationInterval = 10000; @Before @@ -111,10 +106,6 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset.getId(), device1.getId(), "Contains"); createEntityRelation(asset.getId(), device2.getId(), "Contains"); - - calculatedField = createOccupancyCF("Occupied spaces", asset.getId(), List.of(deviceProfile.getId())); - - checkInitialCalculation(); } @After @@ -124,8 +115,33 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll deleteTenant(savedTenant.getId()); } + @Test + public void testNoTelemetryOnDevices_checkDefaultValueUsed() throws Exception { + Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + Device device3 = createDevice("Device 3", "1234567890333"); + Device device4 = createDevice("Device 4", "1234567890444"); + + createEntityRelation(asset2.getId(), device3.getId(), "Contains"); + createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + + createOccupancyCF("Occupied spaces", asset2.getId()); + + await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode occupancy = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancy).isNotNull(); + assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + }); + } + @Test public void testUpdateTelemetry_checkMetricsCalculation() throws Exception { + createOccupancyCF("Occupied spaces", asset.getId()); + checkInitialCalculation(); + postTelemetry(device1.getId(), "{\"occupied\":false}"); await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) @@ -141,6 +157,9 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll @Test public void testUpdateTelemetry_checkMetricsCalculationNotExecutedUntilDeduplicationInterval() throws Exception { + createOccupancyCF("Occupied spaces", asset.getId()); + checkInitialCalculation(); + postTelemetry(device1.getId(), "{\"occupied\":false}"); await().alias("update telemetry -> no changes").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) @@ -161,35 +180,33 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll } @Test - public void testChangeProfile_checkMetricsCalculation() throws Exception { - DeviceProfile deviceProfile2 = doPost("/api/deviceProfile", createDeviceProfile("Device Profile 2"), DeviceProfile.class); - device1.setDeviceProfileId(deviceProfile2.getId()); - device1 = doPost("/api/device?accessToken=" + accessToken1, device1, Device.class); + public void testCreateRelation_checkMetricsCalculation() throws Exception { + createOccupancyCF("Occupied spaces", asset.getId()); + checkInitialCalculation(); - postTelemetry(device1.getId(), "{\"occupied\":false}"); + Device device3 = createDevice("Device 3", deviceProfile.getId(), "1234567890333"); + + postTelemetry(device3.getId(), "{\"occupied\":true}"); - await().alias("change profile and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + createEntityRelation(asset.getId(), device3.getId(), "Contains"); + + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); assertThat(occupancy).isNotNull(); assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("3"); }); } @Test - public void testAddEntityToProfile_checkMetricsCalculation() throws Exception { - Device device3 = createDevice("Device 3", deviceProfile.getId(), "1234567890333"); - - postTelemetry(device3.getId(), "{\"occupied\":true}"); - - await().alias("add entity to profile and no calculation (there is no relation between device and asset)").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) - .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) - .untilAsserted(this::checkInitialCalculationValues); + public void testDeleteRelation_checkMetricsCalculation() throws Exception { + createOccupancyCF("Occupied spaces", asset.getId()); + checkInitialCalculation(); - createEntityRelation(asset.getId(), device3.getId(), "Contains"); + deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) @@ -197,8 +214,8 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); assertThat(occupancy).isNotNull(); assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("3"); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); }); } @@ -213,22 +230,22 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset2.getId(), device3.getId(), "Contains"); createEntityRelation(asset2.getId(), device4.getId(), "Contains"); - CalculatedField calculatedField2 = createOccupancyCF("Occupied spaces 2", assetProfile.getId(), List.of(deviceProfile.getId())); + createOccupancyCF("Occupied spaces 2", assetProfile.getId()); await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); - - ObjectNode occupancy2 = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy2).isNotNull(); - assertThat(occupancy2.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancy2.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + ObjectNode occupancyAsset1 = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancyAsset1).isNotNull(); + assertThat(occupancyAsset1.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancyAsset1.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); + assertThat(occupancyAsset1.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + + ObjectNode occupancyAsset2 = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); + assertThat(occupancyAsset2).isNotNull(); + assertThat(occupancyAsset2.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancyAsset2.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); + assertThat(occupancyAsset2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); }); postTelemetry(device3.getId(), "{\"occupied\":true}"); @@ -243,23 +260,26 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll assertThat(occupancy2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); }); } - - - @Test - public void testDeleteRelation_checkMetricsCalculation() throws Exception { - deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); - - await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) - .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) - .untilAsserted(() -> { - ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); - }); - } - +// +// @Test +// public void testChangeProfile_checkMetricsCalculation() throws Exception { +// DeviceProfile deviceProfile2 = doPost("/api/deviceProfile", createDeviceProfile("Device Profile 2"), DeviceProfile.class); +// device1.setDeviceProfileId(deviceProfile2.getId()); +// device1 = doPost("/api/device?accessToken=" + accessToken1, device1, Device.class); +// +// postTelemetry(device1.getId(), "{\"occupied\":false}"); +// +// await().alias("change profile and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) +// .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) +// .untilAsserted(() -> { +// ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); +// assertThat(occupancy).isNotNull(); +// assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); +// assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); +// assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); +// }); +// } +// // @Test // public void testCfWithoutTargetProfileSpecified_checkMetricsCalculation() throws Exception { // Device device3 = createDevice("Device 3", "1234567890333"); @@ -296,18 +316,24 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); } - private CalculatedField createOccupancyCF(String name, EntityId entityId, List profiles) { + private CalculatedField createOccupancyCF(String name, EntityId entityId) { + Map arguments = new HashMap<>(); + Argument argument = new Argument(); + argument.setRefEntityKey(new ReferencedEntityKey("occupied", ArgumentType.TS_LATEST, null)); + argument.setDefaultValue("false"); + arguments.put("oc", argument); + Map aggMetrics = new HashMap<>(); AggMetric freeSpaces = new AggMetric(); freeSpaces.setFunction(AggFunction.COUNT); - freeSpaces.setFilter("return oc == false"); + freeSpaces.setFilter("return oc == false;"); freeSpaces.setInput(new AggKeyInput("oc")); aggMetrics.put("freeSpaces", freeSpaces); AggMetric occupiedSpaces = new AggMetric(); occupiedSpaces.setFunction(AggFunction.COUNT); - occupiedSpaces.setFilter("return oc == true"); + occupiedSpaces.setFilter("return oc == true;"); occupiedSpaces.setInput(new AggKeyInput("oc")); aggMetrics.put("occupiedSpaces", occupiedSpaces); @@ -320,23 +346,16 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll output.setType(OutputType.TIME_SERIES); return createAggCf(name, entityId, - buildSource(EntitySearchDirection.FROM, "Contains", profiles), - Map.of("oc", new ReferencedEntityKey("occupied", ArgumentType.TS_LATEST, null)), + new RelationPathLevel(EntitySearchDirection.FROM, "Contains"), + arguments, aggMetrics, output); } - private AggSource buildSource(EntitySearchDirection direction, String relationType, List profiles) { - AggSource source = new AggSource(); - source.setRelation(new RelationPathLevel(direction, relationType)); - source.setEntityProfiles(profiles); - return source; - } - private CalculatedField createAggCf(String name, EntityId entityId, - AggSource aggSource, - Map inputs, + RelationPathLevel relation, + Map inputs, Map metrics, Output output) { CalculatedField calculatedField = new CalculatedField(); @@ -345,8 +364,8 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll calculatedField.setType(CalculatedFieldType.LATEST_VALUES_AGGREGATION); LatestValuesAggregationCalculatedFieldConfiguration configuration = new LatestValuesAggregationCalculatedFieldConfiguration(); - configuration.setSource(aggSource); - configuration.setInputs(inputs); + configuration.setRelation(relation); + configuration.setArguments(inputs); configuration.setDeduplicationIntervalMillis(deduplicationInterval); configuration.setMetrics(metrics); configuration.setOutput(output); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java deleted file mode 100644 index 65d545bc6e..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/CfAggTrigger.java +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.cf.configuration.aggregation; - -import lombok.Builder; -import lombok.Data; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; - -import java.util.List; -import java.util.function.Predicate; - -@Data -@Builder -public class CfAggTrigger { - - private List entityProfiles; - private List inputs; - - public boolean matches(EntityId profileId, Predicate cfAggTrigger) { - if (matchesProfile(profileId)) { - return cfAggTrigger.test(this); - } - return false; - } - - public boolean matchesProfile(EntityId profileId) { - return entityProfiles.isEmpty() || entityProfiles.contains(profileId); - } - - public boolean matchesTimeSeries(List telemetry) { - if (telemetry == null || telemetry.isEmpty()) { - return false; - } - for (TsKvEntry tsKvEntry : telemetry) { - ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKvEntry.getKey(), ArgumentType.TS_LATEST, null); - if (inputs.contains(latestKey)) { - return true; - } - } - return false; - } - - public boolean matchesAttributes(List attributes, AttributeScope scope) { - if (attributes == null || attributes.isEmpty()) { - return false; - } - for (AttributeKvEntry attributeKvEntry : attributes) { - ReferencedEntityKey latestKey = new ReferencedEntityKey(attributeKvEntry.getKey(), ArgumentType.ATTRIBUTE, scope); - if (inputs.contains(latestKey)) { - return true; - } - } - return false; - } - - public boolean matchesTimeSeriesKeys(List telemetry) { - if (telemetry == null || telemetry.isEmpty()) { - return false; - } - - for (String key : telemetry) { - ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); - if (inputs.contains(latestKey)) { - return true; - } - } - - return false; - } - - public boolean matchesAttributesKeys(List attributes, AttributeScope scope) { - if (attributes == null || attributes.isEmpty()) { - return false; - } - - for (String key : attributes) { - ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); - if (inputs.contains(latestKey)) { - return true; - } - } - - return false; - } - -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java index 2760e1855b..89f7e9a339 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java @@ -17,18 +17,18 @@ package org.thingsboard.server.common.data.cf.configuration.aggregation; import lombok.Data; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentsBasedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.relation.RelationPathLevel; -import java.util.List; import java.util.Map; @Data -public class LatestValuesAggregationCalculatedFieldConfiguration implements CalculatedFieldConfiguration { +public class LatestValuesAggregationCalculatedFieldConfiguration implements ArgumentsBasedCalculatedFieldConfiguration { - private AggSource source; - private Map inputs; + private RelationPathLevel relation; + private Map arguments; private long deduplicationIntervalMillis; private Map metrics; private Output output; @@ -42,11 +42,4 @@ public class LatestValuesAggregationCalculatedFieldConfiguration implements Calc public void validate() { } - public CfAggTrigger buildTrigger() { - return CfAggTrigger.builder() - .inputs(List.copyOf(inputs.values())) - .entityProfiles(List.copyOf(source.getEntityProfiles())) - .build(); - } - } From 83d866c6baf9d6692316457cedfbd61132ff8757 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Tue, 14 Oct 2025 18:20:51 +0300 Subject: [PATCH 38/88] lwm2m: bootstrap new: update tests-6 --- .../lwm2m/AbstractLwM2MIntegrationTest.java | 45 ++++---- .../lwm2m/client/LwM2MTestClient.java | 101 ++++++++++-------- .../AbstractSecurityLwM2MIntegrationTest.java | 17 ++- .../NoSecLwM2MIntegrationBSNoTriggerTest.java | 4 +- .../security/sql/PskLwm2mIntegrationTest.java | 9 +- .../sql/X509_NoTrustLwM2MIntegrationTest.java | 3 +- .../sql/X509_TrustLwM2MIntegrationTest.java | 6 +- .../lwm2m/Lwm2mServerIdentifier.java | 46 +++----- .../bootstrap/LwM2MServerSecurityConfig.java | 2 +- .../secure/LwM2MBootstrapConfig.java | 4 +- ...LwM2MBootstrapConfigStoreTaskProvider.java | 46 ++++---- .../store/LwM2MConfigurationChecker.java | 13 +-- .../uplink/DefaultLwM2mUplinkMsgHandler.java | 3 +- .../validator/DeviceProfileDataValidator.java | 22 ++-- .../DeviceProfileDataValidatorTest.java | 7 +- 15 files changed, 162 insertions(+), 166 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java index 338cf166ae..226ef0dc57 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java @@ -33,12 +33,10 @@ import org.eclipse.leshan.server.registration.Registration; import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.jupiter.api.TestInstance; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.http.HttpStatus; -import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.TestPropertySource; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; @@ -86,6 +84,7 @@ import org.thingsboard.server.transport.lwm2m.server.client.ResourceUpdateResult import org.thingsboard.server.transport.lwm2m.server.uplink.DefaultLwM2mUplinkMsgHandler; import org.thingsboard.server.transport.lwm2m.server.uplink.LwM2mUplinkMsgHandler; +import java.io.IOException; import java.net.ServerSocket; import java.util.ArrayList; import java.util.Arrays; @@ -124,8 +123,6 @@ import static org.thingsboard.server.transport.lwm2m.ota.AbstractOtaLwM2MIntegra @Slf4j @DaoSqlTest -//@TestInstance(TestInstance.Lifecycle.PER_CLASS) -//@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) @TestPropertySource(properties = { "transport.lwm2m.enabled=true" }) @@ -149,8 +146,6 @@ public abstract class AbstractLwM2MIntegrationTest extends AbstractTransportInte public static final String host = "localhost"; public static final String hostBs = "localhost"; public static final Integer shortServerId = 123; - public static final Integer shortServerIdBs0 = 0; - public static final int serverId = 1; public static final String COAP = "coap://"; public static final String COAPS = "coaps://"; @@ -320,17 +315,10 @@ public abstract class AbstractLwM2MIntegrationTest extends AbstractTransportInte @After public void after() throws Exception { - this.clientDestroy(true); - + this.clientDestroy(); if (executor != null && !executor.isShutdown()) { executor.shutdownNow(); - if (!executor.awaitTermination(2, TimeUnit.SECONDS)) { - log.warn("Executor did not terminate cleanly, forcing GC"); - } } - Thread.sleep(300); - System.gc(); - log.warn("Test lwm2m after completed: {}", this.getClass().getSimpleName()); } private void init() throws Exception { @@ -574,7 +562,7 @@ public abstract class AbstractLwM2MIntegrationTest extends AbstractTransportInte public void createNewClient(Security security, Security securityBs, boolean isRpc, String endpoint, Integer clientDtlsCidLength, boolean queueMode, String deviceIdStr, Integer value3_0_9) throws Exception { - this.clientDestroy(false); + this.clientDestroy(); lwM2MTestClient = new LwM2MTestClient(this.executor, endpoint, resources); try (ServerSocket socket = new ServerSocket(0)) { @@ -661,13 +649,30 @@ public abstract class AbstractLwM2MIntegrationTest extends AbstractTransportInte } - private void clientDestroy(boolean isAfter) { + private void clientDestroy() { try { if (lwM2MTestClient != null && lwM2MTestClient.getLeshanClient() != null) { - if (isAfter) { - sendObserveCancelAllWithAwait(lwM2MTestClient.getDeviceIdStr()); - awaitDeleteDevice(lwM2MTestClient.getDeviceIdStr()); + boolean serverAlive = false; + for (int port = AbstractLwM2MIntegrationTest.port; port <= securityPortBs; port++) { + try (ServerSocket socket = new ServerSocket(port)) { + log.info("Port {} is free.", port); + } catch (IOException e) { + log.debug("Port {} is busy — CoAP server still active.", port); + serverAlive = true; + break; + } } + if (serverAlive) { + try { + sendObserveCancelAllWithAwait(lwM2MTestClient.getDeviceIdStr()); + awaitDeleteDevice(lwM2MTestClient.getDeviceIdStr()); + } catch (Exception e) { + log.warn("Failed to cleanup LwM2M observations before destroy: {}", e.getMessage()); + } + } else { + log.info("No active CoAP server found on ports 5685–5688. Skipping observe cleanup."); + } + lwM2MTestClient.destroy(); } } catch (Exception e) { @@ -718,7 +723,7 @@ public abstract class AbstractLwM2MIntegrationTest extends AbstractTransportInte protected AbstractLwM2MBootstrapServerCredential getBootstrapServerCredentialNoSec(boolean isBootstrap) { AbstractLwM2MBootstrapServerCredential bootstrapServerCredential = new NoSecLwM2MBootstrapServerCredential(); bootstrapServerCredential.setServerPublicKey(""); - bootstrapServerCredential.setShortServerId(isBootstrap ? shortServerIdBs0 : shortServerId); + bootstrapServerCredential.setShortServerId(isBootstrap ? null : shortServerId); bootstrapServerCredential.setBootstrapServerIs(isBootstrap); bootstrapServerCredential.setHost(isBootstrap ? hostBs : host); bootstrapServerCredential.setPort(isBootstrap ? portBs : port); diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java index 3bb127d736..40fb6d3ff0 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java @@ -70,6 +70,7 @@ import org.thingsboard.server.transport.lwm2m.utils.LwM2mValueConverterImpl; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.HashMap; @@ -141,7 +142,7 @@ public class LwM2MTestClient { private LwM2mTemperatureSensor lwM2mTemperatureSensor12; private String deviceIdStr; - public void init(Security security, Security securityBs, int port, boolean isRpc, + public void init(Security securityLwm2m, Security securityBs, int port, boolean isRpc, LwM2mUplinkMsgHandler defaultLwM2mUplinkMsgHandler, LwM2mClientContext clientContext, Integer cIdLength, boolean queueMode, boolean supportFormatOnly_SenMLJSON_SenMLCBOR, Integer value3_0_9) throws InvalidDDFFileException, IOException { @@ -151,38 +152,31 @@ public class LwM2MTestClient { ObjectsInitializer initializer = createFreshInitializer(); - forceNullSecurityId(securityBs); - forceNullSecurityId(security); - if (securityBs != null && security != null) { - // SECURITIES -// securityBs.setId(0); -// security.setId(1); - - LwM2mInstanceEnabler[] instances = new LwM2mInstanceEnabler[]{securityBs, security}; - initializer.setInstancesForObject(SECURITY, instances); - log.warn("Security BS section: securityBsId [{}] Security Lwm2m section: securityLwm2mId [{}] ", securityBs.getId(), security.getId()); - // SERVER - Server lwm2mServer = new Server(shortServerId, TimeUnit.MINUTES.toSeconds(60)); -// lwm2mServer.setId(0); - instances = new LwM2mInstanceEnabler[]{lwm2mServer}; - initializer.setInstancesForObject(SERVER, instances); + // SECURITY + if (securityLwm2m != null && securityLwm2m.getId() != null) { + forceNullSecurityId(securityLwm2m); + } + if (securityBs!= null && securityBs.getId() != null) { + forceNullSecurityId(securityBs); + } + if (securityBs != null && securityLwm2m != null) { + log.warn("Security Both: securityBs: [{}] and security Lwm2m [{}]", securityBs.getId(), securityLwm2m.getId()); + initializer.setInstancesForObject(SECURITY, securityBs, securityLwm2m); } else if (securityBs != null) { - // SECURITY + log.warn("Security BS only: securityBs: [{}] ", securityBs.getId()); initializer.setInstancesForObject(SECURITY, securityBs); - // SERVER - initializer.setClassForObject(SERVER, Server.class); - log.warn("Security BS section: securityBsId [{}] ", securityBs.getId()); - } else { + } else if (securityLwm2m != null){ // SECURITY - initializer.setInstancesForObject(SECURITY, security); - // SERVER - Server lwm2mServer = new Server(shortServerId, TimeUnit.MINUTES.toSeconds(60)); -// lwm2mServer.setId(0); - initializer.setInstancesForObject(SERVER, lwm2mServer); - log.warn("Security Lwm2m section: securityLwm2mId [{}] Server Lwm2m section: securityLwm2mId [{}] ", security.getId(), lwm2mServer.getId()); + log.warn("Security Lwm2m only: security Lwm2m [{}]", securityLwm2m.getId()); + initializer.setInstancesForObject(SECURITY, securityLwm2m); } - + // SERVER + Server serverLwm2m = new Server(shortServerId, TimeUnit.MINUTES.toSeconds(60)); + initializer.setInstancesForObject(SERVER, serverLwm2m); + // DEVICE initializer.setInstancesForObject(DEVICE, lwM2MDevice = new SimpleLwM2MDevice(executor, value3_0_9)); + + // OTHER t initializer.setInstancesForObject(FIRMWARE, fwLwM2MDevice = new FwLwM2MDevice()); initializer.setInstancesForObject(SOFTWARE_MANAGEMENT, swLwM2MDevice = new SwLwM2MDevice()); initializer.setClassForObject(ACCESS_CONTROL, DummyInstanceEnabler.class); @@ -429,25 +423,43 @@ public class LwM2MTestClient { public void destroy() { if (leshanClient != null) { - leshanClient.destroy(true); - } - if (lwM2MDevice != null) { - lwM2MDevice.destroy(); - } - if (fwLwM2MDevice != null) { - fwLwM2MDevice.destroy(); - } - if (swLwM2MDevice != null) { - swLwM2MDevice.destroy(); - } - if (lwM2MBinaryAppDataContainer != null) { - lwM2MBinaryAppDataContainer.destroy(); + try { + leshanClient.destroy(true); + } catch (Exception e) { + log.warn("Failed to destroy Leshan client", e); + } finally { + leshanClient = null; + } } - if (lwM2MTemperatureSensor != null) { - lwM2MTemperatureSensor.destroy(); + + // ThingsBoard custom LwM2M objects + destroySafe(lwM2MDevice); + destroySafe(fwLwM2MDevice); + destroySafe(swLwM2MDevice); + destroySafe(lwM2MBinaryAppDataContainer); + destroySafe(lwM2MTemperatureSensor); + + lwM2MDevice = null; + fwLwM2MDevice = null; + swLwM2MDevice = null; + lwM2MBinaryAppDataContainer = null; + lwM2MTemperatureSensor = null; + } + + + private void destroySafe(Object obj) { + if (obj == null) return; + try { + Method destroy = obj.getClass().getMethod("destroy"); + destroy.invoke(obj); + } catch (NoSuchMethodException e) { + // не має destroy() — ігноруємо + } catch (Exception e) { + log.warn("Failed to destroy {}", obj.getClass().getSimpleName(), e); } } + public void start(boolean isStartLw) { if (leshanClient != null) { leshanClient.start(); @@ -470,8 +482,7 @@ public class LwM2MTestClient { } private ObjectsInitializer createFreshInitializer() { -// List models = new ArrayList<>(ObjectLoader.loadAllDefault()); - List models = ObjectLoader.loadAllDefault(); + List models = new ArrayList<>(ObjectLoader.loadAllDefault()); for (String resourceName : lwm2mClientResources) { try (InputStream in = LwM2MTestClient.class.getClassLoader() .getResourceAsStream("lwm2m/" + resourceName)) { diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/AbstractSecurityLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/AbstractSecurityLwM2MIntegrationTest.java index 264c3b84c7..a16b782acf 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/AbstractSecurityLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/AbstractSecurityLwM2MIntegrationTest.java @@ -96,7 +96,6 @@ public abstract class AbstractSecurityLwM2MIntegrationTest extends AbstractLwM2M protected static final String SERVER_STORE_PWD = "server_ks_password"; protected static final String SERVER_CERT_ALIAS = "server"; protected static final String SERVER_CERT_ALIAS_BS = "bootstrap"; - protected static final Security SECURITY_NO_SEC_BS = noSecBootstrap(URI_BS);; protected final X509Certificate serverX509Cert; // server certificate signed by rootCA protected final X509Certificate serverX509CertBs; // serverBs certificate signed by rootCA protected final PublicKey serverPublicKeyFromCert; // server public key used for RPK @@ -179,14 +178,14 @@ public abstract class AbstractSecurityLwM2MIntegrationTest extends AbstractLwM2M defaultBootstrapCredentials.setLwm2mServer(serverCredentials); } - public void basicTestConnectionBefore(String clientEndpoint, - String awaitAlias, - LwM2MProfileBootstrapConfigType type, - Set expectedStatuses, - LwM2MClientState finishState) throws Exception { + public void basicTestConnectionStartBS(String clientEndpoint, + String awaitAlias, + LwM2MProfileBootstrapConfigType type, + Set expectedStatuses, + LwM2MClientState finishState) throws Exception { Lwm2mDeviceProfileTransportConfiguration transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsNoSec(type)); LwM2MDeviceCredentials deviceCredentials = getDeviceCredentialsNoSec(createNoSecClientCredentials(clientEndpoint)); - this.basicTestConnection(null , SECURITY_NO_SEC_BS, + this.basicTestConnection(null , noSecBootstrap(URI_BS), deviceCredentials, clientEndpoint, transportConfiguration, @@ -244,7 +243,7 @@ public abstract class AbstractSecurityLwM2MIntegrationTest extends AbstractLwM2M LwM2MDeviceCredentials deviceCredentials = getDeviceCredentialsNoSec(createNoSecClientCredentials(clientEndpoint)); this.basicTestConnectionBootstrapRequestTrigger( SECURITY_NO_SEC, - SECURITY_NO_SEC_BS, + noSecBootstrap(URI_BS), deviceCredentials, clientEndpoint, transportConfiguration, @@ -360,7 +359,7 @@ public abstract class AbstractSecurityLwM2MIntegrationTest extends AbstractLwM2M default: throw new IllegalStateException("Unexpected value: " + mode); } - bootstrapServerCredential.setShortServerId(isBootstrap ? shortServerIdBs0 : shortServerId); + bootstrapServerCredential.setShortServerId(isBootstrap ? null : shortServerId); bootstrapServerCredential.setBootstrapServerIs(isBootstrap); bootstrapServerCredential.setHost(isBootstrap ? hostBs : host); bootstrapServerCredential.setPort(isBootstrap ? securityPortBs : securityPort); diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSNoTriggerTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSNoTriggerTest.java index edfe805cf8..b218c39aec 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSNoTriggerTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/NoSecLwM2MIntegrationBSNoTriggerTest.java @@ -27,13 +27,13 @@ public class NoSecLwM2MIntegrationBSNoTriggerTest extends AbstractSecurityLwM2MI public void testWithNoSecConnectBsSuccess_UpdateTwoSectionsBootstrapAndLm2m_ConnectLwm2mSuccess() throws Exception { String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "NoTrigger" + BOTH.name(); String awaitAlias = "await on client state (NoSecBS two section)"; - basicTestConnectionBefore(clientEndpoint, awaitAlias, BOTH, expectedStatusesRegistrationBsSuccess, ON_REGISTRATION_SUCCESS); + basicTestConnectionStartBS(clientEndpoint, awaitAlias, BOTH, expectedStatusesRegistrationBsSuccess, ON_REGISTRATION_SUCCESS); } @Test public void testWithNoSecConnectBsSuccess_UpdateLwm2mSectionAndLm2m_ConnectLwm2mSuccess() throws Exception { String clientEndpoint = CLIENT_ENDPOINT_NO_SEC_BS + "NoTrigger" + LWM2M_ONLY.name(); String awaitAlias = "await on client state (NoSecBS Lwm2m section)"; - basicTestConnectionBefore(clientEndpoint, awaitAlias, LWM2M_ONLY, expectedStatusesRegistrationBsSuccess, ON_REGISTRATION_SUCCESS); + basicTestConnectionStartBS(clientEndpoint, awaitAlias, LWM2M_ONLY, expectedStatusesRegistrationBsSuccess, ON_REGISTRATION_SUCCESS); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/PskLwm2mIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/PskLwm2mIntegrationTest.java index e4bf935306..275383104e 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/PskLwm2mIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/PskLwm2mIntegrationTest.java @@ -58,8 +58,7 @@ public class PskLwm2mIntegrationTest extends AbstractSecurityLwM2MIntegrationTes Hex.decodeHex(keyPsk.toCharArray())); Lwm2mDeviceProfileTransportConfiguration transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(PSK, NONE)); LwM2MDeviceCredentials deviceCredentials = getDeviceCredentialsSecure(clientCredentials, null, null, PSK, false); - this.basicTestConnection(security, - null, + this.basicTestConnection(security, null, deviceCredentials, clientEndpoint, transportConfiguration, @@ -85,8 +84,7 @@ public class PskLwm2mIntegrationTest extends AbstractSecurityLwM2MIntegrationTes Lwm2mDeviceProfileTransportConfiguration transportConfiguration = getTransportConfiguration(TELEMETRY_WITH_ONE_OBSERVE, getBootstrapServerCredentialsSecure(PSK, NONE)); LwM2MDeviceCredentials deviceCredentials = getDeviceCredentialsSecure(clientCredentials, null, null, PSK, false); String awaitAlias = "await on client state (Psk_Lwm2m)"; - Device lwm2mDevice = this.basicTestConnection(security, - null, + Device lwm2mDevice = this.basicTestConnection(security, null, deviceCredentials, clientEndpoint, transportConfiguration, @@ -121,8 +119,7 @@ public class PskLwm2mIntegrationTest extends AbstractSecurityLwM2MIntegrationTes Lwm2mDeviceProfileTransportConfiguration transportConfiguration = getTransportConfiguration(TELEMETRY_WITH_ONE_OBSERVE, getBootstrapServerCredentialsSecure(PSK, NONE)); LwM2MDeviceCredentials deviceCredentials = getDeviceCredentialsSecure(clientCredentials, null, null, PSK, false); String awaitAlias = "await on client state (Psk_Lwm2m)"; - Device lwm2mDevice = this.basicTestConnection(security, - null, + Device lwm2mDevice = this.basicTestConnection(security, null, deviceCredentials, clientEndpoint, transportConfiguration, diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/X509_NoTrustLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/X509_NoTrustLwM2MIntegrationTest.java index d1576ede1b..fd8a51b041 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/X509_NoTrustLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/X509_NoTrustLwM2MIntegrationTest.java @@ -118,8 +118,7 @@ public class X509_NoTrustLwM2MIntegrationTest extends AbstractSecurityLwM2MInteg serverX509CertBs.getEncoded()); Lwm2mDeviceProfileTransportConfiguration transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(X509, BOTH)); LwM2MDeviceCredentials deviceCredentials = getDeviceCredentialsSecure(clientCredentials, privateKey, certificate, X509, false); - this.basicTestConnection(security, - null, + this.basicTestConnection(security, null, deviceCredentials, clientEndpoint, transportConfiguration, diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/X509_TrustLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/X509_TrustLwM2MIntegrationTest.java index 81b708ba33..b7a6290741 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/X509_TrustLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/security/sql/X509_TrustLwM2MIntegrationTest.java @@ -50,8 +50,7 @@ public class X509_TrustLwM2MIntegrationTest extends AbstractSecurityLwM2MIntegra serverX509Cert.getEncoded()); Lwm2mDeviceProfileTransportConfiguration transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(X509, NONE)); LwM2MDeviceCredentials deviceCredentials = getDeviceCredentialsSecure(clientCredentials, privateKey, certificate, X509, false); - this.basicTestConnection(security, - null, + this.basicTestConnection(security, null, deviceCredentials, clientEndpoint, transportConfiguration, @@ -77,8 +76,7 @@ public class X509_TrustLwM2MIntegrationTest extends AbstractSecurityLwM2MIntegra serverX509CertBs.getEncoded()); Lwm2mDeviceProfileTransportConfiguration transportConfiguration = getTransportConfiguration(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, getBootstrapServerCredentialsSecure(X509, BOTH)); LwM2MDeviceCredentials deviceCredentials = getDeviceCredentialsSecure(clientCredentials, privateKey, certificate, X509, false); - this.basicTestConnection(security, - null, + this.basicTestConnection(security,null, deviceCredentials, clientEndpoint, transportConfiguration, diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/credentials/lwm2m/Lwm2mServerIdentifier.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/credentials/lwm2m/Lwm2mServerIdentifier.java index a9f81ab655..95aa95597f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/credentials/lwm2m/Lwm2mServerIdentifier.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/credentials/lwm2m/Lwm2mServerIdentifier.java @@ -23,22 +23,21 @@ package org.thingsboard.server.common.data.device.credentials.lwm2m; public enum Lwm2mServerIdentifier { /** - * Bootstrap Short Server ID (0). - * Reserved for the Bootstrap Server — used exclusively during the bootstrap phase. + * Not used for identifying an LwM2M Server (0). */ - BOOTSTRAP(0, "Bootstrap Short Server ID", true), + NOT_USED_IDENTIFYING_LWM2M_SERVER_MIN(0, "Bootstrap Short Server ID", false), /** * Primary LwM2M Server Short Server ID (1). * Upper boundary for valid LwM2M Server Identifiers (1–65534). */ - PRIMARY_LWM2M_SERVER(1, "LwM2M Server Short Server ID", false), + PRIMARY_LWM2M_SERVER(1, "LwM2M Server Short Server ID", true), /** * Maximum valid LwM2M Server ID (65534). * Upper boundary for valid LwM2M Server Identifiers (1–65534). */ - LWM2M_SERVER_MAX(65534, "LwM2M Server Short Server ID", false), + LWM2M_SERVER_MAX(65534, "LwM2M Server Short Server ID", true), /** * Not used for identifying an LwM2M Server (65535). @@ -46,16 +45,16 @@ public enum Lwm2mServerIdentifier { * MUST NOT be assigned to any LwM2M Server according to OMA-TS-LightweightM2M-Core, §6.2.1. * OMA LwM2M Core / v1.2: Server / Short Server ID): «MAX_ID 65535 is a reserved value and MUST NOT be used for identifying an Object» */ - NOT_USED_IDENTIFYING_LWM2M_SERVER(65535, "Reserved sentinel value (no active server)", false); + NOT_USED_IDENTIFYING_LWM2M_SERVER_MAX(65535, "Reserved sentinel value (no active server)", false); private final int id; private final String description; - private final boolean isBootstrap; + private final boolean isLwm2mServer; - Lwm2mServerIdentifier(int id, String description, boolean isBootstrap) { + Lwm2mServerIdentifier(int id, String description, boolean isLwm2mServer) { this.id = id; this.description = description; - this.isBootstrap = isBootstrap; + this.isLwm2mServer = isLwm2mServer; } /** @@ -73,50 +72,35 @@ public enum Lwm2mServerIdentifier { } /** - * @return true if this ID represents a Bootstrap Server. + * @return true if this ID represents a Lwm2m Server. */ - public boolean isBootstrap() { - return isBootstrap; - } - - /** - * Checks whether a given numeric ID belongs to the Bootstrap Server (0). - * OMA Spec (LwM2M v1.0 / v1.1): - * Short Server ID Resource (Resource ID: 0) - * The Short Server ID identifies a Server Object Instance. - * The value 0 is reserved for the Bootstrap Server. - * A value between 1 and 65534 identifies a LwM2M Server. - * The value 65535 MUST NOT be used. - * @param id Short Server ID value. - * @return true if id == 0. - */ - public static boolean isBootstrap(int id) { - return id == BOOTSTRAP.id; + public boolean isLwm2mServer() { + return isLwm2mServer; } /** * Checks whether a given ID represents a valid LwM2M Server (1–65534). - * * @param id Short Server ID value. * @return true if the ID belongs to a standard LwM2M Server. */ public static boolean isLwm2mServer(int id) { return id >= PRIMARY_LWM2M_SERVER.id && id <= LWM2M_SERVER_MAX.id; } + public static boolean isNotLwm2mServer(int id) { + return id < PRIMARY_LWM2M_SERVER.id || id > LWM2M_SERVER_MAX.id; + } /** * Checks whether the provided ID is within the valid LwM2M range [0–65535]. - * * @param id ID to check. * @return true if valid, false otherwise. */ public static boolean isValid(int id) { - return id >= 0 && id <= 65535; + return id >= NOT_USED_IDENTIFYING_LWM2M_SERVER_MIN.getId() && id <= NOT_USED_IDENTIFYING_LWM2M_SERVER_MAX.getId(); } /** * Returns a {@link Lwm2mServerIdentifier} instance matching the given ID. - * * @param id numeric ID. * @return corresponding enum constant. * @throws IllegalArgumentException if no constant matches the given ID. diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/lwm2m/bootstrap/LwM2MServerSecurityConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/lwm2m/bootstrap/LwM2MServerSecurityConfig.java index 29c130b902..e5eb4be902 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/lwm2m/bootstrap/LwM2MServerSecurityConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/lwm2m/bootstrap/LwM2MServerSecurityConfig.java @@ -26,7 +26,7 @@ public class LwM2MServerSecurityConfig implements Serializable { @Schema(description = "Server short Id. Used as link to associate server Object Instance. This identifier uniquely identifies each LwM2M Server configured for the LwM2M Client. " + "This Resource MUST be set when the Bootstrap-Server Resource has a value of 'false'. " + - "The values ID:1 and ID:65534 values MUST NOT be used for identifying the LwM2M Server.", example = "123", accessMode = Schema.AccessMode.READ_ONLY) + "The values ID:0 and ID:65535 values MUST NOT be used for identifying the LwM2M Server.", example = "123", accessMode = Schema.AccessMode.READ_ONLY) protected Integer shortServerId = 123; /** Security -> ObjectId = 0 'LWM2M Security' */ @Schema(description = "Is Bootstrap Server or Lwm2m Server. " + diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/secure/LwM2MBootstrapConfig.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/secure/LwM2MBootstrapConfig.java index 517bcabac7..aaca3374a2 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/secure/LwM2MBootstrapConfig.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/secure/LwM2MBootstrapConfig.java @@ -127,7 +127,9 @@ public class LwM2MBootstrapConfig implements Serializable { private BootstrapConfig.ServerConfig setServerConfig (AbstractLwM2MBootstrapServerCredential serverCredential) { BootstrapConfig.ServerConfig serverConfig = new BootstrapConfig.ServerConfig(); - serverConfig.shortId = serverCredential.getShortServerId(); + if (serverCredential.getShortServerId() != null) { + serverConfig.shortId = serverCredential.getShortServerId(); + } serverConfig.lifetime = serverCredential.getLifetime(); serverConfig.defaultMinPeriod = serverCredential.getDefaultMinPeriod(); serverConfig.notifIfDisabled = serverCredential.isNotifIfDisabled(); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java index 33d1fb207c..7020869130 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java @@ -48,9 +48,9 @@ import static org.eclipse.leshan.core.LwM2mId.ACCESS_CONTROL; import static org.eclipse.leshan.core.LwM2mId.SECURITY; import static org.eclipse.leshan.core.LwM2mId.SERVER; import static org.eclipse.leshan.server.bootstrap.BootstrapUtil.toWriteRequest; -import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.BOOTSTRAP; import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.LWM2M_SERVER_MAX; import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.PRIMARY_LWM2M_SERVER; +import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.isLwm2mServer; @Slf4j public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTaskProvider { @@ -147,7 +147,7 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask log.error("Invalid lwm2mSecurityInstance [{}] by short server id [{}]", path.getObjectInstanceId(), lwm2mShortServerId); } } else { - this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().putIfAbsent(BOOTSTRAP.getId(), path.getObjectInstanceId()); + this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().putIfAbsent(0, path.getObjectInstanceId()); } } else if (path.getObjectId() == 1) { if (link.getAttributes().get("ssid") != null) { @@ -186,35 +186,35 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask } - /** Map - * 1) Only Lwm2m Server - * - Short Server ID == 1 - 65534 lwm2m) + /** Map => LwM2MBootstrapClientInstanceIds + * 1) Both + * - Short Server ID == null bs) * SECURITY = 0; InstanceId = 0 + * - Short Server ID == 1 - 65534 lwm2m) + * SECURITY = 0; InstanceId = 1 * SERVER = 1; InstanceId = 0 - * 2) Both - * - Short Server ID == 0 bs) + * 2) Only BS Server + * - Short Server ID == null bs) * SECURITY = 0; InstanceId = 0 - * SERVER = 1; InstanceId = null + * 3) Only Lwm2m Server * - Short Server ID == 1 - 65534 lwm2m) - * SECURITY = 0; InstanceId = 1 + * SECURITY = 0; InstanceId = 0 * SERVER = 1; InstanceId = 0 * */ public List> toRequests(BootstrapConfig bootstrapConfigNew, ContentFormat contentFormat, String endpoint) { - Integer bootstrapSecurityInstanceId = this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(BOOTSTRAP.getId()) == null ? - 0 : this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(BOOTSTRAP.getId()); + Integer bootstrapSecurityInstanceId = this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(null) == null ? + -2 : this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(null); List> requests = new ArrayList<>(); Set pathsDelete = new HashSet<>(); ConcurrentHashMap> requestsWrite = new ConcurrentHashMap<>(); /// handle security & handle - int lwm2mSecurityInstanceIdMax = -1; - int lwm2mServerInstanceIdMax = -1; // bootstrap Security new - There can only be one instance of bootstrap at a time. /// bs: handle security only for (BootstrapConfig.ServerSecurity security : new TreeMap<>(bootstrapConfigNew.security).values()) { - if (security.bootstrapServer && security.serverId == BOOTSTRAP.getId()) { + if (security.bootstrapServer && bootstrapSecurityInstanceId > -1) { // delete old bootstrap Security String path = "/" + SECURITY + "/" + bootstrapSecurityInstanceId; pathsDelete.add(path); @@ -230,19 +230,21 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask /// lwm2m server: handle security & server //max Lwm2m Security instance old id if new + int lwm2mSecurityInstanceIdMax = -1; for (Integer shortId : this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().keySet()) { - if (shortId >= BOOTSTRAP.getId() && shortId <= LWM2M_SERVER_MAX.getId()) { - lwm2mSecurityInstanceIdMax = this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(shortId) > - lwm2mSecurityInstanceIdMax ? this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(shortId) : - lwm2mSecurityInstanceIdMax; + if (isLwm2mServer(shortId)) { + lwm2mSecurityInstanceIdMax = Math.max( + this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(shortId), + lwm2mSecurityInstanceIdMax); } } //max Lwm2m Server instance old id if new + int lwm2mServerInstanceIdMax = -1; for (Integer shortId : this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().keySet()) { - if (shortId >= PRIMARY_LWM2M_SERVER.getId() && shortId <= LWM2M_SERVER_MAX.getId()) { - lwm2mServerInstanceIdMax = this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().get(shortId) > - lwm2mServerInstanceIdMax ? this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().get(shortId) : - lwm2mServerInstanceIdMax; + if (isLwm2mServer(shortId)) { + lwm2mServerInstanceIdMax = Math.max( + this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().get(shortId), + lwm2mServerInstanceIdMax); } } // Lwm2m update or new diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MConfigurationChecker.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MConfigurationChecker.java index 113c3451e0..3916849da3 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MConfigurationChecker.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MConfigurationChecker.java @@ -21,9 +21,9 @@ import org.eclipse.leshan.server.bootstrap.InvalidConfigurationException; import java.util.Map; -import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.BOOTSTRAP; -import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.NOT_USED_IDENTIFYING_LWM2M_SERVER; -import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.isLwm2mServer; +import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.NOT_USED_IDENTIFYING_LWM2M_SERVER_MIN; +import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.NOT_USED_IDENTIFYING_LWM2M_SERVER_MAX; +import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.isNotLwm2mServer; public class LwM2MConfigurationChecker extends ConfigurationChecker { @@ -78,15 +78,16 @@ public class LwM2MConfigurationChecker extends ConfigurationChecker { * This Resource MUST be set when the Bootstrap-Server Resource has false value. * Specific ID:0 and ID:65535 values MUST NOT be used for identifying the LwM2M Server (Section 6.3 of the LwM2M version 1.0 specification). */ - if (!security.bootstrapServer && !isLwm2mServer(srvCfg.shortId)) { - throw new InvalidConfigurationException("Specific ID:" + BOOTSTRAP.getId() + " and ID:" + NOT_USED_IDENTIFYING_LWM2M_SERVER.getId() + " values MUST NOT be used for identifying the LwM2M Server"); + if (!security.bootstrapServer && isNotLwm2mServer(srvCfg.shortId)) { + throw new InvalidConfigurationException("Specific ID:" + NOT_USED_IDENTIFYING_LWM2M_SERVER_MIN.getId() + " and ID:" + NOT_USED_IDENTIFYING_LWM2M_SERVER_MAX.getId() + " values MUST NOT be used for identifying the LwM2M Server"); } } } protected static BootstrapConfig.ServerSecurity getSecurityEntry(BootstrapConfig config, int shortId) { for (Map.Entry es : config.security.entrySet()) { - if (es.getValue().serverId == shortId) { + if ((es.getValue().serverId == null && shortId == 0) || + (es.getValue().serverId != null && es.getValue().serverId == shortId)) { return es.getValue(); } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2mUplinkMsgHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2mUplinkMsgHandler.java index 431b623da9..c3d02b8ce1 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2mUplinkMsgHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2mUplinkMsgHandler.java @@ -853,7 +853,8 @@ public class DefaultLwM2mUplinkMsgHandler extends LwM2MExecutorAwareService impl ResourceUpdateResult updateResource = new ResourceUpdateResult(lwM2MClient); request.getObjectInstances().forEach(instance -> instance.getResources().forEach((resId, lwM2mResource) ->{ - this.updateResourcesValue(updateResource, lwM2mResource, versionId + "/" + resId, Mode.REPLACE, 0); + String path = versionId.endsWith("/") ? versionId + resId : versionId + "/" + resId; + this.updateResourcesValue(updateResource, lwM2mResource, path, Mode.REPLACE, 0); }) ); clientContext.update(lwM2MClient); diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java index 2ea8c343d7..ac600f9201 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java @@ -69,10 +69,9 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.BOOTSTRAP; import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.LWM2M_SERVER_MAX; import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.PRIMARY_LWM2M_SERVER; -import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.isLwm2mServer; +import static org.thingsboard.server.common.data.device.credentials.lwm2m.Lwm2mServerIdentifier.isNotLwm2mServer; @Slf4j @Component @@ -342,19 +341,18 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator Date: Tue, 14 Oct 2025 21:53:40 +0300 Subject: [PATCH 39/88] lwm2m: bootstrap new: update tests-7 --- .../service/validator/DeviceProfileDataValidatorTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidatorTest.java index 18bfa1fd81..4c427544d4 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidatorTest.java @@ -127,7 +127,7 @@ class DeviceProfileDataValidatorTest { @Test void testValidateDeviceProfile_Lwm2mBootstrap_ShortServerId_Ok() { Integer shortServerId = 123; - Integer shortServerIdBs = 0; + Integer shortServerIdBs = null; DeviceProfile deviceProfile = getDeviceProfile(shortServerId, shortServerIdBs); validator.validateDataImpl(tenantId, deviceProfile); @@ -135,8 +135,8 @@ class DeviceProfileDataValidatorTest { } @Test - void testValidateDeviceProfile_Lwm2mShortServerId_Ok_BootstrapShortServerId_null_Error() { - verifyValidationError(123, null, "Bootstrap" + msgErrorNotNull); + void testValidateDeviceProfile_Lwm2mShortServerId_Ok_BootstrapShortServerId_0_Error() { + verifyValidationError(123, 0, msgErrorBsRange); } @Test From 4137a4a615ef4f8cfc84bb82878661feba9db4dd Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 15 Oct 2025 10:05:48 +0300 Subject: [PATCH 40/88] refactoring --- ...CalculatedFieldEntityMessageProcessor.java | 105 ++++++++----- ...alculatedFieldManagerMessageProcessor.java | 139 ++++++------------ .../CalculatedFieldRelatedEntityMsg.java | 9 +- 3 files changed, 117 insertions(+), 136 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index e253633727..63aaf2bd68 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -171,7 +171,8 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM throw cfe; } throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); - } } + } + } public void process(CalculatedFieldArgumentResetMsg msg) throws CalculatedFieldException { log.debug("[{}] Processing CF argument reset msg.", entityId); @@ -202,63 +203,91 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM actorCtx.stop(actorCtx.getSelf()); } } else { - EntityId msgEntityId = msg.getEntityId(); - if (msgEntityId instanceof CalculatedFieldId cfId) { - var state = removeState(cfId); - if (state != null) { - cfStateService.deleteState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); - } else { - msg.getCallback().onSuccess(); - } + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + var state = removeState(cfId); + if (state != null) { + cfStateService.deleteState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); } else { - if (states.isEmpty()) { - msg.getCallback().onSuccess(); - } - for (Map.Entry entry : states.entrySet()) { - LatestValuesAggregationCalculatedFieldState state = (LatestValuesAggregationCalculatedFieldState) entry.getValue(); - state.getArguments().forEach((argName, argEntry) -> { - AggArgumentEntry aggArgEntry = (AggArgumentEntry) argEntry; - aggArgEntry.getAggInputs().remove(msgEntityId); - }); - state.getInputs().remove(msgEntityId); - state.setLastMetricsEvalTs(-1); - processStateIfReady(state, Collections.emptyMap(), state.getCtx(), Collections.emptyList(), null, null, msg.getCallback()); - } + msg.getCallback().onSuccess(); } } } public void process(CalculatedFieldRelatedEntityMsg msg) throws CalculatedFieldException { - log.debug("[{}] Processing CF related entity msg.", msg.getEntityId()); - CalculatedFieldCtx cfCtx = msg.getCalculatedField(); - var state = states.get(cfCtx.getCfId()); - Map fetchedArguments = fetchAggArguments(msg.getCalculatedField(), msg.getEntityId()); + log.debug("[{}] Processing CF {} related entity msg.", msg.getRelatedEntityId(), msg.getAction()); + switch (msg.getAction()) { + case UPDATED -> handleRelationUpdate(msg); + case DELETED -> handleRelationDelete(msg); + default -> msg.getCallback().onSuccess(); + } + } + + private void handleRelationUpdate(CalculatedFieldRelatedEntityMsg msg) throws CalculatedFieldException { + CalculatedFieldCtx ctx = msg.getCalculatedField(); + var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); + var state = states.get(ctx.getCfId()); try { + boolean justRestored = false; if (state == null) { - state = createState(cfCtx); - } else { - state.setCtx(cfCtx, actorCtx); + state = createState(ctx); + justRestored = true; } if (state.isSizeOk()) { - if (state instanceof LatestValuesAggregationCalculatedFieldState latestValuesState) { - latestValuesState.setLastMetricsEvalTs(-1); + Map updatedArgs = new HashMap<>(); + if (!justRestored) { + updatedArgs = updateAggregationState(msg.getRelatedEntityId(), state, ctx); } - state.update(fetchedArguments, cfCtx); - state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, cfCtx.getCfId(), entityId), cfCtx.getMaxStateSize()); - states.put(cfCtx.getCfId(), state); - processStateIfReady(state, fetchedArguments, cfCtx, Collections.singletonList(cfCtx.getCfId()), null, null, msg.getCallback()); + processStateIfReady(state, updatedArgs, ctx, new ArrayList<>(), null, null, callback); } else { - throw new RuntimeException(cfCtx.getSizeExceedsLimitMessage()); + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(); } } catch (Exception e) { - log.debug("[{}][{}] Failed to initialize CF state", entityId, cfCtx.getCfId(), e); + log.debug("[{}][{}] Failed to initialize CF state", entityId, ctx.getCfId(), e); if (e instanceof CalculatedFieldException cfe) { throw cfe; } - throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(entityId).cause(e).build(); + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); + } + } + + private Map updateAggregationState(EntityId relatedEntityId, CalculatedFieldState state, CalculatedFieldCtx ctx) { + Map fetchedArgs = fetchAggArguments(ctx, relatedEntityId); + Map updatedArgs = state.update(fetchedArgs, ctx); + + if (state instanceof LatestValuesAggregationCalculatedFieldState latestValuesState) { + latestValuesState.setLastMetricsEvalTs(-1); + } + + state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); + + return updatedArgs; + } + + private void handleRelationDelete(CalculatedFieldRelatedEntityMsg msg) throws CalculatedFieldException { + CalculatedFieldCtx ctx = msg.getCalculatedField(); + CalculatedFieldId cfId = ctx.getCfId(); + CalculatedFieldState state = states.get(cfId); + if (state == null) { + msg.getCallback().onSuccess(); + return; + } + if (state instanceof LatestValuesAggregationCalculatedFieldState aggState) { + cleanupAggregationState(msg.getRelatedEntityId(), aggState); + processStateIfReady(state, Collections.emptyMap(), state.getCtx(), Collections.emptyList(), null, null, msg.getCallback()); + } else { + msg.getCallback().onSuccess(); } } + private void cleanupAggregationState(EntityId relatedEntityId, LatestValuesAggregationCalculatedFieldState state) { + state.getArguments().values().forEach(argEntry -> { + AggArgumentEntry aggEntry = (AggArgumentEntry) argEntry; + aggEntry.getAggInputs().remove(relatedEntityId); + }); + state.getInputs().remove(relatedEntityId); + state.setLastMetricsEvalTs(-1); + } + @SneakyThrows private Map fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId) { ListenableFuture> argumentsFuture = cfService.fetchAggEntityArguments(ctx, entityId); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 620eee41ee..37ec4cc3d1 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.actors.calculatedField; import lombok.extern.slf4j.Slf4j; +import org.apache.logging.log4j.util.TriConsumer; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; @@ -29,6 +30,7 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; @@ -303,56 +305,14 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onRelationUpdated(ComponentLifecycleMsg msg, TbCallback callback) { try { - MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); - EntityId toId = entityRelation.getTo(); EntityId fromId = entityRelation.getFrom(); String relationType = entityRelation.getType(); - EntityId toIdProfile = getProfileId(tenantId, toId); - EntityId fromIdProfile = getProfileId(tenantId, fromId); - - List toIdMatches = new ArrayList<>(); - List cfsByToId = getCalculatedFieldsByEntityId(toId); - List cfsByToProfileId = getCalculatedFieldsByEntityId(toIdProfile); - List cfsByToIdOrItsProfileId = new ArrayList<>(); - cfsByToIdOrItsProfileId.addAll(cfsByToId); - cfsByToIdOrItsProfileId.addAll(cfsByToProfileId); - - cfsByToIdOrItsProfileId.forEach(cf -> { - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - RelationPathLevel relation = configuration.getRelation(); - if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType())) { - toIdMatches.add(cf); - } - }); - - MultipleTbCallback toCfsCallback = new MultipleTbCallback(toIdMatches.size(), callbackForToAndFrom); - toIdMatches.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, toCfsCallback, (entityId, cb) -> initRelatedEntity(entityId, fromId, ctx, cb)); - }); - - List fromIdMatches = new ArrayList<>(); - List cfsByFromId = getCalculatedFieldsByEntityId(fromId); - List cfsByFromProfileId = getCalculatedFieldsByEntityId(fromIdProfile); - List cfsByFromIdOrItsProfileId = new ArrayList<>(); - cfsByFromIdOrItsProfileId.addAll(cfsByFromId); - cfsByFromIdOrItsProfileId.addAll(cfsByFromProfileId); - - cfsByFromIdOrItsProfileId.forEach(cf -> { - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - RelationPathLevel relation = configuration.getRelation(); - if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType())) { - fromIdMatches.add(cf); - } - }); - - MultipleTbCallback fromCfsCallback = new MultipleTbCallback(fromIdMatches.size(), callbackForToAndFrom); - fromIdMatches.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, fromCfsCallback, (entityId, cb) -> initRelatedEntity(entityId, toId, ctx, cb)); - }); - + MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); + processRelationByDirection(EntitySearchDirection.TO, relationType, toId, callbackForToAndFrom, (entityId, ctx, cb) -> initRelatedEntity(entityId, fromId, ctx, cb)); + processRelationByDirection(EntitySearchDirection.FROM, relationType, fromId, callbackForToAndFrom, (entityId, ctx, cb) -> initRelatedEntity(entityId, toId, ctx, cb)); } catch (Exception e) { callback.onSuccess(); } @@ -360,59 +320,43 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onRelationDeleted(ComponentLifecycleMsg msg, TbCallback callback) { try { - MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); - EntityId toId = entityRelation.getTo(); EntityId fromId = entityRelation.getFrom(); String relationType = entityRelation.getType(); - EntityId toIdProfile = getProfileId(tenantId, toId); - EntityId fromIdProfile = getProfileId(tenantId, fromId); - - List toIdMatches = new ArrayList<>(); - List cfsByToId = getCalculatedFieldsByEntityId(toId); - List cfsByToProfileId = getCalculatedFieldsByEntityId(toIdProfile); - List cfsByToIdOrItsProfileId = new ArrayList<>(); - cfsByToIdOrItsProfileId.addAll(cfsByToId); - cfsByToIdOrItsProfileId.addAll(cfsByToProfileId); - - cfsByToIdOrItsProfileId.forEach(cf -> { - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - RelationPathLevel relation = configuration.getRelation(); - if (EntitySearchDirection.TO.equals(relation.direction()) && relationType.equals(relation.relationType())) { - toIdMatches.add(cf); - } - }); - MultipleTbCallback toCfsCallback = new MultipleTbCallback(toIdMatches.size(), callbackForToAndFrom); - toIdMatches.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, toCfsCallback, (entityId, cb) -> deleteRelatedEntity(entityId, fromId, cb)); - }); + MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); + processRelationByDirection(EntitySearchDirection.TO, relationType, toId, callbackForToAndFrom, (entityId, ctx, cb) -> deleteRelatedEntity(entityId, fromId, ctx, cb)); + processRelationByDirection(EntitySearchDirection.FROM, relationType, fromId, callbackForToAndFrom, (entityId, ctx, cb) -> deleteRelatedEntity(entityId, toId, ctx, cb)); + } catch (Exception e) { + callback.onSuccess(); + } + } - List fromIdMatches = new ArrayList<>(); - List cfsByFromId = getCalculatedFieldsByEntityId(fromId); - List cfsByFromProfileId = getCalculatedFieldsByEntityId(fromIdProfile); - List cfsByFromIdOrItsProfileId = new ArrayList<>(); - cfsByFromIdOrItsProfileId.addAll(cfsByFromId); - cfsByFromIdOrItsProfileId.addAll(cfsByFromProfileId); - - cfsByFromIdOrItsProfileId.forEach(cf -> { - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); - RelationPathLevel relation = configuration.getRelation(); - if (EntitySearchDirection.FROM.equals(relation.direction()) && relationType.equals(relation.relationType())) { - fromIdMatches.add(cf); - } - }); + private void processRelationByDirection(EntitySearchDirection direction, + String relationType, + EntityId mainId, + MultipleTbCallback parentCallback, + TriConsumer relationAction) { + List cfsByEntityIdAndProfile = getCalculatedFieldsByEntityIdAndProfile(mainId); + if (cfsByEntityIdAndProfile.isEmpty()) { + parentCallback.onSuccess(); + return; + } - MultipleTbCallback fromCfsCallback = new MultipleTbCallback(fromIdMatches.size(), callbackForToAndFrom); - fromIdMatches.forEach(ctx -> { - applyToTargetCfEntityActors(ctx, fromCfsCallback, (entityId, cb) -> deleteRelatedEntity(entityId, toId, cb)); - }); + List matchingCfs = cfsByEntityIdAndProfile.stream() + .filter(cf -> { + var config = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + RelationPathLevel relation = config.getRelation(); + return direction.equals(relation.direction()) && relationType.equals(relation.relationType()); + }) + .toList(); + MultipleTbCallback directionCallback = new MultipleTbCallback(matchingCfs.size(), parentCallback); - } catch (Exception e) { - callback.onSuccess(); - } + matchingCfs.forEach(ctx -> + applyToTargetCfEntityActors(ctx, directionCallback, (entityId, cb) -> relationAction.accept(entityId, ctx, cb)) + ); } private void onCfCreated(ComponentLifecycleMsg msg, TbCallback callback) throws CalculatedFieldException { @@ -629,9 +573,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware EntityId entityId = msg.getEntityId(); log.debug("Received changed owner msg from entity [{}]", entityId); updateEntityOwner(entityId); - List cfs = new ArrayList<>(); - cfs.addAll(getCalculatedFieldsByEntityId(entityId)); - cfs.addAll(getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId))); + List cfs = getCalculatedFieldsByEntityIdAndProfile(entityId); if (cfs.isEmpty()) { msgCallback.onSuccess(); return; @@ -695,6 +637,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return result; } + private List getCalculatedFieldsByEntityIdAndProfile(EntityId entityId) { + List cfsByEntityIdAndProfile = new ArrayList<>(); + cfsByEntityIdAndProfile.addAll(getCalculatedFieldsByEntityId(entityId)); + cfsByEntityIdAndProfile.addAll(getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId))); + return cfsByEntityIdAndProfile; + } + private List getCalculatedFieldLinksByEntityId(EntityId entityId) { if (entityId == null) { return Collections.emptyList(); @@ -722,14 +671,14 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware getOrCreateActor(entityId).tell(msg); } - private void deleteRelatedEntity(EntityId entityId, EntityId relatedEntityId, TbCallback callback) { + private void deleteRelatedEntity(EntityId entityId, EntityId relatedEntityId, CalculatedFieldCtx cf, TbCallback callback) { log.debug("Pushing delete related entity msg to specific actor [{}]", relatedEntityId); - getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, relatedEntityId, callback)); + getOrCreateActor(entityId).tell(new CalculatedFieldRelatedEntityMsg(tenantId, relatedEntityId, ActionType.DELETED, cf, callback)); } private void initRelatedEntity(EntityId entityId, EntityId relatedEntityId, CalculatedFieldCtx cf, TbCallback callback) { log.debug("Pushing init related entity msg to specific actor [{}]", relatedEntityId); - getOrCreateActor(entityId).tell(new CalculatedFieldRelatedEntityMsg(tenantId, relatedEntityId, cf, callback)); + getOrCreateActor(entityId).tell(new CalculatedFieldRelatedEntityMsg(tenantId, relatedEntityId, ActionType.UPDATED, cf, callback)); } private void deleteCfForEntity(EntityId entityId, CalculatedFieldId cfId, TbCallback callback) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java index d3bdd9cc5c..bf73a1b1b0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldRelatedEntityMsg.java @@ -16,6 +16,7 @@ package org.thingsboard.server.actors.calculatedField; import lombok.Data; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.MsgType; @@ -27,16 +28,18 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; public class CalculatedFieldRelatedEntityMsg implements ToCalculatedFieldSystemMsg { private final TenantId tenantId; - private final EntityId entityId; + private final EntityId relatedEntityId; + private final ActionType action; private final CalculatedFieldCtx calculatedField; private final TbCallback callback; public CalculatedFieldRelatedEntityMsg(TenantId tenantId, - EntityId entityId, + EntityId relatedEntityId, ActionType action, CalculatedFieldCtx calculatedField, TbCallback callback) { this.tenantId = tenantId; - this.entityId = entityId; + this.relatedEntityId = relatedEntityId; + this.action = action; this.calculatedField = calculatedField; this.callback = callback; } From fac84c4cfc636d79685c6de47bb5ca316f16592d Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 15 Oct 2025 12:34:17 +0300 Subject: [PATCH 41/88] extended deleteTenant API to allow tenant admins delete tenant --- .../server/controller/TenantController.java | 2 +- .../permission/TenantAdminPermissions.java | 2 +- .../controller/TenantControllerTest.java | 23 +++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index 3ded7b7171..2f8be6589f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -115,7 +115,7 @@ public class TenantController extends BaseController { @ApiOperation(value = "Delete Tenant (deleteTenant)", notes = "Deletes the tenant, it's customers, rule chains, devices and all other related entities. Referencing non-existing tenant Id will cause an error." + SYSTEM_AUTHORITY_PARAGRAPH) - @PreAuthorize("hasAuthority('SYS_ADMIN')") + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") @RequestMapping(value = "/tenant/{tenantId}", method = RequestMethod.DELETE) @ResponseStatus(value = HttpStatus.OK) public void deleteTenant(@Parameter(description = TENANT_ID_PARAM_DESCRIPTION) diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index 7a824ca735..bc5e5b8697 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -73,7 +73,7 @@ public class TenantAdminPermissions extends AbstractPermissions { }; private static final PermissionChecker tenantPermissionChecker = - new PermissionChecker.GenericPermissionChecker(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY) { + new PermissionChecker.GenericPermissionChecker(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY, Operation.DELETE) { @Override @SuppressWarnings("unchecked") diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java index ba45be8e28..a33e8ca411 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java @@ -243,6 +243,29 @@ public class TenantControllerTest extends AbstractControllerTest { .andExpect(statusReason(containsString(msgErrorNoFound("Tenant", tenantIdStr)))); } + @Test + public void testDeleteTenantByTenantAdmin() throws Exception { + loginSysAdmin(); + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + Tenant savedTenant = saveTenant(tenant); + + //login as tenant admin + User tenantAdminUser = new User(); + tenantAdminUser.setAuthority(Authority.TENANT_ADMIN); + tenantAdminUser.setTenantId(savedTenant.getId()); + tenantAdminUser.setEmail("tenantToDelete@thingsboard.io"); + + createUserAndLogin(tenantAdminUser, TENANT_ADMIN_PASSWORD); + + String tenantIdStr = savedTenant.getId().getId().toString(); + deleteTenant(savedTenant.getId()); + loginSysAdmin(); + doGet("/api/tenant/" + tenantIdStr) + .andExpect(status().isNotFound()) + .andExpect(statusReason(containsString(msgErrorNoFound("Tenant", tenantIdStr)))); + } + @Test public void testFindTenants() throws Exception { loginSysAdmin(); From 846d9f6a8bc49540b93d0ef8c7cfe63c9e64f592 Mon Sep 17 00:00:00 2001 From: ArtemDzhereleiko Date: Wed, 15 Oct 2025 13:28:21 +0300 Subject: [PATCH 42/88] UI: Remove ondestroy --- .../pages/admin/two-factor-auth-settings.component.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/admin/two-factor-auth-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/two-factor-auth-settings.component.ts index 38a6230b6f..2770f904bb 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/two-factor-auth-settings.component.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/two-factor-auth-settings.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, DestroyRef, OnDestroy, OnInit, QueryList, ViewChildren } from '@angular/core'; +import { Component, DestroyRef, OnInit, QueryList, ViewChildren } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; import { Store } from '@ngrx/store'; @@ -39,7 +39,7 @@ import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; templateUrl: './two-factor-auth-settings.component.html', styleUrls: [ './settings-card.scss', './two-factor-auth-settings.component.scss'] }) -export class TwoFactorAuthSettingsComponent extends PageComponent implements OnInit, HasConfirmForm, OnDestroy { +export class TwoFactorAuthSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { private readonly posIntValidation = [Validators.required, Validators.min(1), Validators.pattern(/^\d*$/)]; @@ -72,10 +72,6 @@ export class TwoFactorAuthSettingsComponent extends PageComponent implements OnI }); } - ngOnDestroy() { - super.ngOnDestroy(); - } - confirmForm(): UntypedFormGroup { return this.twoFaFormGroup; } From b1bffdfd586afe3a0f77d897487c6cfc1d7d74ad Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 15 Oct 2025 15:56:17 +0300 Subject: [PATCH 43/88] fixed provisioning flow for non-valid provisionState attribute --- .../device/DeviceProvisionServiceImpl.java | 11 ++++++++--- .../msa/connectivity/CoapClientTest.java | 18 +++++++++++++++--- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java index ce7abd3bf1..ffd16c1287 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java @@ -186,9 +186,14 @@ public class DeviceProvisionServiceImpl implements DeviceProvisionService { try { Optional provisionState = attributesService.find(device.getTenantId(), device.getId(), AttributeScope.SERVER_SCOPE, DEVICE_PROVISION_STATE).get(); - if (provisionState != null && provisionState.isPresent() && provisionState.get().getValueAsString().equals(PROVISIONED_STATE)) { - notify(device, provisionRequest, TbMsgType.PROVISION_FAILURE, false); - throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); + if (provisionState != null && provisionState.isPresent()) { + if (provisionState.get().getValueAsString().equals(PROVISIONED_STATE)) { + notify(device, provisionRequest, TbMsgType.PROVISION_FAILURE, false); + throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); + } else { + log.error("[{}][{}] Unknown provision state: {}!", device.getName(), DEVICE_PROVISION_STATE, provisionState.get().getValueAsString()); + throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); + } } else { saveProvisionStateAttribute(device).get(); notify(device, provisionRequest, TbMsgType.PROVISION_SUCCESS, true); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/CoapClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/CoapClientTest.java index bc3d8b1b81..5697d40db4 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/CoapClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/CoapClientTest.java @@ -21,6 +21,7 @@ import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileProvisionType; @@ -28,6 +29,8 @@ import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.msa.AbstractCoapClientTest; import org.thingsboard.server.msa.DisableUIListeners; +import java.util.Map; + import static org.assertj.core.api.Assertions.assertThat; import static org.thingsboard.server.msa.prototypes.DevicePrototypes.defaultDevicePrototype; @@ -52,18 +55,27 @@ public class CoapClientTest extends AbstractCoapClientTest{ DeviceProfile deviceProfile = testRestClient.getDeviceProfileById(device.getDeviceProfileId()); deviceProfile = updateDeviceProfileWithProvisioningStrategy(deviceProfile, DeviceProfileProvisionType.CHECK_PRE_PROVISIONED_DEVICES); - DeviceCredentials expectedDeviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); + DeviceCredentials deviceCreds = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); JsonNode provisionResponse = JacksonUtil.fromBytes(createCoapClientAndPublish(device.getName())); - assertThat(provisionResponse.get("credentialsType").asText()).isEqualTo(expectedDeviceCredentials.getCredentialsType().name()); - assertThat(provisionResponse.get("credentialsValue").asText()).isEqualTo(expectedDeviceCredentials.getCredentialsId()); + assertThat(provisionResponse.get("credentialsType").asText()).isEqualTo(deviceCreds.getCredentialsType().name()); + assertThat(provisionResponse.get("credentialsValue").asText()).isEqualTo(deviceCreds.getCredentialsId()); assertThat(provisionResponse.get("status").asText()).isEqualTo("SUCCESS"); + JsonNode attributes = testRestClient.getAttributes(device.getId(), AttributeScope.SERVER_SCOPE, "provisionState"); + assertThat(attributes.get(0).get("value").asText()).isEqualTo("provisioned"); + // provision second time should fail JsonNode provisionResponse2 = JacksonUtil.fromBytes(createCoapClientAndPublish(device.getName())); assertThat(provisionResponse2.get("status").asText()).isEqualTo("FAILURE"); + // update provision attribute to non-valid value + testRestClient.postTelemetryAttribute(device.getId(), AttributeScope.SERVER_SCOPE.name(), JacksonUtil.valueToTree(Map.of("provisionState", "non-valid"))); + + JsonNode provisionResponse3 = JacksonUtil.fromBytes(createCoapClientAndPublish(device.getName())); + assertThat(provisionResponse3.get("status").asText()).isEqualTo("FAILURE"); + updateDeviceProfileWithProvisioningStrategy(deviceProfile, DeviceProfileProvisionType.DISABLED); } From a693cabc05de347138d9116ce01b3c0593543889 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 15 Oct 2025 16:44:36 +0300 Subject: [PATCH 44/88] added more tests --- ...CalculatedFieldEntityMessageProcessor.java | 46 ++- ...alculatedFieldManagerMessageProcessor.java | 6 - ...tractCalculatedFieldProcessingService.java | 3 +- .../service/cf/ctx/state/ArgumentEntry.java | 6 +- .../ctx/state/BaseCalculatedFieldState.java | 11 + .../cf/ctx/state/CalculatedFieldCtx.java | 6 +- .../ctx/state/SimpleCalculatedFieldState.java | 10 - .../ctx/state/SingleValueArgumentEntry.java | 9 + .../state/aggregation/AggArgumentEntry.java | 8 +- ...java => AggSingleEntityArgumentEntry.java} | 17 +- ...ValuesAggregationCalculatedFieldState.java | 103 ++++-- .../aggregation/function/AvgAggEntry.java | 2 +- .../state/aggregation/function/new_agg.json | 4 +- .../utils/CalculatedFieldArgumentUtils.java | 4 +- .../server/utils/CalculatedFieldUtils.java | 10 +- ...tValuesAggregationCalculatedFieldTest.java | 344 ++++++++++++------ ...gregationCalculatedFieldConfiguration.java | 1 + 17 files changed, 388 insertions(+), 202 deletions(-) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/{AggSingleArgumentEntry.java => AggSingleEntityArgumentEntry.java} (79%) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 63aaf2bd68..ce29dc06e0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -53,7 +53,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; @@ -67,6 +67,7 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -329,7 +330,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } else if (proto.getAttrDataCount() > 0) { processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); } else if (proto.getRemovedTsKeysCount() > 0) { - processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithFetchedValue(ctx, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithFetchedValue(ctx, msg.getEntityId(), proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); } else if (proto.getRemovedAttrKeysCount() > 0) { processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithDefaultValue(ctx, msg.getEntityId(), proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); } else { @@ -405,7 +406,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private void processRemovedTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { - processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArgumentsWithFetchedValue(ctx, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArgumentsWithFetchedValue(ctx, entityId, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); } private void processRemovedAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { @@ -565,7 +566,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); String argName = aggArgNames.get(key); if (argName != null) { - arguments.put(argName, new AggSingleArgumentEntry(originator, item)); + arguments.put(argName, new AggSingleEntityArgumentEntry(originator, item)); } } } @@ -618,7 +619,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); String argName = aggArgNames.get(key); if (argName != null) { - arguments.put(argName, new AggSingleArgumentEntry(entityId, item)); + arguments.put(argName, new AggSingleEntityArgumentEntry(entityId, item)); } } } @@ -631,14 +632,21 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return Collections.emptyMap(); } List geofencingArgumentNames = ctx.getLinkedEntityAndCurrentOwnerGeofencingArgumentNames(); - return mapToArgumentsWithDefaultValue(argNames, ctx.getArguments(), geofencingArgumentNames, scope, removedAttrKeys); + List relatedArgumentNames = ctx.getRelatedEntityArgumentNames(); + return mapToArgumentsWithDefaultValue(entityId, argNames, ctx.getArguments(), geofencingArgumentNames, relatedArgumentNames, scope, removedAttrKeys); } private Map mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, AttributeScopeProto scope, List removedAttrKeys) { - return mapToArgumentsWithDefaultValue(ctx.getMainEntityArguments(), ctx.getArguments(), ctx.getMainEntityGeofencingArgumentNames(), scope, removedAttrKeys); + return mapToArgumentsWithDefaultValue(null, ctx.getMainEntityArguments(), ctx.getArguments(), ctx.getMainEntityGeofencingArgumentNames(), new ArrayList<>(), scope, removedAttrKeys); } - private Map mapToArgumentsWithDefaultValue(Map argNames, Map configArguments, List geofencingArgNames, AttributeScopeProto scope, List removedAttrKeys) { + private Map mapToArgumentsWithDefaultValue(EntityId msgEntityId, + Map argNames, + Map configArguments, + List geofencingArgNames, + List relatedEntityArgNames, + AttributeScopeProto scope, + List removedAttrKeys) { Map arguments = new HashMap<>(); for (String removedKey : removedAttrKeys) { ReferencedEntityKey key = new ReferencedEntityKey(removedKey, ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); @@ -652,22 +660,36 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } Argument argument = configArguments.get(argName); String defaultValue = (argument != null) ? argument.getDefaultValue() : null; - arguments.put(argName, StringUtils.isNotEmpty(defaultValue) + SingleValueArgumentEntry argumentEntry = StringUtils.isNotEmpty(defaultValue) ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) - : new SingleValueArgumentEntry()); + : new SingleValueArgumentEntry(); + if (relatedEntityArgNames.contains(argName)) { + arguments.put(argName, new AggSingleEntityArgumentEntry(msgEntityId, argumentEntry)); + continue; + } + arguments.put(argName, argumentEntry); } return arguments; } - private Map mapToArgumentsWithFetchedValue(CalculatedFieldCtx ctx, List removedTelemetryKeys) { + private Map mapToArgumentsWithFetchedValue(CalculatedFieldCtx ctx, EntityId entityId, List removedTelemetryKeys) { Map deletedArguments = ctx.getArguments().entrySet().stream() .filter(entry -> removedTelemetryKeys.contains(entry.getValue().getRefEntityKey().getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, deletedArguments); - fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); + if (CalculatedFieldType.LATEST_VALUES_AGGREGATION.equals(ctx.getCfType())) { + fetchedArgs = fetchedArgs.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + argEntry -> new AggSingleEntityArgumentEntry(entityId, argEntry.getValue()) + )); + } else { + fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); + } + return fetchedArgs; } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 37ec4cc3d1..af5a672157 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -516,12 +516,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware .toList(); } - private List getCfsWithRelationToEntity(EntityId entityId) { - return aggCalculatedFields.values().stream() - .filter(cf -> !findRelationsForCf(entityId, cf).isEmpty()) - .toList(); - } - private List findRelationsForCf(EntityId entityId, CalculatedFieldCtx cf) { List result = new ArrayList<>(); if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration configuration) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index c30a12a8f3..94695bcc8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -250,7 +250,8 @@ public abstract class AbstractCalculatedFieldProcessingService { .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))), MoreExecutors.directExecutor()); } - public ListenableFuture fetchAggArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) {List>> futures = aggEntities.stream() + public ListenableFuture fetchAggArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) { + List>> futures = aggEntities.stream() .map(entityId -> { ListenableFuture singleAggEntryFut = fetchSingleAggArgumentEntry(tenantId, entityId, argument, startTs); return Futures.transform(singleAggEntryFut, singleAggEntry -> Map.entry(entityId, singleAggEntry), MoreExecutors.directExecutor()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 4e3d00ee62..ca0a12e1d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import java.util.List; @@ -39,7 +39,7 @@ import java.util.Map; @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING"), @JsonSubTypes.Type(value = AggArgumentEntry.class, name = "AGGREGATE_LATEST"), - @JsonSubTypes.Type(value = AggSingleArgumentEntry.class, name = "AGGREGATE_LATEST_SINGLE") + @JsonSubTypes.Type(value = AggSingleEntityArgumentEntry.class, name = "AGGREGATE_LATEST_SINGLE") }) public interface ArgumentEntry { @@ -75,7 +75,7 @@ public interface ArgumentEntry { } static ArgumentEntry createAggSingleArgument(EntityId entityId, KvEntry kvEntry) { - return new AggSingleArgumentEntry(entityId, kvEntry); + return new AggSingleEntityArgumentEntry(entityId, kvEntry); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index f75711a107..2d853fd3fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Getter; import lombok.Setter; +import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -134,4 +135,14 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, this.latestTimestamp = Math.max(this.latestTimestamp, newTs); } + protected Object formatResult(double result, Integer decimals) { + if (decimals == null) { + return result; + } + if (decimals.equals(0)) { + return TbUtils.toInt(result); + } + return TbUtils.toFixed(result, decimals); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index f8b9f5b665..3a9d75b3d4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -107,6 +107,7 @@ public class CalculatedFieldCtx { private boolean relationQueryDynamicArguments; private List mainEntityGeofencingArgumentNames; private List linkedEntityAndCurrentOwnerGeofencingArgumentNames; + private List relatedEntityArgumentNames; private long scheduledUpdateIntervalMillis; @@ -126,6 +127,7 @@ public class CalculatedFieldCtx { this.argNames = new ArrayList<>(); this.mainEntityGeofencingArgumentNames = new ArrayList<>(); this.linkedEntityAndCurrentOwnerGeofencingArgumentNames = new ArrayList<>(); + this.relatedEntityArgumentNames = new ArrayList<>(); this.output = calculatedField.getConfiguration().getOutput(); if (calculatedField.getConfiguration() instanceof ArgumentsBasedCalculatedFieldConfiguration argBasedConfig) { this.arguments.putAll(argBasedConfig.getArguments()); @@ -153,6 +155,7 @@ public class CalculatedFieldCtx { } } this.argNames.addAll(arguments.keySet()); + this.relatedEntityArgumentNames.addAll(relatedEntityArguments.values()); if (argBasedConfig instanceof ExpressionBasedCalculatedFieldConfiguration expressionBasedConfig) { this.expression = expressionBasedConfig.getExpression(); this.useLatestTs = CalculatedFieldType.SIMPLE.equals(calculatedField.getType()) && ((SimpleCalculatedFieldConfiguration) argBasedConfig).isUseLatestTs(); @@ -174,7 +177,7 @@ public class CalculatedFieldCtx { } this.requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { - this.scheduledUpdateIntervalMillis = aggConfig.getDeduplicationIntervalMillis(); + this.useLatestTs = aggConfig.isUseLatestTs(); } this.systemContext = systemContext; this.tbelInvokeService = systemContext.getTbelInvokeService(); @@ -578,6 +581,7 @@ public class CalculatedFieldCtx { } if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig && other.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig + && thisConfig.getDeduplicationIntervalMillis() != otherConfig.getDeduplicationIntervalMillis() && !thisConfig.getMetrics().equals(otherConfig.getMetrics())) { return true; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 65cb595632..8886482ef8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -62,16 +62,6 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { .build()); } - private Object formatResult(double expressionResult, Integer decimals) { - if (decimals == null) { - return expressionResult; - } - if (decimals.equals(0)) { - return TbUtils.toInt(expressionResult); - } - return TbUtils.toFixed(expressionResult, decimals); - } - private JsonNode createResultJson(boolean useLatestTs, String outputName, Object result) { ObjectNode valuesNode = JacksonUtil.newObjectNode(); if (result instanceof Double doubleValue) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 288b486e83..e81201c961 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -45,6 +45,15 @@ public class SingleValueArgumentEntry implements ArgumentEntry { public static final Long DEFAULT_VERSION = -1L; + public SingleValueArgumentEntry(ArgumentEntry entry) { + if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + this.ts = singleValueArgumentEntry.ts; + this.kvEntryValue = singleValueArgumentEntry.kvEntryValue; + this.version = singleValueArgumentEntry.version; + this.forceResetPrevious = singleValueArgumentEntry.forceResetPrevious; + } + } + public SingleValueArgumentEntry(TsKvProto entry) { this.ts = entry.getTs(); if (entry.hasVersion()) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java index 12ae2c4638..7e3a8623e4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java @@ -48,11 +48,11 @@ public class AggArgumentEntry implements ArgumentEntry { if (entry instanceof AggArgumentEntry aggArgumentEntry) { aggInputs.putAll(aggArgumentEntry.aggInputs); return true; - } else if (entry instanceof AggSingleArgumentEntry aggSingleArgumentEntry) { - if (aggSingleArgumentEntry.isDeleted()) { - aggInputs.remove(aggSingleArgumentEntry.getEntityId()); + } else if (entry instanceof AggSingleEntityArgumentEntry aggSingleEntityArgumentEntry) { + if (aggSingleEntityArgumentEntry.isDeleted()) { + aggInputs.remove(aggSingleEntityArgumentEntry.getEntityId()); } else { - aggInputs.put(aggSingleArgumentEntry.getEntityId(), aggSingleArgumentEntry); + aggInputs.put(aggSingleEntityArgumentEntry.getEntityId(), aggSingleEntityArgumentEntry); } return true; } else { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java similarity index 79% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java index 6b81d5380c..32ce77311d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java @@ -30,34 +30,39 @@ import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; @Data @NoArgsConstructor @AllArgsConstructor -public class AggSingleArgumentEntry extends SingleValueArgumentEntry { +public class AggSingleEntityArgumentEntry extends SingleValueArgumentEntry { private EntityId entityId; private boolean deleted; - public AggSingleArgumentEntry(EntityId entityId, TsKvProto entry) { + public AggSingleEntityArgumentEntry(EntityId entityId, ArgumentEntry entry) { super(entry); this.entityId = entityId; } - public AggSingleArgumentEntry(EntityId entityId, AttributeValueProto entry) { + public AggSingleEntityArgumentEntry(EntityId entityId, TsKvProto entry) { super(entry); this.entityId = entityId; } - public AggSingleArgumentEntry(EntityId entityId, KvEntry entry) { + public AggSingleEntityArgumentEntry(EntityId entityId, AttributeValueProto entry) { super(entry); this.entityId = entityId; } - public AggSingleArgumentEntry(EntityId entityId, long ts, BasicKvEntry kvEntryValue, Long version) { + public AggSingleEntityArgumentEntry(EntityId entityId, KvEntry entry) { + super(entry); + this.entityId = entityId; + } + + public AggSingleEntityArgumentEntry(EntityId entityId, long ts, BasicKvEntry kvEntryValue, Long version) { super(ts, kvEntryValue, version); this.entityId = entityId; } @Override public boolean updateEntry(ArgumentEntry entry) { - if (entry instanceof AggSingleArgumentEntry singleValueEntry) { + if (entry instanceof AggSingleEntityArgumentEntry singleValueEntry) { if (singleValueEntry.getTs() <= ts) { return false; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java index d25cde020c..1df27b4cbd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import lombok.Data; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.actors.TbActorRef; @@ -43,10 +45,11 @@ import java.util.Map; import java.util.Map.Entry; @Slf4j -@Data +@Getter public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedFieldState { private long lastArgsRefreshTs = -1; + @Setter private long lastMetricsEvalTs = -1; private long deduplicationInterval = -1; private Map metrics; @@ -76,8 +79,7 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF @Override public void init() { super.init(); -// long scheduledUpdateIntervalMillis = ctx.getScheduledUpdateIntervalMillis(); -// ctx.scheduleReevaluation(scheduledUpdateIntervalMillis, actorCtx); + ctx.scheduleReevaluation(deduplicationInterval, actorCtx); } @Override @@ -102,41 +104,51 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF @Override public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception { + if (!shouldRecalculate()) { + return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() + .result(null) + .build()); + } + Output output = ctx.getOutput(); + ObjectNode aggResult = aggregateMetrics(output); + lastMetricsEvalTs = System.currentTimeMillis(); + ctx.scheduleReevaluation(deduplicationInterval, actorCtx); + return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() + .type(output.getType()) + .scope(output.getScope()) + .result(createResultJson(ctx.isUseLatestTs(), aggResult)) + .build()); + } + + private boolean shouldRecalculate() { boolean intervalPassed = lastMetricsEvalTs <= System.currentTimeMillis() - deduplicationInterval; boolean argsUpdatedDuringInterval = lastArgsRefreshTs > lastMetricsEvalTs; - if (intervalPassed && argsUpdatedDuringInterval) { - ObjectNode aggResult = JacksonUtil.newObjectNode(); - for (Entry entry : metrics.entrySet()) { - String metricKey = entry.getKey(); - AggMetric metric = entry.getValue(); - - AggEntry aggMetric = AggFunctionFactory.createAggFunction(metric.getFunction()); - - for (Map entityInputs : inputs.values()) { - if (applyAggregation(metric.getFilter(), entityInputs)) { - Object arg = resolveAggregationInput(metric.getInput(), entityInputs); - if (arg != null) { - aggMetric.update(arg); - } - } - } + return intervalPassed && argsUpdatedDuringInterval; + } + + private ObjectNode aggregateMetrics(Output output) throws Exception { + ObjectNode aggResult = JacksonUtil.newObjectNode(); + for (Entry entry : metrics.entrySet()) { + String metricKey = entry.getKey(); + AggMetric metric = entry.getValue(); - aggMetric.result().ifPresent(result -> { - aggResult.set(metricKey, JacksonUtil.valueToTree(result)); - }); + AggEntry aggMetricEntry = AggFunctionFactory.createAggFunction(metric.getFunction()); + aggregateMetric(metric, aggMetricEntry); + aggMetricEntry.result().ifPresent(result -> { + aggResult.set(metricKey, JacksonUtil.valueToTree(formatResult(result, output.getDecimalsByDefault()))); + }); + } + return aggResult; + } + + private void aggregateMetric(AggMetric metric, AggEntry aggEntry) throws Exception { + for (Map entityInputs : inputs.values()) { + if (applyAggregation(metric.getFilter(), entityInputs)) { + Object arg = resolveAggregationInput(metric.getInput(), entityInputs); + if (arg != null) { + aggEntry.update(arg); + } } - Output output = ctx.getOutput(); - lastMetricsEvalTs = System.currentTimeMillis(); - ctx.scheduleReevaluation(deduplicationInterval, actorCtx); - return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() - .type(output.getType()) - .scope(output.getScope()) - .result(aggResult) - .build()); - } else { - return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() - .result(null) - .build()); } } @@ -158,4 +170,25 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF } } + private Object formatResult(Object aggregationResult, Integer decimals) { + try { + double result = Double.parseDouble(aggregationResult.toString()); + return formatResult(result, decimals); + } catch (Exception e) { + throw new IllegalArgumentException("Aggregation result cannot be parsed: " + aggregationResult, e); + } + } + + protected JsonNode createResultJson(boolean useLatestTs, JsonNode result) { + long latestTs = getLatestTimestamp(); + if (useLatestTs && latestTs != -1) { + ObjectNode resultNode = JacksonUtil.newObjectNode(); + resultNode.put("ts", latestTs); + resultNode.set("values", result); + return resultNode; + } else { + return result; + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java index ad1f2ee8a8..afe6abb93e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java @@ -35,7 +35,7 @@ public class AvgAggEntry extends BaseAggEntry { @Override protected double prepareResult() { - return sum.divide(BigDecimal.valueOf(count), 2, RoundingMode.HALF_UP).doubleValue(); + return sum.divide(BigDecimal.valueOf(count), 10, RoundingMode.HALF_UP).doubleValue(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json index 32cd053d41..c6b841b673 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json @@ -7,8 +7,8 @@ "allEnabledUntil": 1769907492297 }, "entityId": { - "entityType": "ASSET_PROFILE", - "id": "2b759c60-a8f4-11f0-be29-7fa922118588" + "entityType": "ASSET", + "id": "f8ad0800-a9a6-11f0-bbe6-459b63b420fe" }, "configuration": { "type": "LATEST_VALUES_AGGREGATION", diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index 6485508602..74cdb0cddb 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -34,7 +34,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -59,7 +59,7 @@ public class CalculatedFieldArgumentUtils { if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { return ArgumentEntry.createAggSingleArgument(entityId, kvEntry.get()); } else { - return new AggSingleArgumentEntry(); + return new AggSingleEntityArgumentEntry(); } } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index d4ee7bb2e3..19ad4bfb7c 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -47,7 +47,7 @@ import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; @@ -236,7 +236,7 @@ public class CalculatedFieldUtils { LatestValuesAggregationCalculatedFieldState aggState = (LatestValuesAggregationCalculatedFieldState) state; Map> arguments = new HashMap<>(); proto.getAggArgumentsList().forEach(argProto -> { - AggSingleArgumentEntry entry = fromAggSingleValueArgumentProto(argProto); + AggSingleEntityArgumentEntry entry = fromAggSingleValueArgumentProto(argProto); arguments.computeIfAbsent(argProto.getValue().getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); }); arguments.forEach((argName, entityInputs) -> { @@ -248,14 +248,14 @@ public class CalculatedFieldUtils { return state; } - public static AggSingleArgumentEntry fromAggSingleValueArgumentProto(AggSingleArgumentEntryProto proto) { + public static AggSingleEntityArgumentEntry fromAggSingleValueArgumentProto(AggSingleArgumentEntryProto proto) { if (!proto.hasValue()) { - return new AggSingleArgumentEntry(); + return new AggSingleEntityArgumentEntry(); } EntityId entityId = ProtoUtils.fromProto(proto.getEntityId()); SingleValueArgumentProto singleValueArgument = proto.getValue(); TsValueProto tsValueProto = singleValueArgument.getValue(); - return new AggSingleArgumentEntry( + return new AggSingleEntityArgumentEntry( entityId, tsValueProto.getTs(), (BasicKvEntry) KvProtoUtil.fromTsValueProto(singleValueArgument.getArgName(), tsValueProto), diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java index cd9c1c578b..a4b43e362b 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -58,6 +58,7 @@ import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.cf.CalculatedFieldIntegrationTest.POLL_INTERVAL; @DaoSqlTest @@ -116,7 +117,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll } @Test - public void testNoTelemetryOnDevices_checkDefaultValueUsed() throws Exception { + public void testCreateCfOnProfile_checkInitialAggregation() throws Exception { Asset asset2 = createAsset("Asset 2", assetProfile.getId()); Device device3 = createDevice("Device 3", "1234567890333"); Device device4 = createDevice("Device 4", "1234567890444"); @@ -124,22 +125,153 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset2.getId(), device3.getId(), "Contains"); createEntityRelation(asset2.getId(), device4.getId(), "Contains"); - createOccupancyCF("Occupied spaces", asset2.getId()); + createOccupancyCF(assetProfile.getId()); - await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "1", + "totalSpaces", "2" + )); + + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); + }); + + postTelemetry(device3.getId(), "{\"occupied\":true}"); + + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "1", + "totalSpaces", "2" + )); + }); + } + + @Test + public void testAddEntityToProfile_checkAggregation() throws Exception { + createOccupancyCF(assetProfile.getId()); + + Device device3 = createDevice("Device 3", "1234567890333"); + Device device4 = createDevice("Device 4", "1234567890444"); + postTelemetry(device3.getId(), "{\"occupied\":true}"); + postTelemetry(device4.getId(), "{\"occupied\":true}"); + + Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + + await().alias("add entity to profile with no related entities and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode occupancy = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + assertThat(occupancy.get("freeSpaces").get(0).get("value").isNull()).isTrue(); + assertThat(occupancy.get("occupiedSpaces").get(0).get("value").isNull()).isTrue(); + assertThat(occupancy.get("totalSpaces").get(0).get("value").isNull()).isTrue(); + }); + + createEntityRelation(asset2.getId(), device3.getId(), "Contains"); + createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + + await().alias("create relations and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "0", + "occupiedSpaces", "2", + "totalSpaces", "2" + )); + }); + + postTelemetry(device3.getId(), "{\"occupied\":false}"); + + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "1", + "totalSpaces", "2" + )); }); } @Test - public void testUpdateTelemetry_checkMetricsCalculation() throws Exception { - createOccupancyCF("Occupied spaces", asset.getId()); + public void testChangeEntityProfile_checkAggregation() throws Exception { + Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + Device device3 = createDevice("Device 3", "1234567890333"); + Device device4 = createDevice("Device 4", "1234567890444"); + + createEntityRelation(asset2.getId(), device3.getId(), "Contains"); + createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + + createOccupancyCF(assetProfile.getId()); + + await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "1", + "totalSpaces", "2" + )); + + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); + }); + + AssetProfile newAssetProfile = createAssetProfile("New Asset Profile"); + asset2.setAssetProfileId(newAssetProfile.getId()); + doPost("/api/asset", asset2, Asset.class); + + postTelemetry(device3.getId(), "{\"occupied\":true}"); + + await().alias("change profile and no aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); + }); + } + + @Test + public void testCreateCfOnAssetAndNoTelemetryOnDevices_checkDefaultValueUsed() throws Exception { + Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + Device device3 = createDevice("Device 3", "1234567890333"); + Device device4 = createDevice("Device 4", "1234567890444"); + + createEntityRelation(asset2.getId(), device3.getId(), "Contains"); + createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + + createOccupancyCF(asset2.getId()); + + await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); + }); + } + + @Test + public void testCreateCfAndUpdateTelemetry_checkAggregation() throws Exception { + createOccupancyCF(asset.getId()); checkInitialCalculation(); postTelemetry(device1.getId(), "{\"occupied\":false}"); @@ -147,17 +279,38 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); }); } @Test - public void testUpdateTelemetry_checkMetricsCalculationNotExecutedUntilDeduplicationInterval() throws Exception { - createOccupancyCF("Occupied spaces", asset.getId()); + public void testDeleteCf_checkNoAggregation() throws Exception { + CalculatedField cf = createOccupancyCF(asset.getId()); + checkInitialCalculation(); + + doDelete("/api/calculatedField/" + cf.getId().getId().toString()) + .andExpect(status().isOk()); + + postTelemetry(device1.getId(), "{\"occupied\":false}"); + + await().alias("delete cf and update telemetry and no aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "1", + "totalSpaces", "2" + )); + }); + } + + @Test + public void testUpdateTelemetry_checkAggregationNotExecutedUntilDeduplicationInterval() throws Exception { + createOccupancyCF(asset.getId()); checkInitialCalculation(); postTelemetry(device1.getId(), "{\"occupied\":false}"); @@ -171,136 +324,92 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll await().alias("create CF and perform initial calculation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); }); } @Test - public void testCreateRelation_checkMetricsCalculation() throws Exception { - createOccupancyCF("Occupied spaces", asset.getId()); - checkInitialCalculation(); + public void testDeleteTelemetry_checkAggregationWithPreviousValuesOrDefault() throws Exception { + Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + Device device3 = createDevice("Device 3", "1234567890333"); + Device device4 = createDevice("Device 4", "1234567890444"); - Device device3 = createDevice("Device 3", deviceProfile.getId(), "1234567890333"); + createEntityRelation(asset2.getId(), device3.getId(), "Contains"); + createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + postTelemetry(device3.getId(), "{\"occupied\":false}"); + postTelemetry(device4.getId(), "{\"occupied\":true}"); postTelemetry(device3.getId(), "{\"occupied\":true}"); - createEntityRelation(asset.getId(), device3.getId(), "Contains"); + createOccupancyCF(asset2.getId()); - await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("3"); + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "0", + "occupiedSpaces", "2", + "totalSpaces", "2" + )); }); - } - - @Test - public void testDeleteRelation_checkMetricsCalculation() throws Exception { - createOccupancyCF("Occupied spaces", asset.getId()); - checkInitialCalculation(); - deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); + doDelete("/api/plugins/telemetry/DEVICE/" + device3.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=0&endTs=" + System.currentTimeMillis(), String.class); + doDelete("/api/plugins/telemetry/DEVICE/" + device4.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=0&endTs=" + System.currentTimeMillis(), String.class); - await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("delete latest telemetry and perform aggregation with previous or default values").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy).isNotNull(); - assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); }); } @Test - public void testCfOnProfile_checkMetricsCalculation() throws Exception { - Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + public void testCreateRelation_checkAggregation() throws Exception { + createOccupancyCF(asset.getId()); + checkInitialCalculation(); + Device device3 = createDevice("Device 3", deviceProfile.getId(), "1234567890333"); - postTelemetry(device3.getId(), "{\"occupied\":false}"); - Device device4 = createDevice("Device 4", deviceProfile.getId(), "1234567890444"); - postTelemetry(device4.getId(), "{\"occupied\":false}"); - createEntityRelation(asset2.getId(), device3.getId(), "Contains"); - createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + postTelemetry(device3.getId(), "{\"occupied\":true}"); - createOccupancyCF("Occupied spaces 2", assetProfile.getId()); + createEntityRelation(asset.getId(), device3.getId(), "Contains"); - await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancyAsset1 = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancyAsset1).isNotNull(); - assertThat(occupancyAsset1.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancyAsset1.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancyAsset1.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); - - ObjectNode occupancyAsset2 = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancyAsset2).isNotNull(); - assertThat(occupancyAsset2.get("freeSpaces").get(0).get("value").asText()).isEqualTo("2"); - assertThat(occupancyAsset2.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); - assertThat(occupancyAsset2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "2", + "totalSpaces", "3" + )); }); + } - postTelemetry(device3.getId(), "{\"occupied\":true}"); + @Test + public void testDeleteRelation_checkAggregation() throws Exception { + createOccupancyCF(asset.getId()); + checkInitialCalculation(); - await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) + deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); + + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { - ObjectNode occupancy2 = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); - assertThat(occupancy2).isNotNull(); - assertThat(occupancy2.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy2.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("1"); - assertThat(occupancy2.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "1", + "occupiedSpaces", "0", + "totalSpaces", "1" + )); }); } -// -// @Test -// public void testChangeProfile_checkMetricsCalculation() throws Exception { -// DeviceProfile deviceProfile2 = doPost("/api/deviceProfile", createDeviceProfile("Device Profile 2"), DeviceProfile.class); -// device1.setDeviceProfileId(deviceProfile2.getId()); -// device1 = doPost("/api/device?accessToken=" + accessToken1, device1, Device.class); -// -// postTelemetry(device1.getId(), "{\"occupied\":false}"); -// -// await().alias("change profile and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) -// .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) -// .untilAsserted(() -> { -// ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); -// assertThat(occupancy).isNotNull(); -// assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); -// assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("0"); -// assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("1"); -// }); -// } -// -// @Test -// public void testCfWithoutTargetProfileSpecified_checkMetricsCalculation() throws Exception { -// Device device3 = createDevice("Device 3", "1234567890333"); -// postTelemetry(device3.getId(), "{\"occupied\":true}"); -// createEntityRelation(asset.getId(), device3.getId(), "Contains"); -// -// var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) calculatedField.getConfiguration(); -// configuration.getSource().setEntityProfiles(Collections.emptyList()); -// calculatedField.setConfiguration(configuration); -// saveCalculatedField(calculatedField); -// -// await().alias("update cf and perform aggregation for 3 devices").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) -// .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) -// .untilAsserted(() -> { -// ObjectNode occupancy = getLatestTelemetry(asset.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); -// assertThat(occupancy).isNotNull(); -// assertThat(occupancy.get("freeSpaces").get(0).get("value").asText()).isEqualTo("1"); -// assertThat(occupancy.get("occupiedSpaces").get(0).get("value").asText()).isEqualTo("2"); -// assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("3"); -// }); -// } private void checkInitialCalculation() { await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) @@ -316,7 +425,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); } - private CalculatedField createOccupancyCF(String name, EntityId entityId) { + private CalculatedField createOccupancyCF(EntityId entityId) { Map arguments = new HashMap<>(); Argument argument = new Argument(); argument.setRefEntityKey(new ReferencedEntityKey("occupied", ArgumentType.TS_LATEST, null)); @@ -344,8 +453,9 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll Output output = new Output(); output.setType(OutputType.TIME_SERIES); + output.setDecimalsByDefault(0); - return createAggCf(name, entityId, + return createAggCf("Occupied spaces", entityId, new RelationPathLevel(EntitySearchDirection.FROM, "Contains"), arguments, aggMetrics, @@ -393,6 +503,12 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll return doPost("/api/asset", asset, Asset.class); } + private void verifyTelemetry(EntityId entityId, Map expectedResults) throws Exception { + ObjectNode result = getLatestTelemetry(entityId, expectedResults.keySet().toArray(new String[0])); + assertThat(result).isNotNull(); + expectedResults.forEach((key, value) -> assertThat(result.get(key).get(0).get("value").asText()).isEqualTo(value)); + } + private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java index 89f7e9a339..43a3360ce6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java @@ -32,6 +32,7 @@ public class LatestValuesAggregationCalculatedFieldConfiguration implements Argu private long deduplicationIntervalMillis; private Map metrics; private Output output; + private boolean useLatestTs; @Override public CalculatedFieldType getType() { From e6479d5856e4c8860e674d2d309a502c978aa25b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 16 Oct 2025 12:38:37 +0300 Subject: [PATCH 45/88] removed relation by profile processing --- .../processing/AbstractConsumerService.java | 16 +-- .../server/dao/relation/RelationService.java | 15 --- .../configuration/aggregation/AggSource.java | 30 ----- .../ProfileEntityRelationPathQuery.java | 21 ---- .../dao/relation/BaseRelationService.java | 100 ---------------- .../server/dao/relation/RelationCacheKey.java | 7 +- .../server/dao/relation/RelationDao.java | 7 -- .../dao/sql/relation/JpaRelationDao.java | 108 ------------------ .../dao/sql/relation/RelationRepository.java | 36 ------ 9 files changed, 5 insertions(+), 335 deletions(-) delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 36a35d9e44..6e162256a4 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -188,13 +188,9 @@ public abstract class AbstractConsumerService> findByRelationPathQueryAsync(TenantId tenantId, EntityRelationPathQuery relationPathQuery); - ListenableFuture> findByProfileEntityRelationPathQueryAsync(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery); - - List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery); - - ListenableFuture> findByFromAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId from, String relationType, EntityId targetProfileId); - - List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId profileId); - - ListenableFuture> findByToAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId to, String relationType, EntityId targetProfileId); - - List findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId profileId); - - void evictRelationsByEntityAndProfile(TenantId tenantId, EntityId entityId, EntityId profileId); - // TODO: This method may be useful for some validations in the future // ListenableFuture checkRecursiveRelation(EntityId from, EntityId to); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java deleted file mode 100644 index 84f6f92eb3..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggSource.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.cf.configuration.aggregation; - -import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.relation.RelationPathLevel; - -import java.util.List; - -@Data -public class AggSource { - - private RelationPathLevel relation; - private List entityProfiles; - -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java deleted file mode 100644 index 32b338ff6f..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/relation/ProfileEntityRelationPathQuery.java +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.relation; - -import org.thingsboard.server.common.data.id.EntityId; - -public record ProfileEntityRelationPathQuery(EntityId rootEntityId, RelationPathLevel level, EntityId targetEntityProfileId) { -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 2d584ebebe..18d1806fe4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -45,7 +45,6 @@ import org.thingsboard.server.common.data.relation.EntityRelationInfo; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.EntitySearchDirection; -import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.relation.RelationTypeGroup; @@ -515,105 +514,6 @@ public class BaseRelationService implements RelationService { return executor.submit(() -> relationDao.findByRelationPathQuery(tenantId, relationPathQuery)); } - @Override - public ListenableFuture> findByProfileEntityRelationPathQueryAsync(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery) { - log.trace("Executing findByProfileEntityRelationPathQueryAsync, tenantId [{}], relationPathQuery {}", tenantId, relationPathQuery); - validateId(tenantId, id -> "Invalid tenant id: " + id); - validate(relationPathQuery); - RelationPathLevel relationPathLevel = relationPathQuery.level(); - return switch (relationPathLevel.direction()) { - case FROM -> findByFromAndTypeAndEntityProfileAsync(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); - case TO -> findByToAndTypeAndEntityProfileAsync(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); - }; - } - - @Override - public List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery relationPathQuery) { - log.trace("Executing findByProfileEntityRelationPathQuery, tenantId [{}], relationPathQuery {}", tenantId, relationPathQuery); - validateId(tenantId, id -> "Invalid tenant id: " + id); - validate(relationPathQuery); - return relationDao.findByProfileEntityRelationPathQuery(tenantId, relationPathQuery); -// RelationPathLevel relationPathLevel = relationPathQuery.level(); -// return switch (relationPathLevel.direction()) { -// case FROM -> findByFromAndTypeAndEntityProfile(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); -// case TO -> findByToAndTypeAndEntityProfile(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), relationPathQuery.targetEntityProfileId()); -// }; - } - - @Override - public ListenableFuture> findByFromAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId from, String relationType, EntityId targetProfileId) { - log.trace("Executing findByFromAndTypeAndEntityProfileAsync [{}][{}][{}]", from, relationType, targetProfileId); - validate(from); - validateType(relationType); - if (targetProfileId == null) { - return findByFromAndTypeAsync(tenantId, from, relationType, RelationTypeGroup.COMMON); - } - return executor.submit(() -> findByFromAndTypeAndEntityProfile(tenantId, from, relationType, targetProfileId)); - } - - @Override - public List findByFromAndTypeAndEntityProfile(TenantId tenantId, EntityId from, String relationType, EntityId targetProfileId) { - if (targetProfileId == null) { - return findByFromAndType(tenantId, from, relationType, RelationTypeGroup.COMMON); - } -// RelationCacheKey cacheKey = RelationCacheKey.builder().from(from).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.FROM).entityProfile(targetProfileId).build(); -// return cache.getAndPutInTransaction(cacheKey, -// () -> relationDao.findByFromAndTypeAndProfile(tenantId, from, relationType, RelationTypeGroup.COMMON, targetProfileId), -// RelationCacheValue::getRelations, -// relations -> RelationCacheValue.builder().relations(relations).build(), false); - - return relationDao.findByFromAndTypeAndProfile(tenantId, from, relationType, RelationTypeGroup.COMMON, targetProfileId); - } - - @Override - public ListenableFuture> findByToAndTypeAndEntityProfileAsync(TenantId tenantId, EntityId to, String relationType, EntityId targetProfileId) { - log.trace("Executing findByToAndTypeAndEntityProfileAsync [{}][{}][{}]", to, relationType, targetProfileId); - validate(to); - validateType(relationType); - if (targetProfileId == null) { - return findByToAndTypeAsync(tenantId, to, relationType, RelationTypeGroup.COMMON); - } - return executor.submit(() -> findByToAndTypeAndEntityProfile(tenantId, to, relationType, targetProfileId)); - } - - @Override - public List findByToAndTypeAndEntityProfile(TenantId tenantId, EntityId to, String relationType, EntityId targetProfileId) { - if (targetProfileId == null) { - return findByFromAndType(tenantId, to, relationType, RelationTypeGroup.COMMON); - } -// RelationCacheKey cacheKey = RelationCacheKey.builder().to(to).type(relationType).typeGroup(RelationTypeGroup.COMMON).direction(EntitySearchDirection.TO).entityProfile(targetProfileId).build(); -// return cache.getAndPutInTransaction(cacheKey, -// () -> relationDao.findByToAndTypeAndProfile(tenantId, to, relationType, RelationTypeGroup.COMMON, targetProfileId), -// RelationCacheValue::getRelations, -// relations -> RelationCacheValue.builder().relations(relations).build(), false); - - return relationDao.findByToAndTypeAndProfile(tenantId, to, relationType, RelationTypeGroup.COMMON, targetProfileId); - } - - @Override - public void evictRelationsByEntityAndProfile(TenantId tenantId, EntityId entityId, EntityId profileId) { - -// List keys = new ArrayList<>(5); -// keys.add(new RelationCacheKey(entityId, null, event.getType(), event.getTypeGroup())); -// keys.add(new RelationCacheKey(event.getFrom(), null, event.getType(), event.getTypeGroup(), EntitySearchDirection.FROM)); -// keys.add(new RelationCacheKey(event.getFrom(), null, null, event.getTypeGroup(), EntitySearchDirection.FROM)); -// keys.add(new RelationCacheKey(null, event.getTo(), event.getType(), event.getTypeGroup(), EntitySearchDirection.TO)); -// keys.add(new RelationCacheKey(null, event.getTo(), null, event.getTypeGroup(), EntitySearchDirection.TO)); -// cache.evict(keys); -// log.debug("Processed evict event: {}", event); - - List keys = new ArrayList<>(2); - keys.add(RelationCacheKey.builder().from(entityId).entityProfile(profileId).build()); - keys.add(RelationCacheKey.builder().to(entityId).entityProfile(profileId).build()); - cache.evict(keys); - log.debug("Processed evict relations by keys: {}", keys); - } - - private void validate(ProfileEntityRelationPathQuery relationPathQuery) { - validateId((UUIDBased) relationPathQuery.rootEntityId(), id -> "Invalid root entity id: " + id); - relationPathQuery.level().validate(); - } - private void validate(EntityRelationPathQuery relationPathQuery) { validateId((UUIDBased) relationPathQuery.rootEntityId(), id -> "Invalid root entity id: " + id); List levels = relationPathQuery.levels(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java index 344af0a6f3..d6f0525c9d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java @@ -40,14 +40,9 @@ public class RelationCacheKey implements Serializable { private final String type; private final RelationTypeGroup typeGroup; private final EntitySearchDirection direction; - private final EntityId entityProfile; public RelationCacheKey(EntityId from, EntityId to, String type, RelationTypeGroup typeGroup) { - this(from, to, type, typeGroup, null, null); - } - - public RelationCacheKey(EntityId from, EntityId to, String type, RelationTypeGroup typeGroup, EntitySearchDirection direction) { - this(from, to, type, typeGroup, direction, null); + this(from, to, type, typeGroup, null); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index 6318f1fc9d..ad53164ad7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -20,7 +20,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; -import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -37,12 +36,8 @@ public interface RelationDao { List findAllByFromAndType(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup); - List findByFromAndTypeAndProfile(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup, EntityId profileId); - List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup); - List findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId); - List findAllByTo(TenantId tenantId, EntityId to); List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup); @@ -79,6 +74,4 @@ public interface RelationDao { List findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery relationPathQuery); - List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery query); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index afecca26c1..b2871313ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntitySearchDirection; -import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery; import org.thingsboard.server.common.data.relation.RelationPathLevel; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -42,12 +41,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; -import static org.thingsboard.server.dao.model.ModelConstants.ASSET_TABLE_NAME; -import static org.thingsboard.server.dao.model.ModelConstants.DEVICE_TABLE_NAME; import static org.thingsboard.server.dao.model.ModelConstants.RELATION_FROM_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.RELATION_FROM_TYPE_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.RELATION_TABLE_NAME; @@ -107,11 +103,6 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } - @Override - public List findByFromAndTypeAndProfile(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup, EntityId profileId) { - return DaoUtil.convertDataList(relationRepository.findByFromAndProfile(from.getId(), from.getEntityType().name(), typeGroup.name(), relationType, profileId.getId())); - } - @Override public List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup) { return DaoUtil.convertDataList( @@ -121,17 +112,6 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } - @Override - public List findByToAndTypeAndProfile(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup, EntityId profileId) { - return DaoUtil.convertDataList( - relationRepository.findByToAndProfile( - to.getId(), - to.getEntityType().name(), - typeGroup.name(), - relationType, - profileId.getId())); - } - @Override public List findAllByTo(TenantId tenantId, EntityId to) { return DaoUtil.convertDataList( @@ -412,92 +392,4 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple return sb.toString(); } - @Override - public List findByProfileEntityRelationPathQuery(TenantId tenantId, ProfileEntityRelationPathQuery query) { - String sql = buildProfileEntityRelationPathSql(query); - Object[] params = buildProfileEntityRelationPathParams(query); - - log.trace("[{}] profile entity relation path query: {}", tenantId, sql); - - return jdbcTemplate.queryForList(sql, params).stream() - .map(row -> { - var entityRelation = new EntityRelation(); - var fromId = (UUID) row.get(RELATION_FROM_ID_PROPERTY); - var fromType = (String) row.get(RELATION_FROM_TYPE_PROPERTY); - var toId = (UUID) row.get(RELATION_TO_ID_PROPERTY); - var toType = (String) row.get(RELATION_TO_TYPE_PROPERTY); - var grp = (String) row.get(RELATION_TYPE_GROUP_PROPERTY); - var type = (String) row.get(RELATION_TYPE_PROPERTY); - var version = (Long) row.get(VERSION_COLUMN); - - entityRelation.setFrom(EntityIdFactory.getByTypeAndUuid(fromType, fromId)); - entityRelation.setTo(EntityIdFactory.getByTypeAndUuid(toType, toId)); - entityRelation.setType(type); - entityRelation.setTypeGroup(RelationTypeGroup.valueOf(grp)); - entityRelation.setVersion(version); - return entityRelation; - }) - .collect(Collectors.toList()); - } - - private Object[] buildProfileEntityRelationPathParams(ProfileEntityRelationPathQuery query) { - final List params = new ArrayList<>(); - - params.add(query.rootEntityId().getId()); - params.add(query.rootEntityId().getEntityType().name()); - - params.add(query.level().relationType()); - - if (query.targetEntityProfileId() != null) { - params.add(query.targetEntityProfileId().getId()); - params.add(query.targetEntityProfileId().getId()); - } - - return params.toArray(); - } - - private static String buildProfileEntityRelationPathSql(ProfileEntityRelationPathQuery query) { - EntitySearchDirection direction = query.level().direction(); - - StringBuilder sb = new StringBuilder(); - - sb.append("\n") - .append("SELECT r.from_id, r.from_type, r.to_id, r.to_type,\n") - .append(" r.relation_type_group, r.relation_type, r.version\n") - .append("FROM ").append(RELATION_TABLE_NAME).append(" r\n"); - - sb.append("JOIN ").append(DEVICE_TABLE_NAME).append(" d ON "); - if (EntitySearchDirection.FROM == direction) { - sb.append("r.to_id = d.id AND r.to_type = 'DEVICE'").append("\n"); - } else { - sb.append("r.from_id = d.id AND r.from_type = 'DEVICE'").append("\n"); - } - - sb.append("JOIN ").append(ASSET_TABLE_NAME).append(" a ON "); - if (EntitySearchDirection.FROM == direction) { - sb.append("r.to_id = a.id AND r.to_type = 'ASSET'").append("\n"); - } else { - sb.append("r.from_id = a.id AND r.from_type = 'ASSET'").append("\n"); - } - - if (EntitySearchDirection.FROM == direction) { - sb.append("WHERE r.from_id = ?").append("\n") - .append("AND r.from_type = ?").append("\n"); - } else { - sb.append("WHERE r.to_id = ?").append("\n") - .append("AND r.to_type = ?").append("\n"); - } - - sb.append("AND r.relation_type = ?").append("\n") - .append("AND r.relation_type_group = '").append(RelationTypeGroup.COMMON).append("'\n"); - - if (query.targetEntityProfileId() != null) { - sb.append("AND ((d.device_profile_id = ?) OR (a.asset_profile_id = ?))").append("\n"); - } - - sb.append("AND (d.id IS NOT NULL OR a.id IS NOT NULL)"); - - return sb.toString(); - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index 0ebd5b6ceb..4b879f9d95 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -27,7 +27,6 @@ import org.thingsboard.server.dao.model.sql.RelationCompositeKey; import org.thingsboard.server.dao.model.sql.RelationEntity; import java.util.List; -import java.util.Optional; import java.util.UUID; public interface RelationRepository @@ -97,39 +96,4 @@ public interface RelationRepository @Param("toType") String toType, @Param("batchSize") int batchSize); - @Query(value = """ - SELECT r.from_id, r.from_type, r.relation_type_group, r.relation_type, r.to_id, r.to_type, r.additional_info, r.version - FROM relation r - LEFT JOIN device d ON r.to_id = d.id AND r.to_type = 'DEVICE' - LEFT JOIN asset a ON r.to_id = a.id AND r.to_type = 'ASSET' - WHERE r.from_id = :fromId - AND r.from_type = :fromType - AND r.relation_type = :relationType - AND r.relation_type_group = :relationTypeGroup - AND ((d.device_profile_id = :profileId) OR (a.asset_profile_id = :profileId)) - AND (d.id IS NOT NULL OR a.id IS NOT NULL) - """, nativeQuery = true) - List findByFromAndProfile(@Param("fromId") UUID fromId, - @Param("fromType") String fromType, - @Param("relationTypeGroup") String relationTypeGroup, - @Param("relationType") String relationType, - @Param("profileId") UUID profileId); - - @Query(value = """ - SELECT r.from_id, r.from_type, r.relation_type_group, r.relation_type, r.to_id, r.to_type, r.additional_info, r.version - FROM relation r - LEFT JOIN device d ON r.from_id = d.id AND r.from_type = 'DEVICE' - LEFT JOIN asset a ON r.from_id = a.id AND r.from_type = 'ASSET' - WHERE r.to_id = :toId - AND r.to_type = :toType - AND r.relation_type = :relationType - AND r.relation_type_group = :relationTypeGroup - AND ((d.device_profile_id = :profileId) OR (a.asset_profile_id = :profileId)) - AND (d.id IS NOT NULL OR a.id IS NOT NULL) - """, nativeQuery = true) - List findByToAndProfile(@Param("toId") UUID toId, - @Param("toType") String toType, - @Param("relationTypeGroup") String relationTypeGroup, - @Param("relationType") String relationType, - @Param("profileId") UUID profileId); } From 25ac2f2b487c48723b0f7d4c6ae8837b6b6de14e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 16 Oct 2025 16:32:35 +0300 Subject: [PATCH 46/88] added more tests and changed state proto --- .../cf/ctx/state/CalculatedFieldCtx.java | 15 +- ...ValuesAggregationCalculatedFieldState.java | 5 +- .../service/cf/ctx/state/aggregation/agg.json | 65 ------ .../server/utils/CalculatedFieldUtils.java | 4 + ...tValuesAggregationCalculatedFieldTest.java | 188 +++++++++++++++++- common/proto/src/main/proto/queue.proto | 1 + 6 files changed, 196 insertions(+), 82 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 3a9d75b3d4..f69d611b64 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -581,8 +581,7 @@ public class CalculatedFieldCtx { } if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig && other.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig - && thisConfig.getDeduplicationIntervalMillis() != otherConfig.getDeduplicationIntervalMillis() - && !thisConfig.getMetrics().equals(otherConfig.getMetrics())) { + && (thisConfig.getDeduplicationIntervalMillis() != otherConfig.getDeduplicationIntervalMillis() || !thisConfig.getMetrics().equals(otherConfig.getMetrics()))) { return true; } return false; @@ -596,7 +595,7 @@ public class CalculatedFieldCtx { var thisConfig = (AlarmCalculatedFieldConfiguration) calculatedField.getConfiguration(); var otherConfig = (AlarmCalculatedFieldConfiguration) other.getCalculatedField().getConfiguration(); if (!thisConfig.getCreateRules().equals(otherConfig.getCreateRules()) || - !Objects.equals(thisConfig.getClearRule(), otherConfig.getClearRule())) { + !Objects.equals(thisConfig.getClearRule(), otherConfig.getClearRule())) { return true; } } @@ -611,7 +610,7 @@ public class CalculatedFieldCtx { private boolean hasGeofencingZoneGroupConfigurationChanges(CalculatedFieldCtx other) { if (calculatedField.getConfiguration() instanceof GeofencingCalculatedFieldConfiguration thisConfig - && other.calculatedField.getConfiguration() instanceof GeofencingCalculatedFieldConfiguration otherConfig) { + && other.calculatedField.getConfiguration() instanceof GeofencingCalculatedFieldConfiguration otherConfig) { return !thisConfig.getZoneGroups().equals(otherConfig.getZoneGroups()); } return false; @@ -663,10 +662,10 @@ public class CalculatedFieldCtx { @Override public String toString() { return "CalculatedFieldCtx{" + - "cfId=" + cfId + - ", cfType=" + cfType + - ", entityId=" + entityId + - '}'; + "cfId=" + cfId + + ", cfType=" + cfType + + ", entityId=" + entityId + + '}'; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java index 1df27b4cbd..6ae023ed1e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java @@ -48,6 +48,7 @@ import java.util.Map.Entry; @Getter public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedFieldState { + @Setter private long lastArgsRefreshTs = -1; @Setter private long lastMetricsEvalTs = -1; @@ -74,6 +75,7 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF lastArgsRefreshTs = -1; lastMetricsEvalTs = -1; metrics = null; + inputs.clear(); } @Override @@ -104,7 +106,8 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF @Override public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception { - if (!shouldRecalculate()) { + boolean shouldRecalculate = updatedArgs == null || updatedArgs.isEmpty(); + if (!shouldRecalculate() && !shouldRecalculate) { return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() .result(null) .build()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json deleted file mode 100644 index b39092ca74..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/agg.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "type": "LATEST_VALUES_AGGREGATION", - "name": "Occupied spaces", - "debugSettings": { - "failuresEnabled": true, - "allEnabled": true, - "allEnabledUntil": 1769907492297 - }, - "entityId": { - "entityType": "ASSET_PROFILE", - "id": "bb8ddd40-a8bc-11f0-869b-e9d81fa6eaf1" - }, - "configuration": { - "type": "LATEST_VALUES_AGGREGATION", - "source": { - "relation": { - "direction": "FROM", - "relationType": "Contains" - }, - "entityProfiles": [ - { - "entityType": "DEVICE_PROFILE", - "id": "d7a05580-a4cf-11f0-87cb-2d6683c4fccf" - } - ] - }, - "inputs": { - "oc": { - "key": "occupied", - "type": "TS_LATEST" - } - }, - "deduplicationIntervalMillis": 10000, - "metrics": { - "totalSpaces": { - "function": "COUNT", - "input": { - "type": "function", - "function" : "return 1;" - } - }, - "occupiedSpaces": { - "function": "COUNT", - "filter": "return oc == true", - "input": { - "type": "key", - "key" : "oc" - } - }, - "freeSpaces": { - "function": "COUNT", - "filter": "return oc == false", - "input": { - "type": "key", - "key" : "oc" - } - } - }, - "output": { - "type": "TIME_SERIES", - "decimals": 2 - }, - "useLatestTsFromInputs": "true" - } -} diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 19ad4bfb7c..00f4cbde0f 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -96,6 +96,9 @@ public class CalculatedFieldUtils { .setId(toProto(stateId)) .setType(state.getType().name()); + if (state instanceof LatestValuesAggregationCalculatedFieldState aggState) { + builder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); + } state.getArguments().forEach((argName, argEntry) -> { if (argEntry instanceof AggArgumentEntry aggArgumentEntry) { aggArgumentEntry.getAggInputs() @@ -242,6 +245,7 @@ public class CalculatedFieldUtils { arguments.forEach((argName, entityInputs) -> { aggState.getArguments().put(argName, new AggArgumentEntry(entityInputs, false)); }); + aggState.setLastArgsRefreshTs(proto.getLastArgsUpdateTs()); } } diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java index a4b43e362b..8c2c2ca68c 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -15,10 +15,13 @@ */ package org.thingsboard.server.cf; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.Tenant; @@ -61,6 +64,7 @@ import static org.awaitility.Awaitility.await; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.cf.CalculatedFieldIntegrationTest.POLL_INTERVAL; +@Slf4j @DaoSqlTest public class LatestValuesAggregationCalculatedFieldTest extends AbstractControllerTest { @@ -341,13 +345,17 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset2.getId(), device3.getId(), "Contains"); createEntityRelation(asset2.getId(), device4.getId(), "Contains"); - postTelemetry(device3.getId(), "{\"occupied\":false}"); - postTelemetry(device4.getId(), "{\"occupied\":true}"); - postTelemetry(device3.getId(), "{\"occupied\":true}"); + long currentTime = System.currentTimeMillis(); + long firstTs = currentTime - 10; + long secondTs = currentTime - 10; + long thirdTs = currentTime - 5; + postTelemetry(device3.getId(), "{\"ts\": " + firstTs + ", \"values\": {\"occupied\":true}}"); + postTelemetry(device4.getId(), "{\"ts\": " + secondTs + ", \"values\": {\"occupied\":true}}"); + postTelemetry(device3.getId(), "{\"ts\": " + thirdTs + ", \"values\": {\"occupied\":true}}"); createOccupancyCF(asset2.getId()); - await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -357,15 +365,15 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll )); }); - doDelete("/api/plugins/telemetry/DEVICE/" + device3.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=0&endTs=" + System.currentTimeMillis(), String.class); - doDelete("/api/plugins/telemetry/DEVICE/" + device4.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=0&endTs=" + System.currentTimeMillis(), String.class); + doDelete("/api/plugins/telemetry/DEVICE/" + device3.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=" + thirdTs + "&endTs=" + thirdTs + 1, String.class); + doDelete("/api/plugins/telemetry/DEVICE/" + device4.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=" + secondTs + "&endTs=" + secondTs + 1, String.class); await().alias("delete latest telemetry and perform aggregation with previous or default values").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( - "freeSpaces", "2", - "occupiedSpaces", "0", + "freeSpaces", "1", + "occupiedSpaces", "1", "totalSpaces", "2" )); }); @@ -411,6 +419,140 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll }); } + @Test + public void testUpdateRelationPath_checkAggregation() throws Exception { + CalculatedField cf = createOccupancyCF(asset.getId()); + checkInitialCalculation(); + + Device device3 = createDevice("Device 3", "1234567890333"); + createEntityRelation(asset.getId(), device3.getId(), "Has"); + postTelemetry(device3.getId(), "{\"occupied\":true}"); + + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + configuration.setRelation(new RelationPathLevel(EntitySearchDirection.FROM, "Has")); + saveCalculatedField(cf); + + await().alias("update relation path and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "0", + "occupiedSpaces", "1", + "totalSpaces", "1" + )); + }); + } + + @Test + public void testUpdateArguments_checkAggregation() throws Exception { + CalculatedField cf = createOccupancyCF(asset.getId()); + checkInitialCalculation(); + + postTelemetry(device1.getId(), "{\"occupiedStatus\":false}"); + postTelemetry(device2.getId(), "{\"occupiedStatus\":false}"); + + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + Argument argument = new Argument(); + argument.setRefEntityKey(new ReferencedEntityKey("oc", ArgumentType.TS_LATEST, null)); + argument.setDefaultValue("false"); + configuration.setArguments(Map.of("oc", argument)); + saveCalculatedField(cf); + + await().alias("update arguments and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); + }); + } + + @Test + public void testUpdateMetrics_checkAggregation() throws Exception { + postTelemetry(device1.getId(), "{\"temperature\":24.2}"); + postTelemetry(device2.getId(), "{\"temperature\":19.6}"); + CalculatedField cf = createAvgTemperatureCF(asset.getId()); + + await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); + }); + + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + AggMetric aggMetric = new AggMetric(); + aggMetric.setInput(new AggKeyInput("temp")); + aggMetric.setFilter("return temp < 100;"); + aggMetric.setFunction(AggFunction.MAX); + configuration.setMetrics(Map.of("maxTemperature", aggMetric)); + saveCalculatedField(cf); + + postTelemetry(device1.getId(), "{\"temperature\":101.3}"); + postTelemetry(device2.getId(), "{\"temperature\":25.8}"); + + await().alias("update metrics and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("maxTemperature", "26")); + }); + } + + @Test + public void testUpdateOutput_checkAggregation() throws Exception { + postTelemetry(device1.getId(), "{\"temperature\":24.2}"); + postTelemetry(device2.getId(), "{\"temperature\":19.6}"); + CalculatedField cf = createAvgTemperatureCF(asset.getId()); + + await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); + }); + + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + Output output = new Output(); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + configuration.setOutput(output); + saveCalculatedField(cf); + + await().alias("update output and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode avgTemperature = getServerAttributes(asset.getId(), "avgTemperature"); + assertThat(avgTemperature).isNotNull(); + assertThat(avgTemperature.get(0)).isNotNull(); + assertThat(avgTemperature.get(0).get("value").asText()).isEqualTo("24.2"); + }); + } + + @Test + public void testUpdateDeduplicationInterval_checkAggregationNotExecutedUntilDeduplicationInterval() throws Exception { + postTelemetry(device1.getId(), "{\"temperature\":24.2}"); + postTelemetry(device2.getId(), "{\"temperature\":19.6}"); + CalculatedField cf = createAvgTemperatureCF(asset.getId()); + + await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); + }); + + var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + configuration.setDeduplicationIntervalMillis(2 * deduplicationInterval); + saveCalculatedField(cf); + + postTelemetry(device2.getId(), "{\"temperature\":32.1}"); + + await().alias("update deduplication interval and perform aggregation").atMost(2 * deduplicationInterval, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("avgTemperature", "28")); + }); + } + private void checkInitialCalculation() { await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) @@ -425,6 +567,32 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll assertThat(occupancy.get("totalSpaces").get(0).get("value").asText()).isEqualTo("2"); } + private CalculatedField createAvgTemperatureCF(EntityId entityId) { + Map arguments = new HashMap<>(); + Argument argument = new Argument(); + argument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null)); + argument.setDefaultValue("20"); + arguments.put("temp", argument); + + Map aggMetrics = new HashMap<>(); + + AggMetric avgMetric = new AggMetric(); + avgMetric.setFunction(AggFunction.AVG); + avgMetric.setFilter("return temp >= 20;"); + avgMetric.setInput(new AggKeyInput("temp")); + aggMetrics.put("avgTemperature", avgMetric); + + Output output = new Output(); + output.setType(OutputType.TIME_SERIES); + output.setDecimalsByDefault(0); + + return createAggCf("Average temperature", entityId, + new RelationPathLevel(EntitySearchDirection.FROM, "Contains"), + arguments, + aggMetrics, + output); + } + private CalculatedField createOccupancyCF(EntityId entityId) { Map arguments = new HashMap<>(); Argument argument = new Argument(); @@ -513,4 +681,8 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); } + private ArrayNode getServerAttributes(EntityId entityId, String... keys) throws Exception { + return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/attributes/SERVER_SCOPE?keys=" + String.join(",", keys), ArrayNode.class); + } + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 764f4c3ec9..1e01916e55 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -928,6 +928,7 @@ message CalculatedFieldStateProto { repeated GeofencingArgumentProto geofencingArguments = 5; AlarmStateProto alarmState = 6; repeated AggSingleArgumentEntryProto aggArguments = 7; + int64 lastArgsUpdateTs = 8; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. From d96a69de925655338d8947845c7289906aeadcca Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 16 Oct 2025 17:32:11 +0300 Subject: [PATCH 47/88] fixed firmware update when ota package has url instead of file --- .../ota/DefaultOtaPackageStateService.java | 2 +- .../controller/DeviceControllerTest.java | 43 +++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/ota/DefaultOtaPackageStateService.java b/application/src/main/java/org/thingsboard/server/service/ota/DefaultOtaPackageStateService.java index 4d9e145711..0b4d3bec6f 100644 --- a/application/src/main/java/org/thingsboard/server/service/ota/DefaultOtaPackageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/ota/DefaultOtaPackageStateService.java @@ -328,7 +328,7 @@ public class DefaultOtaPackageStateService implements OtaPackageStateService { attributes.add(new BaseAttributeKvEntry(ts, new LongDataEntry(getAttributeKey(otaPackageType, SIZE), otaPackage.getDataSize()))); } - if (otaPackage.getChecksumAlgorithm() != null) { + if (otaPackage.getChecksumAlgorithm() == null) { attrToRemove.add(getAttributeKey(otaPackageType, CHECKSUM_ALGORITHM)); } else { attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(getAttributeKey(otaPackageType, CHECKSUM_ALGORITHM), otaPackage.getChecksumAlgorithm().name()))); diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java index 36cede9e84..4551e530d2 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java @@ -81,6 +81,7 @@ import org.thingsboard.server.service.state.DeviceStateService; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; @@ -387,6 +388,48 @@ public class DeviceControllerTest extends AbstractControllerTest { .andExpect(statusReason(containsString("Device can`t be referencing to device profile from different tenant!"))); } + @Test + public void testSaveDeviceWithFirmware() throws Exception { + loginTenantAdmin(); + DeviceProfile profile = createDeviceProfile("Profile to test ota updates"); + profile = doPost("/api/deviceProfile", profile, DeviceProfile.class); + + SaveOtaPackageInfoRequest firmwareInfo = new SaveOtaPackageInfoRequest(); + firmwareInfo.setDeviceProfileId(profile.getId()); + firmwareInfo.setType(FIRMWARE); + String title = "title"; + firmwareInfo.setTitle(title); + String fwVersion = "1.0"; + firmwareInfo.setVersion(fwVersion); + String url = "test.url"; + firmwareInfo.setUrl(url); + firmwareInfo.setUsesUrl(true); + OtaPackageInfo savedFw = doPost("/api/otaPackage", firmwareInfo, OtaPackageInfo.class); + + Device device = new Device(); + device.setName("My ota device"); + device.setDeviceProfileId(profile.getId()); + device.setFirmwareId(savedFw.getId()); + device = doPost("/api/device", device, Device.class); + + //check shared attributes + Device finalDevice = device; + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> { + List> attributes = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + finalDevice.getId() + + "/values/attributes/SHARED_SCOPE", new TypeReference>>() { + }); + return findAttrValue("fw_version", attributes).equals(fwVersion) && + findAttrValue("fw_title", attributes).equals(title) && + findAttrValue("fw_url", attributes).equals(url); + }); + } + + private static Object findAttrValue(String key, List> attributes) { + Optional> attr = attributes.stream() + .filter(att -> att.get("key").equals(key)).findFirst(); + return attr.isPresent() ? attr.get().get("value") : ""; + } + @Test public void testSaveDeviceWithFirmwareFromDifferentTenant() throws Exception { loginDifferentTenant(); From c593f3e95b48fdfa15c6fde9697d5ba266f2e802 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Thu, 16 Oct 2025 17:43:21 +0300 Subject: [PATCH 48/88] lwm2m: bootstrap new: add reboot device and reboot device bootstrap --- .../lwm2m/Lwm2mServerIdentifier.java | 25 ++-- ...LwM2MBootstrapConfigStoreTaskProvider.java | 2 +- .../validator/DeviceProfileDataValidator.java | 6 +- .../device-credentials-lwm2m.component.html | 14 +- .../device-credentials-lwm2m.component.ts | 128 +++++++++++++++++- .../device/device-credentials.component.html | 3 +- .../device/device-credentials.component.ts | 4 + .../lwm2m-device-config-server.component.html | 4 +- .../lwm2m-device-config-server.component.ts | 6 +- .../assets/locale/locale.constant-en_US.json | 4 +- 10 files changed, 164 insertions(+), 32 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/credentials/lwm2m/Lwm2mServerIdentifier.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/credentials/lwm2m/Lwm2mServerIdentifier.java index 95aa95597f..f4f020776b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/credentials/lwm2m/Lwm2mServerIdentifier.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/credentials/lwm2m/Lwm2mServerIdentifier.java @@ -47,11 +47,11 @@ public enum Lwm2mServerIdentifier { */ NOT_USED_IDENTIFYING_LWM2M_SERVER_MAX(65535, "Reserved sentinel value (no active server)", false); - private final int id; + private final Integer id; private final String description; private final boolean isLwm2mServer; - Lwm2mServerIdentifier(int id, String description, boolean isLwm2mServer) { + Lwm2mServerIdentifier(Integer id, String description, boolean isLwm2mServer) { this.id = id; this.description = description; this.isLwm2mServer = isLwm2mServer; @@ -60,7 +60,7 @@ public enum Lwm2mServerIdentifier { /** * @return the integer value of this Short Server ID. */ - public int getId() { + public Integer getId() { return id; } @@ -83,20 +83,11 @@ public enum Lwm2mServerIdentifier { * @param id Short Server ID value. * @return true if the ID belongs to a standard LwM2M Server. */ - public static boolean isLwm2mServer(int id) { - return id >= PRIMARY_LWM2M_SERVER.id && id <= LWM2M_SERVER_MAX.id; + public static boolean isLwm2mServer(Integer id) { + return id != null && id >= PRIMARY_LWM2M_SERVER.id && id <= LWM2M_SERVER_MAX.id; } - public static boolean isNotLwm2mServer(int id) { - return id < PRIMARY_LWM2M_SERVER.id || id > LWM2M_SERVER_MAX.id; - } - - /** - * Checks whether the provided ID is within the valid LwM2M range [0–65535]. - * @param id ID to check. - * @return true if valid, false otherwise. - */ - public static boolean isValid(int id) { - return id >= NOT_USED_IDENTIFYING_LWM2M_SERVER_MIN.getId() && id <= NOT_USED_IDENTIFYING_LWM2M_SERVER_MAX.getId(); + public static boolean isNotLwm2mServer(Integer id) { + return id == null || id < PRIMARY_LWM2M_SERVER.id || id > LWM2M_SERVER_MAX.id; } /** @@ -105,7 +96,7 @@ public enum Lwm2mServerIdentifier { * @return corresponding enum constant. * @throws IllegalArgumentException if no constant matches the given ID. */ - public static Lwm2mServerIdentifier fromId(int id) { + public static Lwm2mServerIdentifier fromId(Integer id) { for (Lwm2mServerIdentifier s : values()) { if (s.id == id) { return s; diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java index 7020869130..67f4aa32f6 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java @@ -147,7 +147,7 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask log.error("Invalid lwm2mSecurityInstance [{}] by short server id [{}]", path.getObjectInstanceId(), lwm2mShortServerId); } } else { - this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().putIfAbsent(0, path.getObjectInstanceId()); + this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().putIfAbsent(null, path.getObjectInstanceId()); } } else if (path.getObjectId() == 1) { if (link.getAttributes().get("ssid") != null) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java index ac600f9201..401b199598 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java @@ -343,7 +343,11 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator - + @@ -72,6 +72,12 @@ device.lwm2m-security-config.client-public-key-hint + @@ -103,5 +109,11 @@ + diff --git a/ui-ngx/src/app/modules/home/components/device/device-credentials-lwm2m.component.ts b/ui-ngx/src/app/modules/home/components/device/device-credentials-lwm2m.component.ts index ca91c14682..ec9f8ff5fb 100644 --- a/ui-ngx/src/app/modules/home/components/device/device-credentials-lwm2m.component.ts +++ b/ui-ngx/src/app/modules/home/components/device/device-credentials-lwm2m.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, forwardRef, OnDestroy } from '@angular/core'; +import {Component, forwardRef, Input, OnDestroy} from '@angular/core'; import { ControlValueAccessor, UntypedFormBuilder, @@ -32,9 +32,12 @@ import { Lwm2mSecurityType, Lwm2mSecurityTypeTranslationMap } from '@shared/models/lwm2m-security-config.models'; -import { Subject } from 'rxjs'; -import { takeUntil } from 'rxjs/operators'; +import {Subject, throwError, timeout, catchError, of} from 'rxjs'; +import {map, takeUntil} from 'rxjs/operators'; import { isDefinedAndNotNull } from '@core/utils'; +import { HttpClient } from '@angular/common/http'; +import {DeviceId} from "@shared/models/id/device-id"; +import {Observable} from "rxjs/internal/Observable"; @Component({ selector: 'tb-device-credentials-lwm2m', @@ -65,7 +68,11 @@ export class DeviceCredentialsLwm2mComponent implements ControlValueAccessor, Va private destroy$ = new Subject(); private propagateChange = (v: any) => {}; - constructor(private fb: UntypedFormBuilder) { + @Input() + deviceId: DeviceId; + + constructor(private fb: UntypedFormBuilder, + private http: HttpClient) { this.lwm2mConfigFormGroup = this.initLwm2mConfigForm(); } @@ -101,6 +108,119 @@ export class DeviceCredentialsLwm2mComponent implements ControlValueAccessor, Va this.destroy$.complete(); } + /** + * AbstractRpcController -> rpcController + * - API + * "/api/plugins/rpc/twoway/${this.deviceId.id}" + * - DiscoveryAll + * requestBody = "{\"method\":\"DiscoverAll\"}"; + * - "Registration Update Trigger", + * requestBody = "{\"method\": \"Execute\", \"params\": {\"id\": \"/1_1.2/0/8\"}} + * - "Bootstrap-Request Trigger" + * requestBody = "{\"method\": \"Execute\", \"params\": {\"id\": \"/1_1.2/0/9\"}} + */ + public rebootDevice(isBootstrapServer: boolean): void { + const urlApi = `/api/plugins/rpc/twoway/${this.deviceId.id}`; + // DiscoveryAll} + this.http.post(urlApi, { method: "DiscoverAll" }) + .pipe( + timeout(10000), // 10 sec + catchError(err => { + console.error('DiscoverAll timeout or error', err); + return throwError(() => err); + }) + ) + .subscribe({ + next: (response: any) => { + console.log('success: Discovery'); + console.log(response); + // result = 'CONTENT' + if (response.result && response.result.toUpperCase() === 'CONTENT') { + const verId = this.getVerId(response.value); + console.log("ObjectId=1 ver:", verId); + const resourceId = isBootstrapServer ? 9 : 8; + const resourcePath = `/1_${verId}/0/${resourceId}`; + + // first rebootTrigger + this.rebootTrigger(resourcePath, urlApi).subscribe(first => { + if (first.result === 'CHANGED') { + console.log('Reboot success first'); + } + else if (first.result === 'BAD_REQUEST' && first.newVersionId && first.newVersionId !== verId) { + // Retry with new version + const correctedPath = `/1_${first.newVersionId}/0/${resourceId}`; + console.log(`Retrying with version ${first.newVersionId}`); + + this.rebootTrigger(correctedPath, urlApi).subscribe(second => { + if (second.result === 'CHANGED') { + console.log('Success reboot after retry'); + } else { + console.error(`error1: Reboot second failed: ${second.toString()}`); + } + }); + } else { + console.error(`error2: Reboot first failed: ${first.toString()}`); + } + }); + } + + else { + console.error(`error3: Bad registration device with id = ${this.deviceId.id} ❗ RPC result is not CONTENT`); + } + }, + error: (e) => { + console.error(`error4: Bad registration device with id = ${this.deviceId.id} ${e.toString()}`); + return throwError(() => e); + }, + complete: () => { + console.log('Discovery stream complete'); } + }); + } + + private getVerId(value: string): string { + const verDef = '1.1'; + try { + const arr = JSON.parse(value); + if (!Array.isArray(arr)) return verDef; + const obj1 = arr.find((s: string) => s.startsWith(' { + console.log(`Sending reboot command to ${resourcePath}`); + + return this.http.post(urlApi, { + method: 'Execute', + params: { id: resourcePath } + }).pipe( + timeout(10000), + map(res => { + console.log(`Reboot for ${resourcePath}`); + console.log(res); + if (res?.result?.toUpperCase() === 'CHANGED') { + return { result: 'CHANGED' }; + } + + if (res?.result?.toUpperCase() === 'BAD_REQUEST' && res?.error) { + const match = (res.error as string).match(/version[:=]\s*([\d.]+)/i); + const newVersionId = match ? match[1] : null; + console.warn(`BAD_REQUEST: suggested version ${newVersionId ?? 'unknown'}`); + return { result: 'BAD_REQUEST', newVersionId }; + } + + return { result: 'ERROR' }; + }), + catchError(err => { + console.error(`Execute error5 for ${resourcePath}:`, err); + return of({ result: 'ERROR' }); + }) + ); + } + private initClientSecurityConfig(config: Lwm2mSecurityConfigModels): void { this.lwm2mConfigFormGroup.patchValue(config, {emitEvent: false}); this.securityConfigClientUpdateValidators(config.client.securityConfigClientMode); diff --git a/ui-ngx/src/app/modules/home/components/device/device-credentials.component.html b/ui-ngx/src/app/modules/home/components/device/device-credentials.component.html index 34c446f759..144f2059c3 100644 --- a/ui-ngx/src/app/modules/home/components/device/device-credentials.component.html +++ b/ui-ngx/src/app/modules/home/components/device/device-credentials.component.html @@ -81,7 +81,8 @@ - + diff --git a/ui-ngx/src/app/modules/home/components/device/device-credentials.component.ts b/ui-ngx/src/app/modules/home/components/device/device-credentials.component.ts index 012bdf9c9c..9aa2c2fa2d 100644 --- a/ui-ngx/src/app/modules/home/components/device/device-credentials.component.ts +++ b/ui-ngx/src/app/modules/home/components/device/device-credentials.component.ts @@ -36,6 +36,7 @@ import { Subject } from 'rxjs'; import { takeUntil } from 'rxjs/operators'; import { generateSecret, isDefinedAndNotNull } from '@core/utils'; import { coerceBoolean } from '@shared/decorators/coercion'; +import {DeviceId} from "@shared/models/id/device-id"; @Component({ selector: 'tb-device-credentials', @@ -88,6 +89,8 @@ export class DeviceCredentialsComponent implements ControlValueAccessor, OnInit, credentialTypeNamesMap = credentialTypeNames; + deviceId: DeviceId; + private propagateChange = null; private propagateChangePending = false; @@ -126,6 +129,7 @@ export class DeviceCredentialsComponent implements ControlValueAccessor, OnInit, writeValue(value: DeviceCredentials | null): void { if (isDefinedAndNotNull(value)) { + this.deviceId = value.deviceId; const credentialsType = this.credentialsTypes.includes(value.credentialsType) ? value.credentialsType : this.credentialsTypes[0]; this.deviceCredentialsFormGroup.patchValue({ credentialsType, diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.html b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.html index 65a2400e16..a6ef4c0847 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.html @@ -24,7 +24,7 @@ 'device-profile.lwm2m.bootstrap-server' : 'device-profile.lwm2m.lwm2m-server') | translate }}
{{ ('device-profile.lwm2m.short-id' | translate) + ': ' }} - {{ serverFormGroup.get('shortServerId').value }} + {{ serverFormGroup.get('shortServerId').value ? serverFormGroup.get('shortServerId').value : '' }}
{{ ('device-profile.lwm2m.mode' | translate) + ': ' }} {{ credentialTypeLwM2MNamesMap.get(securityConfigLwM2MType[serverFormGroup.get('securityMode').value]) }} @@ -54,7 +54,7 @@ - + {{ 'device-profile.lwm2m.short-id' | translate }} help diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts index 71f4264f94..48be7ff3f4 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts @@ -76,7 +76,6 @@ export class Lwm2mDeviceConfigServerComponent implements OnInit, ControlValueAcc readonly shortServerIdMin = 1; readonly shortServerIdMax = 65534; - readonly shortServerIdBs = 0; @Input() @coerceBoolean() @@ -101,9 +100,8 @@ export class Lwm2mDeviceConfigServerComponent implements OnInit, ControlValueAcc securityMode: [Lwm2mSecurityType.NO_SEC], serverPublicKey: [''], clientHoldOffTime: ['', [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], - shortServerId: ['', this.isBootstrap - ? [Validators.required, Validators.pattern('^(' + this.shortServerIdBs + ')$' )] - : [Validators.required, Validators.pattern('[0-9]*'), Validators.min(this.shortServerIdMin), Validators.max(this.shortServerIdMax)] + shortServerId: ['', this.isBootstrap ? + [] : [Validators.required, Validators.pattern('[0-9]*'), Validators.min(this.shortServerIdMin), Validators.max(this.shortServerIdMax)] ], bootstrapServerAccountTimeout: ['', [Validators.required, Validators.min(0), Validators.pattern('[0-9]*')]], binding: [''], diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 912eeedc5f..a3fcc843a1 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1801,6 +1801,8 @@ "bootstrap-tab": "Bootstrap Client", "bootstrap-server": "Bootstrap Server", "lwm2m-server": "LwM2M Server", + "client-reboot": "Registration Update Trigger", + "bootstrap-reboot": "Bootstrap-Request Trigger", "client-publicKey-or-id": "Client Public Key or Id", "client-publicKey-or-id-required": "Client Public Key or Id is required.", "client-publicKey-or-id-tooltip-psk": "The PSK identifier is an arbitrary PSK identifier up to 128 bytes, as described in the standard [RFC7925].\nThe PSK identifier MUST first be converted to a character string and then encoded into octets using UTF-8.", @@ -2302,7 +2304,7 @@ "short-id-required": "Short server ID is required.", "short-id-range": "Short server ID should be in a range from {{ min }} to {{ max }}.", "short-id-pattern": "Short server ID must be a positive integer.", - "short-id-pattern-bs": "Short server ID must be only 0", + "short-id-pattern-bs": "Short server ID must be only null", "lifetime": "Client registration lifetime", "lifetime-required": "Client registration lifetime is required.", "lifetime-pattern": "Client registration lifetime must be a positive integer.", From f951c64767c504142bdaaab1ad69711d995ec73c Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 17 Oct 2025 11:14:07 +0300 Subject: [PATCH 49/88] added state to proto and fixed deduplication logic --- ...ValuesAggregationCalculatedFieldState.java | 23 +++---- .../state/aggregation/function/new_agg.json | 60 ------------------- .../server/utils/CalculatedFieldUtils.java | 16 +++-- ...tValuesAggregationCalculatedFieldTest.java | 16 ++++- common/proto/src/main/proto/queue.proto | 8 ++- 5 files changed, 42 insertions(+), 81 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java index 6ae023ed1e..f3560faafa 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java @@ -106,21 +106,22 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF @Override public ListenableFuture performCalculation(Map updatedArgs, CalculatedFieldCtx ctx) throws Exception { - boolean shouldRecalculate = updatedArgs == null || updatedArgs.isEmpty(); - if (!shouldRecalculate() && !shouldRecalculate) { + boolean cfUpdated = updatedArgs != null && updatedArgs.isEmpty(); + if (shouldRecalculate() || cfUpdated) { + Output output = ctx.getOutput(); + ObjectNode aggResult = aggregateMetrics(output); + lastMetricsEvalTs = System.currentTimeMillis(); + ctx.scheduleReevaluation(deduplicationInterval, actorCtx); + return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() + .type(output.getType()) + .scope(output.getScope()) + .result(createResultJson(ctx.isUseLatestTs(), aggResult)) + .build()); + } else { return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() .result(null) .build()); } - Output output = ctx.getOutput(); - ObjectNode aggResult = aggregateMetrics(output); - lastMetricsEvalTs = System.currentTimeMillis(); - ctx.scheduleReevaluation(deduplicationInterval, actorCtx); - return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() - .type(output.getType()) - .scope(output.getScope()) - .result(createResultJson(ctx.isUseLatestTs(), aggResult)) - .build()); } private boolean shouldRecalculate() { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json deleted file mode 100644 index c6b841b673..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/new_agg.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "type": "LATEST_VALUES_AGGREGATION", - "name": "Occupied spaces", - "debugSettings": { - "failuresEnabled": true, - "allEnabled": true, - "allEnabledUntil": 1769907492297 - }, - "entityId": { - "entityType": "ASSET", - "id": "f8ad0800-a9a6-11f0-bbe6-459b63b420fe" - }, - "configuration": { - "type": "LATEST_VALUES_AGGREGATION", - "relation": { - "direction": "FROM", - "relationType": "Contains" - }, - "arguments": { - "oc": { - "refEntityKey": { - "key": "occupied", - "type": "TS_LATEST" - }, - "defaultValue": "false" - } - }, - "deduplicationIntervalMillis": 20000, - "metrics": { - "totalSpaces": { - "function": "COUNT", - "input": { - "type": "function", - "function" : "return 1;" - } - }, - "occupiedSpaces": { - "function": "COUNT", - "filter": "return oc == true", - "input": { - "type": "key", - "key" : "oc" - } - }, - "freeSpaces": { - "function": "COUNT", - "filter": "return oc == false", - "input": { - "type": "key", - "key" : "oc" - } - } - }, - "output": { - "type": "TIME_SERIES", - "decimals": 2 - }, - "useLatestTsFromInputs": "true" - } -} diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 00f4cbde0f..f8a7d17543 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -35,6 +35,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdPro import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingZoneProto; +import org.thingsboard.server.gen.transport.TransportProtos.LatestValuesAggregationStateProto; import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.TsDoubleValProto; import org.thingsboard.server.gen.transport.TransportProtos.TsRollingArgumentProto; @@ -96,13 +97,11 @@ public class CalculatedFieldUtils { .setId(toProto(stateId)) .setType(state.getType().name()); - if (state instanceof LatestValuesAggregationCalculatedFieldState aggState) { - builder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); - } + LatestValuesAggregationStateProto.Builder aggBuilder = LatestValuesAggregationStateProto.newBuilder(); state.getArguments().forEach((argName, argEntry) -> { if (argEntry instanceof AggArgumentEntry aggArgumentEntry) { aggArgumentEntry.getAggInputs() - .forEach((entityId, entry) -> builder.addAggArguments(toAggSingleArgumentProto(argName, entityId, entry))); + .forEach((entityId, entry) -> aggBuilder.addAggArguments(toAggSingleArgumentProto(argName, entityId, entry))); } else if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { builder.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { @@ -120,6 +119,10 @@ public class CalculatedFieldUtils { alarmStateProto.setClearRuleState(toAlarmRuleStateProto(alarmState.getClearRuleState())); } } + if (state instanceof LatestValuesAggregationCalculatedFieldState aggState) { + aggBuilder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); + builder.setLatestValuesAggregationState(aggBuilder.build()); + } return builder.build(); } @@ -237,15 +240,16 @@ public class CalculatedFieldUtils { } case LATEST_VALUES_AGGREGATION -> { LatestValuesAggregationCalculatedFieldState aggState = (LatestValuesAggregationCalculatedFieldState) state; + LatestValuesAggregationStateProto aggregationStateProto = proto.getLatestValuesAggregationState(); Map> arguments = new HashMap<>(); - proto.getAggArgumentsList().forEach(argProto -> { + aggregationStateProto.getAggArgumentsList().forEach(argProto -> { AggSingleEntityArgumentEntry entry = fromAggSingleValueArgumentProto(argProto); arguments.computeIfAbsent(argProto.getValue().getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); }); arguments.forEach((argName, entityInputs) -> { aggState.getArguments().put(argName, new AggArgumentEntry(entityInputs, false)); }); - aggState.setLastArgsRefreshTs(proto.getLastArgsUpdateTs()); + aggState.setLastArgsRefreshTs(aggregationStateProto.getLastArgsUpdateTs()); } } diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java index 8c2c2ca68c..46b8b78d57 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -489,10 +489,16 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll configuration.setMetrics(Map.of("maxTemperature", aggMetric)); saveCalculatedField(cf); + await().alias("update metrics and perform aggregation").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("maxTemperature", "24")); + }); + postTelemetry(device1.getId(), "{\"temperature\":101.3}"); postTelemetry(device2.getId(), "{\"temperature\":25.8}"); - await().alias("update metrics and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("maxTemperature", "26")); @@ -544,9 +550,15 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll configuration.setDeduplicationIntervalMillis(2 * deduplicationInterval); saveCalculatedField(cf); + await().alias("update deduplication interval and perform aggregation").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); + }); + postTelemetry(device2.getId(), "{\"temperature\":32.1}"); - await().alias("update deduplication interval and perform aggregation").atMost(2 * deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(2 * deduplicationInterval, TimeUnit.MILLISECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("avgTemperature", "28")); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 1e01916e55..b59e01128b 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -920,6 +920,11 @@ message AggSingleArgumentEntryProto { SingleValueArgumentProto value = 2; } +message LatestValuesAggregationStateProto { + int64 lastArgsUpdateTs = 1; + repeated AggSingleArgumentEntryProto aggArguments = 2; +} + message CalculatedFieldStateProto { CalculatedFieldEntityCtxIdProto id = 1; string type = 2; @@ -927,8 +932,7 @@ message CalculatedFieldStateProto { repeated TsRollingArgumentProto rollingValueArguments = 4; repeated GeofencingArgumentProto geofencingArguments = 5; AlarmStateProto alarmState = 6; - repeated AggSingleArgumentEntryProto aggArguments = 7; - int64 lastArgsUpdateTs = 8; + LatestValuesAggregationStateProto latestValuesAggregationState = 7; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. From 6a307041b699f4d2a5d1c27e8755b8538f0fec31 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 17 Oct 2025 15:55:44 +0300 Subject: [PATCH 50/88] added tests for entry --- ...CalculatedFieldEntityMessageProcessor.java | 4 +- ...tractCalculatedFieldProcessingService.java | 4 +- .../DefaultCalculatedFieldQueueService.java | 24 ++-- .../ctx/state/BaseCalculatedFieldState.java | 3 +- .../state/aggregation/AggArgumentEntry.java | 7 +- .../AggSingleEntityArgumentEntry.java | 30 +++-- .../cf/ctx/state/AggArgumentEntryTest.java | 112 ++++++++++++++++++ .../AggSingleEntityArgumentEntryTest.java | 101 ++++++++++++++++ .../server/dao/relation/RelationService.java | 2 + .../data/cf/configuration/Argument.java | 4 + ...gregationCalculatedFieldConfiguration.java | 13 ++ .../dao/relation/BaseRelationService.java | 15 +++ 12 files changed, 288 insertions(+), 31 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java create mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index ce29dc06e0..58113deb08 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -67,7 +67,6 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -686,9 +685,8 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM Map.Entry::getKey, argEntry -> new AggSingleEntityArgumentEntry(entityId, argEntry.getValue()) )); - } else { - fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); } + fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); return fetchedArgs; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 94695bcc8c..7f47dba2ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -332,7 +332,7 @@ public abstract class AbstractCalculatedFieldProcessingService { var attributeOptFuture = attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()); return Futures.transform(attributeOptFuture, attrOpt -> { log.debug("[{}][{}] Fetched attribute for key {}: {}", tenantId, entityId, argument.getRefEntityKey(), attrOpt); - AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, 0L)); + AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, SingleValueArgumentEntry.DEFAULT_VERSION)); return transformAggSingleArgument(entityId, Optional.of(attributeKvEntry)); }, calculatedFieldCallbackExecutor); } @@ -344,7 +344,7 @@ public abstract class AbstractCalculatedFieldProcessingService { timeseriesService.findLatest(tenantId, entityId, key), result -> { log.debug("[{}][{}] Fetched latest timeseries {}: {}", tenantId, entityId, key, result); - Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(defaultTs, createDefaultKvEntry(argument), 0L))); + Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(defaultTs, createDefaultKvEntry(argument), SingleValueArgumentEntry.DEFAULT_VERSION))); return transformAggSingleArgument(entityId, tsKvEntry); }, calculatedFieldCallbackExecutor); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 37c7b2ef6c..147882c3c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -37,8 +37,9 @@ import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationPathLevel; -import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; @@ -191,20 +192,15 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS for (CalculatedFieldCtx cfCtx : cfCtxs) { if (cfCtx.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { RelationPathLevel relation = aggConfig.getRelation(); - switch (relation.direction()) { - case FROM -> { - List byToAndType = relationService.findByToAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); - if (!byToAndType.isEmpty()) { - return true; - } - } - case TO -> { - List byFromAndType = relationService.findByFromAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); - if (!byFromAndType.isEmpty()) { - return true; - } - } + EntitySearchDirection inverseDirection = switch (relation.direction()) { + case FROM -> EntitySearchDirection.TO; + case TO -> EntitySearchDirection.FROM; }; + RelationPathLevel inverseRelation = new RelationPathLevel(inverseDirection, relation.relationType()); + List byRelationPathQuery = relationService.findByRelationPathQuery(tenantId, new EntityRelationPathQuery(entityId, List.of(inverseRelation))); + if (!byRelationPathQuery.isEmpty()) { + return true; + } } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 2d853fd3fd..ee10857e6f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -22,6 +22,7 @@ import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.utils.CalculatedFieldUtils; import java.io.Closeable; @@ -73,7 +74,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, ArgumentEntry existingEntry = arguments.get(key); boolean entryUpdated; - if (existingEntry == null || newEntry.isForceResetPrevious()) { + if (existingEntry == null || !(newEntry instanceof AggSingleEntityArgumentEntry) && newEntry.isForceResetPrevious()) { validateNewEntry(key, newEntry); arguments.put(key, newEntry); entryUpdated = true; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java index 7e3a8623e4..e002aece2c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java @@ -52,7 +52,12 @@ public class AggArgumentEntry implements ArgumentEntry { if (aggSingleEntityArgumentEntry.isDeleted()) { aggInputs.remove(aggSingleEntityArgumentEntry.getEntityId()); } else { - aggInputs.put(aggSingleEntityArgumentEntry.getEntityId(), aggSingleEntityArgumentEntry); + ArgumentEntry argumentEntry = aggInputs.get(aggSingleEntityArgumentEntry.getEntityId()); + if (argumentEntry != null) { + argumentEntry.updateEntry(aggSingleEntityArgumentEntry); + } else { + aggInputs.put(aggSingleEntityArgumentEntry.getEntityId(), aggSingleEntityArgumentEntry); + } } return true; } else { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java index 32ce77311d..6430fe3f1c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java @@ -62,25 +62,35 @@ public class AggSingleEntityArgumentEntry extends SingleValueArgumentEntry { @Override public boolean updateEntry(ArgumentEntry entry) { - if (entry instanceof AggSingleEntityArgumentEntry singleValueEntry) { - if (singleValueEntry.getTs() <= ts) { - return false; + if (entry instanceof AggSingleEntityArgumentEntry aggSingleEntityEntry) { + if (aggSingleEntityEntry.isForceResetPrevious()) { + return applyNewEntry(aggSingleEntityEntry); } - Long newVersion = singleValueEntry.getVersion(); + if (aggSingleEntityEntry.getTs() < this.ts) { + if (!isDefaultValue()) { + return false; + } + } + + Long newVersion = aggSingleEntityEntry.getVersion(); if (newVersion == null || this.version == null || newVersion > this.version) { - this.ts = singleValueEntry.getTs(); - this.version = newVersion; - this.kvEntryValue = singleValueEntry.getKvEntryValue(); - this.entityId = singleValueEntry.getEntityId(); - return true; + return applyNewEntry(aggSingleEntityEntry); } } else { - throw new IllegalArgumentException("Unsupported argument entry type for single value argument entry: " + entry.getType()); + throw new IllegalArgumentException("Unsupported argument entry type for aggregation single entity argument entry: " + entry.getType()); } return false; } + private boolean applyNewEntry(AggSingleEntityArgumentEntry entry) { + this.ts = entry.getTs(); + this.version = entry.getVersion(); + this.kvEntryValue = entry.getKvEntryValue(); + this.entityId = entry.getEntityId(); + return true; + } + @Override public ArgumentEntryType getType() { return ArgumentEntryType.AGGREGATE_LATEST_SINGLE; diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java new file mode 100644 index 0000000000..c1c2d85c55 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java @@ -0,0 +1,112 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class AggArgumentEntryTest { + + private AggArgumentEntry entry; + + private final DeviceId device1 = new DeviceId(UUID.fromString("1984e5f4-9ff0-4187-84ae-e4438bba4c8a")); + private final DeviceId device2 = new DeviceId(UUID.fromString("937fc062-1a9d-438f-aa22-55a93fc908b7")); + + private final long ts = System.currentTimeMillis(); + + @BeforeEach + void setUp() { + Map aggInputs = new HashMap<>(); + aggInputs.put(device1, new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 12L), 1L))); + aggInputs.put(device2, new AggSingleEntityArgumentEntry(device2, new BasicTsKvEntry(ts - 150, new LongDataEntry("key", 16L), 6L))); + + entry = new AggArgumentEntry(aggInputs, false); + } + + @Test + void testUpdateEntryWhenNotAggEntryPassed() { + assertThatThrownBy(() -> entry.updateEntry(new TsRollingArgumentEntry(5, 30000L))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported argument entry type for aggregation argument entry: " + ArgumentEntryType.TS_ROLLING); + } + + @Test + void testUpdateEntryWhenAggArgumentEntryPasser() { + DeviceId device3 = new DeviceId(UUID.randomUUID()); + DeviceId device4 = new DeviceId(UUID.randomUUID()); + + AggArgumentEntry aggArgumentEntry = new AggArgumentEntry(Map.of( + device3, new AggSingleEntityArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 16L), 13L)), + device4, new AggSingleEntityArgumentEntry(device4, new BasicTsKvEntry(ts - 60, new LongDataEntry("key", 23L), 7L)) + ), false); + + assertThat(entry.updateEntry(aggArgumentEntry)).isTrue(); + + Map aggInputs = entry.getAggInputs(); + assertThat(aggInputs.size()).isEqualTo(4); + assertThat(aggInputs.get(device3)).isEqualTo(aggArgumentEntry.getAggInputs().get(device3)); + assertThat(aggInputs.get(device4)).isEqualTo(aggArgumentEntry.getAggInputs().get(device4)); + } + + @Test + void testUpdateEntryWhenAggSingleEntityArgumentEntryPassedAndNoEntriesById() { + DeviceId device3 = new DeviceId(UUID.randomUUID()); + + AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 10L)); + + assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); + + Map aggInputs = entry.getAggInputs(); + assertThat(aggInputs.size()).isEqualTo(3); + assertThat(aggInputs.get(device3)).isEqualTo(singleEntityArgumentEntry); + } + + @Test + void testUpdateEntryWhenAggSingleEntityArgumentEntryPassedAndEntryByIdExist() { + AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device2, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 10L)); + + assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); + + Map aggInputs = entry.getAggInputs(); + assertThat(aggInputs.size()).isEqualTo(2); + assertThat(aggInputs.get(device2)).isEqualTo(singleEntityArgumentEntry); + } + + @Test + void testUpdateEntryWhenDeletedAggSingleEntityArgumentEntryPassed() { + AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device2, true); + + assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); + + Map aggInputs = entry.getAggInputs(); + assertThat(aggInputs.size()).isEqualTo(1); + assertThat(aggInputs.get(device2)).isNull(); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java new file mode 100644 index 0000000000..0401bb0156 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java @@ -0,0 +1,101 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class AggSingleEntityArgumentEntryTest { + + private AggSingleEntityArgumentEntry entry; + + private final DeviceId device1 = new DeviceId(UUID.fromString("1984e5f4-9ff0-4187-84ae-e4438bba4c8a")); + + private final long ts = System.currentTimeMillis(); + + @BeforeEach + void setUp() { + entry = new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 12L), 22L)); + } + + @Test + void testUpdateEntryWhenNotAggEntryPassed() { + assertThatThrownBy(() -> entry.updateEntry(new TsRollingArgumentEntry(5, 30000L))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported argument entry type for aggregation single entity argument entry: " + ArgumentEntryType.TS_ROLLING); + } + + @Test + void testUpdateEntryWhenResetPrevious() { + AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 100L)); + singleEntityArgumentEntry.setForceResetPrevious(true); + + assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); + assertThat(entry.getTs()).isEqualTo(singleEntityArgumentEntry.getTs()); + assertThat(entry.getKvEntryValue()).isEqualTo(singleEntityArgumentEntry.getKvEntryValue()); + assertThat(entry.getVersion()).isEqualTo(singleEntityArgumentEntry.getVersion()); + } + + + @Test + void testUpdateEntryWithTheSameTsAndVersion() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 19L), 22L)))).isFalse(); + } + + @Test + void testUpdateEntryWithTheSameTsAndDifferentVersion() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 134L), 23L)))).isTrue(); + } + + @Test + void testUpdateEntryWhenNewVersionIsNull() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 56L), null)))).isTrue(); + assertThat(entry.getValue()).isEqualTo(56L); + assertThat(entry.getVersion()).isNull(); + } + + @Test + void testUpdateEntryWhenNewVersionIsGreaterThanCurrent() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 76L), 23L)))).isTrue(); + assertThat(entry.getValue()).isEqualTo(76L); + assertThat(entry.getVersion()).isEqualTo(23); + } + + @Test + void testUpdateEntryWhenNewVersionIsLessThanCurrent() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 11L), 20L)))).isFalse(); + } + + @Test + void testUpdateEntryWhenValueWasNotChanged() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 18L), 45L)))).isTrue(); + } + + @Test + void testUpdateEntryWithOldTs() { + assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 150, new LongDataEntry("key", 155L), 45L)))).isFalse(); + } + +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java index a0bc9a72e6..214dc5247e 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java @@ -86,6 +86,8 @@ public interface RelationService { ListenableFuture> findByRelationPathQueryAsync(TenantId tenantId, EntityRelationPathQuery relationPathQuery); + List findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery relationPathQuery); + // TODO: This method may be useful for some validations in the future // ListenableFuture checkRecursiveRelation(EntityId from, EntityId to); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index 4de4551e73..8d4a831cf9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -45,4 +45,8 @@ public class Argument { return hasDynamicSource() && refDynamicSourceConfiguration.getType() == CFArgumentDynamicSourceType.CURRENT_OWNER; } + public boolean hasTsRollingArgument() { + return ArgumentType.TS_ROLLING.equals(refEntityKey.getType()); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java index 43a3360ce6..721930e676 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java @@ -41,6 +41,19 @@ public class LatestValuesAggregationCalculatedFieldConfiguration implements Argu @Override public void validate() { + if (relation == null) { + throw new IllegalArgumentException("Relation must be specified!"); + } + relation.validate(); + if (arguments.containsKey("ctx")) { + throw new IllegalArgumentException("Argument name 'ctx' is reserved and cannot be used."); + } + if (arguments.values().stream().anyMatch(Argument::hasTsRollingArgument)) { + throw new IllegalArgumentException("Calculated field with type: '" + getType() + "' doesn't support TS_ROLLING arguments."); + } + if (metrics.isEmpty()) { + throw new IllegalArgumentException("Latest value aggregation calculated field must have at least one metric."); + } } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 18d1806fe4..ec203e5309 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -514,6 +514,21 @@ public class BaseRelationService implements RelationService { return executor.submit(() -> relationDao.findByRelationPathQuery(tenantId, relationPathQuery)); } + @Override + public List findByRelationPathQuery(TenantId tenantId, EntityRelationPathQuery relationPathQuery) { + log.trace("Executing findByRelationPathQuery, tenantId [{}], relationPathQuery {}", tenantId, relationPathQuery); + validateId(tenantId, id -> "Invalid tenant id: " + id); + validate(relationPathQuery); + if (relationPathQuery.levels().size() == 1) { + RelationPathLevel relationPathLevel = relationPathQuery.levels().get(0); + return switch (relationPathLevel.direction()) { + case FROM -> findByFromAndType(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), RelationTypeGroup.COMMON); + case TO -> findByToAndType(tenantId, relationPathQuery.rootEntityId(), relationPathLevel.relationType(), RelationTypeGroup.COMMON); + }; + } + return relationDao.findByRelationPathQuery(tenantId, relationPathQuery); + } + private void validate(EntityRelationPathQuery relationPathQuery) { validateId((UUIDBased) relationPathQuery.rootEntityId(), id -> "Invalid root entity id: " + id); List levels = relationPathQuery.levels(); From 5e8225413b2ebc18cc40340c3ba0996a50116f1a Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Fri, 17 Oct 2025 20:34:27 +0300 Subject: [PATCH 51/88] lwm2m: bootstrap new: add reboot device and reboot device bootstrap - 2 --- .../lwm2m/client/LwM2MTestClient.java | 2 +- ...LwM2MBootstrapConfigStoreTaskProvider.java | 30 +++++++++++-------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java index 40fb6d3ff0..993e416ded 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2MTestClient.java @@ -522,7 +522,7 @@ public class LwM2MTestClient { log.info("[forceNullSecurityId] Set id=null for {}", security); } catch (NoSuchFieldException e) { try { - // Якщо поле в батьківському класі (наприклад SecurityObjectInstance) + //(SecurityObjectInstance) Field field = security.getClass().getSuperclass().getDeclaredField("id"); field.setAccessible(true); field.set(security, null); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java index 67f4aa32f6..ab69632956 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/store/LwM2MBootstrapConfigStoreTaskProvider.java @@ -188,7 +188,7 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask /** Map => LwM2MBootstrapClientInstanceIds * 1) Both - * - Short Server ID == null bs) + * - (Short) Server ID == null bs) * SECURITY = 0; InstanceId = 0 * - Short Server ID == 1 - 65534 lwm2m) * SECURITY = 0; InstanceId = 1 @@ -218,7 +218,7 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask // delete old bootstrap Security String path = "/" + SECURITY + "/" + bootstrapSecurityInstanceId; pathsDelete.add(path); - // add new bootstrap Security + security.serverId = null; requestsWrite.put(path, toWriteRequest(bootstrapSecurityInstanceId, security, contentFormat)); } } @@ -251,26 +251,31 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask for (BootstrapConfig.ServerSecurity security : new TreeMap<>(bootstrapConfigNew.security).values()) { if (!security.bootstrapServer) { // Security - boolean isUpdate = this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().containsKey(security.serverId); - Integer secureInstanceId; - Integer serverInstanceId; - if (isUpdate) { - secureInstanceId = this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(security.serverId); - serverInstanceId = this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().get(security.serverId); + Integer secureInstanceId = this.lwM2MBootstrapSessionClients.get(endpoint).getSecurityInstances().get(security.serverId); + if (secureInstanceId != null) { pathsDelete.add("/" + SECURITY + "/" + secureInstanceId); - pathsDelete.add("/" + SERVER + "/" + serverInstanceId); + requestsWrite.put("/" + SECURITY + "/" + secureInstanceId, toWriteRequest(secureInstanceId, security, contentFormat)); } else { secureInstanceId = ++lwm2mSecurityInstanceIdMax; + if (bootstrapSecurityInstanceId.equals(secureInstanceId)) { + secureInstanceId = ++lwm2mSecurityInstanceIdMax; + } + requestsWrite.put("/" + SECURITY + "/" + secureInstanceId, toWriteRequest(secureInstanceId, security, contentFormat)); + } + Integer serverInstanceId = this.lwM2MBootstrapSessionClients.get(endpoint).getServerInstances().get(security.serverId); + if (serverInstanceId != null) { + pathsDelete.add("/" + SERVER + "/" + serverInstanceId); + } else { serverInstanceId = ++lwm2mServerInstanceIdMax; } - requestsWrite.put("/" + SECURITY + "/" + secureInstanceId, toWriteRequest(secureInstanceId, security, contentFormat)); + Integer finalServerInstanceId = serverInstanceId; new TreeMap<>(bootstrapConfigNew.servers).values().stream() .filter(server -> server.shortId == security.serverId) .findFirst() .ifPresent(server -> requestsWrite.put( - "/" + SERVER + "/" + serverInstanceId, - toWriteRequest(serverInstanceId, server, contentFormat) + "/" + SERVER + "/" + finalServerInstanceId, + toWriteRequest(finalServerInstanceId, server, contentFormat) ) ); } @@ -290,7 +295,6 @@ public class LwM2MBootstrapConfigStoreTaskProvider implements LwM2MBootstrapTask return (requests); } - private void initSupportedObjectsDefault() { this.supportedObjects = new HashMap<>(); this.supportedObjects.put(SECURITY, "1.1"); From d02582b13bc2ee3d264497d544ac3bd00a045359 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Sun, 19 Oct 2025 18:00:45 +0300 Subject: [PATCH 52/88] lwm2m: bootstrap new: add reboot device and reboot device bootstrap - 3 --- .../lwm2m/server/client/LwM2mClient.java | 6 + .../lwm2m/utils/LwM2MTransportUtil.java | 2 + ui-ngx/src/app/core/http/device.service.ts | 70 ++++++++++- .../device-credentials-lwm2m.component.ts | 114 ++---------------- 4 files changed, 85 insertions(+), 107 deletions(-) diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClient.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClient.java index 64597df9c5..884ba7e033 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClient.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClient.java @@ -66,7 +66,9 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.lwm2m.LwM2mConstants.LWM2M_SEPARATOR_PATH; +import static org.thingsboard.server.transport.lwm2m.utils.LwM2MTransportUtil.BOOTSTRAP_TRIGGER_PARAMS_ID; import static org.thingsboard.server.transport.lwm2m.utils.LwM2MTransportUtil.LWM2M_OBJECT_VERSION_DEFAULT; +import static org.thingsboard.server.transport.lwm2m.utils.LwM2MTransportUtil.REGISTRATION_TRIGGER_PARAMS_ID; import static org.thingsboard.server.transport.lwm2m.utils.LwM2MTransportUtil.convertMultiResourceValuesFromRpcBody; import static org.thingsboard.server.transport.lwm2m.utils.LwM2MTransportUtil.equalsResourceTypeGetSimpleName; import static org.thingsboard.server.transport.lwm2m.utils.LwM2MTransportUtil.fromVersionedIdToObjectId; @@ -342,6 +344,10 @@ public class LwM2mClient { public String isValidObjectVersion(String path) { LwM2mPath pathIds = getLwM2mPathFromString(path); + if (pathIds.isResource() && (pathIds.toString().equals(REGISTRATION_TRIGGER_PARAMS_ID ) || + pathIds.toString().equals(BOOTSTRAP_TRIGGER_PARAMS_ID))) { + return ""; + } LwM2m.Version verSupportedObject = this.getSupportedObjectVersion(pathIds.getObjectId()); if (verSupportedObject == null) { return String.format("Specified object id %s absent in the list supported objects of the client or is security object!", pathIds.getObjectId()); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/LwM2MTransportUtil.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/LwM2MTransportUtil.java index d5ae8febe1..c1fbbcb006 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/LwM2MTransportUtil.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/LwM2MTransportUtil.java @@ -81,6 +81,8 @@ public class LwM2MTransportUtil { public static final String LOG_LWM2M_INFO = "info"; public static final String LOG_LWM2M_ERROR = "error"; public static final String LOG_LWM2M_WARN = "warn"; + public static final String REGISTRATION_TRIGGER_PARAMS_ID = "/1/0/8"; + public static final String BOOTSTRAP_TRIGGER_PARAMS_ID = "/1/0/9";; public static LwM2mOtaConvert convertOtaUpdateValueToString(String pathIdVer, Object value, ResourceModel.Type currentType) { String path = fromVersionedIdToObjectId(pathIdVer); diff --git a/ui-ngx/src/app/core/http/device.service.ts b/ui-ngx/src/app/core/http/device.service.ts index 1e3a304774..88e5f8f7bf 100644 --- a/ui-ngx/src/app/core/http/device.service.ts +++ b/ui-ngx/src/app/core/http/device.service.ts @@ -16,7 +16,7 @@ import { Injectable } from '@angular/core'; import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; -import { Observable, ReplaySubject } from 'rxjs'; +import {catchError, Observable, of, ReplaySubject, throwError, timeout} from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { PageLink } from '@shared/models/page/page-link'; import { PageData } from '@shared/models/page/page-data'; @@ -35,6 +35,7 @@ import { AuthService } from '@core/auth/auth.service'; import { BulkImportRequest, BulkImportResult } from '@shared/import-export/import-export.models'; import { PersistentRpc, RpcStatus } from '@shared/models/rpc.models'; import { ResourcesService } from '@core/services/resources.service'; +import {map} from "rxjs/operators"; @Injectable({ providedIn: 'root' @@ -219,4 +220,71 @@ export class DeviceService { public downloadGatewayDockerComposeFile(deviceId: string): Observable { return this.resourcesService.downloadResource(`/api/device-connectivity/gateway-launch/${deviceId}/docker-compose/download`); } + + public rebootDevice(deviceId: string = '', isBootstrapServer: boolean): void { + const urlApi = `/api/plugins/rpc/twoway/${deviceId}`; + // DiscoveryAll} + this.http.post(urlApi, { method: "DiscoverAll" }) + .pipe( + timeout(10000), // 10 sec + catchError(err => { + console.error('DiscoverAll timeout or error', err); + return throwError(() => err); + }) + ) + .subscribe({ + next: (response: any) => { + console.log('success: Discovery'); + console.log(response); + // result = 'CONTENT' + if (response.result && response.result.toUpperCase() === 'CONTENT') { + const resourceId = isBootstrapServer ? 9 : 8; + const rebutName = isBootstrapServer ? "Bootstrap-Request Trigger" : + "Registration Update Trigger"; + const resourcePath = `/1/0/${resourceId}`; + + // first rebootTrigger + this.rebootTrigger(resourcePath, urlApi).subscribe(responseReboot => { + if (responseReboot.result === 'CHANGED') { + console.info(`info: ${rebutName} success.`); + } else { + console.error(`error: ${rebutName} failed: ${responseReboot.toString()}`); + } + }); + } + else { + console.error(`error3: Bad registration device with id = ${deviceId} ❗ RPC result is not CONTENT`); + } + }, + error: (e) => { + console.error(`error4: Bad registration device with id = ${deviceId} ${e.toString()}`); + return throwError(() => new Error('Could not get JWT token from store.')); + // return throwError(() => e); + }, + complete: () => { + console.log('Discovery stream complete'); } + }); + } + + private rebootTrigger(resourcePath: string, urlApi: string): Observable<{ result: string;}> { + console.log(`Sending reboot command to ${resourcePath}`); + return this.http.post(urlApi, { + method: 'Execute', + params: { id: resourcePath } + }).pipe( + timeout(10000), + map(res => { + console.log(res); + if (res?.result?.toUpperCase() === 'CHANGED') { + return { result: 'CHANGED' }; + } else { + return {result: 'ERROR'} + }; + }), + catchError(err => { + console.error(`Execute error5 for ${resourcePath}:`, err); + return of({ result: 'ERROR' }); + }) + ); + } } diff --git a/ui-ngx/src/app/modules/home/components/device/device-credentials-lwm2m.component.ts b/ui-ngx/src/app/modules/home/components/device/device-credentials-lwm2m.component.ts index ec9f8ff5fb..4ddc3a4852 100644 --- a/ui-ngx/src/app/modules/home/components/device/device-credentials-lwm2m.component.ts +++ b/ui-ngx/src/app/modules/home/components/device/device-credentials-lwm2m.component.ts @@ -32,12 +32,11 @@ import { Lwm2mSecurityType, Lwm2mSecurityTypeTranslationMap } from '@shared/models/lwm2m-security-config.models'; -import {Subject, throwError, timeout, catchError, of} from 'rxjs'; -import {map, takeUntil} from 'rxjs/operators'; +import {Subject} from 'rxjs'; +import {takeUntil} from 'rxjs/operators'; import { isDefinedAndNotNull } from '@core/utils'; -import { HttpClient } from '@angular/common/http'; import {DeviceId} from "@shared/models/id/device-id"; -import {Observable} from "rxjs/internal/Observable"; +import {DeviceService} from "@core/http/device.service"; @Component({ selector: 'tb-device-credentials-lwm2m', @@ -72,7 +71,7 @@ export class DeviceCredentialsLwm2mComponent implements ControlValueAccessor, Va deviceId: DeviceId; constructor(private fb: UntypedFormBuilder, - private http: HttpClient) { + private deviceService: DeviceService) { this.lwm2mConfigFormGroup = this.initLwm2mConfigForm(); } @@ -115,110 +114,13 @@ export class DeviceCredentialsLwm2mComponent implements ControlValueAccessor, Va * - DiscoveryAll * requestBody = "{\"method\":\"DiscoverAll\"}"; * - "Registration Update Trigger", - * requestBody = "{\"method\": \"Execute\", \"params\": {\"id\": \"/1_1.2/0/8\"}} + * requestBody = "{\"method\": \"Execute\", \"params\": {\"id\": \"/1/0/8\"}} * - "Bootstrap-Request Trigger" - * requestBody = "{\"method\": \"Execute\", \"params\": {\"id\": \"/1_1.2/0/9\"}} + * requestBody = "{\"method\": \"Execute\", \"params\": {\"id\": \"/1/0/9\"}} */ - public rebootDevice(isBootstrapServer: boolean): void { - const urlApi = `/api/plugins/rpc/twoway/${this.deviceId.id}`; - // DiscoveryAll} - this.http.post(urlApi, { method: "DiscoverAll" }) - .pipe( - timeout(10000), // 10 sec - catchError(err => { - console.error('DiscoverAll timeout or error', err); - return throwError(() => err); - }) - ) - .subscribe({ - next: (response: any) => { - console.log('success: Discovery'); - console.log(response); - // result = 'CONTENT' - if (response.result && response.result.toUpperCase() === 'CONTENT') { - const verId = this.getVerId(response.value); - console.log("ObjectId=1 ver:", verId); - const resourceId = isBootstrapServer ? 9 : 8; - const resourcePath = `/1_${verId}/0/${resourceId}`; - - // first rebootTrigger - this.rebootTrigger(resourcePath, urlApi).subscribe(first => { - if (first.result === 'CHANGED') { - console.log('Reboot success first'); - } - else if (first.result === 'BAD_REQUEST' && first.newVersionId && first.newVersionId !== verId) { - // Retry with new version - const correctedPath = `/1_${first.newVersionId}/0/${resourceId}`; - console.log(`Retrying with version ${first.newVersionId}`); - - this.rebootTrigger(correctedPath, urlApi).subscribe(second => { - if (second.result === 'CHANGED') { - console.log('Success reboot after retry'); - } else { - console.error(`error1: Reboot second failed: ${second.toString()}`); - } - }); - } else { - console.error(`error2: Reboot first failed: ${first.toString()}`); - } - }); - } - - else { - console.error(`error3: Bad registration device with id = ${this.deviceId.id} ❗ RPC result is not CONTENT`); - } - }, - error: (e) => { - console.error(`error4: Bad registration device with id = ${this.deviceId.id} ${e.toString()}`); - return throwError(() => e); - }, - complete: () => { - console.log('Discovery stream complete'); } - }); - } - - private getVerId(value: string): string { - const verDef = '1.1'; - try { - const arr = JSON.parse(value); - if (!Array.isArray(arr)) return verDef; - const obj1 = arr.find((s: string) => s.startsWith(' { - console.log(`Sending reboot command to ${resourcePath}`); - - return this.http.post(urlApi, { - method: 'Execute', - params: { id: resourcePath } - }).pipe( - timeout(10000), - map(res => { - console.log(`Reboot for ${resourcePath}`); - console.log(res); - if (res?.result?.toUpperCase() === 'CHANGED') { - return { result: 'CHANGED' }; - } - - if (res?.result?.toUpperCase() === 'BAD_REQUEST' && res?.error) { - const match = (res.error as string).match(/version[:=]\s*([\d.]+)/i); - const newVersionId = match ? match[1] : null; - console.warn(`BAD_REQUEST: suggested version ${newVersionId ?? 'unknown'}`); - return { result: 'BAD_REQUEST', newVersionId }; - } - return { result: 'ERROR' }; - }), - catchError(err => { - console.error(`Execute error5 for ${resourcePath}:`, err); - return of({ result: 'ERROR' }); - }) - ); + public rebootDevice(isBootstrapServer: boolean): void { + this.deviceService.rebootDevice(this.deviceId.id, isBootstrapServer); } private initClientSecurityConfig(config: Lwm2mSecurityConfigModels): void { From 37039a995dca6c3fa05395992ce335d4b579083a Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 20 Oct 2025 10:02:55 +0300 Subject: [PATCH 53/88] added minDeduplicationInterval to tenant profile config --- .../main/data/upgrade/basic/schema_update.sql | 8 +++ .../controller/SystemInfoController.java | 1 + .../cf/ctx/state/CalculatedFieldCtx.java | 2 +- ...ValuesAggregationCalculatedFieldState.java | 2 +- ...tValuesAggregationCalculatedFieldTest.java | 64 ++++++++++--------- .../server/common/data/SystemParams.java | 1 + ...gregationCalculatedFieldConfiguration.java | 2 +- .../DefaultTenantProfileConfiguration.java | 2 + .../CalculatedFieldDataValidator.java | 15 ++++- 9 files changed, 63 insertions(+), 34 deletions(-) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 495aee00e2..c5b73c6899 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -34,6 +34,12 @@ SET profile_data = jsonb_set( WHEN (profile_data -> 'configuration') ? 'maxRelationLevelPerCfArgument' THEN NULL ELSE to_jsonb(10) + END, + 'minAllowedDeduplicationIntervalInSecForCF', + CASE + WHEN (profile_data -> 'configuration') ? 'minAllowedDeduplicationIntervalInSecForCF' + THEN NULL + ELSE to_jsonb(3600) END ) ), @@ -43,6 +49,8 @@ WHERE NOT ( (profile_data -> 'configuration') ? 'minAllowedScheduledUpdateIntervalInSecForCF' AND (profile_data -> 'configuration') ? 'maxRelationLevelPerCfArgument' + AND + (profile_data -> 'configuration') ? 'minAllowedDeduplicationIntervalInSecForCF' ); -- UPDATE TENANT PROFILE CONFIGURATION END diff --git a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java index b9968aefa9..82807d0762 100644 --- a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java +++ b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java @@ -164,6 +164,7 @@ public class SystemInfoController extends BaseController { systemParams.setMaxDataPointsPerRollingArg(tenantProfileConfiguration.getMaxDataPointsPerRollingArg()); systemParams.setMinAllowedScheduledUpdateIntervalInSecForCF(tenantProfileConfiguration.getMinAllowedScheduledUpdateIntervalInSecForCF()); systemParams.setMaxRelationLevelPerCfArgument(tenantProfileConfiguration.getMaxRelationLevelPerCfArgument()); + systemParams.setMinAllowedDeduplicationIntervalInSecForCF(tenantProfileConfiguration.getMinAllowedDeduplicationIntervalInSecForCF()); systemParams.setTrendzSettings(trendzSettingsService.findTrendzSettings(currentUser.getTenantId())); } systemParams.setMobileQrEnabled(Optional.ofNullable(qrCodeSettingService.findQrCodeSettings(TenantId.SYS_TENANT_ID)) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index f69d611b64..1d9ccf10d3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -581,7 +581,7 @@ public class CalculatedFieldCtx { } if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig && other.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig - && (thisConfig.getDeduplicationIntervalMillis() != otherConfig.getDeduplicationIntervalMillis() || !thisConfig.getMetrics().equals(otherConfig.getMetrics()))) { + && (thisConfig.getDeduplicationIntervalInSec() != otherConfig.getDeduplicationIntervalInSec() || !thisConfig.getMetrics().equals(otherConfig.getMetrics()))) { return true; } return false; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java index f3560faafa..fdc045a0db 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java @@ -66,7 +66,7 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF super.setCtx(ctx, actorCtx); var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); metrics = configuration.getMetrics(); - deduplicationInterval = configuration.getDeduplicationIntervalMillis(); + deduplicationInterval = configuration.getDeduplicationIntervalInSec(); } @Override diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java index 46b8b78d57..e8c244b57e 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -79,12 +79,16 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll private AssetProfile assetProfile; private Asset asset; - private long deduplicationInterval = 10000; + private long deduplicationInterval = 10; @Before public void beforeTest() throws Exception { loginSysAdmin(); + updateDefaultTenantProfileConfig(tenantProfileConfig -> { + tenantProfileConfig.setMinAllowedDeduplicationIntervalInSecForCF(1); + }); + Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); savedTenant = saveTenant(tenant); @@ -131,7 +135,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createOccupancyCF(assetProfile.getId()); - await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -149,7 +153,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device3.getId(), "{\"occupied\":true}"); - await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -171,7 +175,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll Asset asset2 = createAsset("Asset 2", assetProfile.getId()); - await().alias("add entity to profile with no related entities and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("add entity to profile with no related entities and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode occupancy = getLatestTelemetry(asset2.getId(), "freeSpaces", "occupiedSpaces", "totalSpaces"); @@ -184,7 +188,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset2.getId(), device3.getId(), "Contains"); createEntityRelation(asset2.getId(), device4.getId(), "Contains"); - await().alias("create relations and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create relations and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -196,7 +200,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device3.getId(), "{\"occupied\":false}"); - await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval * 2, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -218,7 +222,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createOccupancyCF(assetProfile.getId()); - await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -240,7 +244,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device3.getId(), "{\"occupied\":true}"); - await().alias("change profile and no aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("change profile and no aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -262,7 +266,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createOccupancyCF(asset2.getId()); - await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform aggregation with default values").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -280,7 +284,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device1.getId(), "{\"occupied\":false}"); - await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -301,7 +305,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device1.getId(), "{\"occupied\":false}"); - await().alias("delete cf and update telemetry and no aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("delete cf and update telemetry and no aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -319,13 +323,13 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device1.getId(), "{\"occupied\":false}"); - await().alias("update telemetry -> no changes").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) + await().alias("update telemetry -> no changes").atMost(deduplicationInterval / 2, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(this::checkInitialCalculationValues); postTelemetry(device2.getId(), "{\"occupied\":false}"); - await().alias("create CF and perform initial calculation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform initial calculation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -355,7 +359,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createOccupancyCF(asset2.getId()); - await().alias("create CF and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -368,7 +372,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll doDelete("/api/plugins/telemetry/DEVICE/" + device3.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=" + thirdTs + "&endTs=" + thirdTs + 1, String.class); doDelete("/api/plugins/telemetry/DEVICE/" + device4.getId() + "/timeseries/delete?keys=occupied&deleteAllDataForKeys=false&rewriteLatestIfDeleted=true&deleteLatest=true&startTs=" + secondTs + "&endTs=" + secondTs + 1, String.class); - await().alias("delete latest telemetry and perform aggregation with previous or default values").atMost(deduplicationInterval * 2, TimeUnit.MILLISECONDS) + await().alias("delete latest telemetry and perform aggregation with previous or default values").atMost(deduplicationInterval * 2, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset2.getId(), Map.of( @@ -390,7 +394,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset.getId(), device3.getId(), "Contains"); - await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -408,7 +412,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll deleteEntityRelation(new EntityRelation(asset.getId(), device1.getId(), "Contains", RelationTypeGroup.COMMON)); - await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create relation and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -432,7 +436,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll configuration.setRelation(new RelationPathLevel(EntitySearchDirection.FROM, "Has")); saveCalculatedField(cf); - await().alias("update relation path and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update relation path and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -458,7 +462,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll configuration.setArguments(Map.of("oc", argument)); saveCalculatedField(cf); - await().alias("update arguments and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update arguments and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of( @@ -475,7 +479,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device2.getId(), "{\"temperature\":19.6}"); CalculatedField cf = createAvgTemperatureCF(asset.getId()); - await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); @@ -489,7 +493,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll configuration.setMetrics(Map.of("maxTemperature", aggMetric)); saveCalculatedField(cf); - await().alias("update metrics and perform aggregation").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) + await().alias("update metrics and perform aggregation").atMost(deduplicationInterval / 2, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("maxTemperature", "24")); @@ -498,7 +502,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device1.getId(), "{\"temperature\":101.3}"); postTelemetry(device2.getId(), "{\"temperature\":25.8}"); - await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("maxTemperature", "26")); @@ -511,7 +515,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device2.getId(), "{\"temperature\":19.6}"); CalculatedField cf = createAvgTemperatureCF(asset.getId()); - await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); @@ -524,7 +528,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll configuration.setOutput(output); saveCalculatedField(cf); - await().alias("update output and perform aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update output and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ArrayNode avgTemperature = getServerAttributes(asset.getId(), "avgTemperature"); @@ -540,17 +544,17 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device2.getId(), "{\"temperature\":19.6}"); CalculatedField cf = createAvgTemperatureCF(asset.getId()); - await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create avg temp cf and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); }); var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); - configuration.setDeduplicationIntervalMillis(2 * deduplicationInterval); + configuration.setDeduplicationIntervalInSec(2 * deduplicationInterval); saveCalculatedField(cf); - await().alias("update deduplication interval and perform aggregation").atMost(deduplicationInterval / 2, TimeUnit.MILLISECONDS) + await().alias("update deduplication interval and perform aggregation").atMost(deduplicationInterval / 2, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); @@ -558,7 +562,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device2.getId(), "{\"temperature\":32.1}"); - await().alias("update telemetry and perform aggregation").atMost(2 * deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("update telemetry and perform aggregation").atMost(2 * deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { verifyTelemetry(asset.getId(), Map.of("avgTemperature", "28")); @@ -566,7 +570,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll } private void checkInitialCalculation() { - await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.MILLISECONDS) + await().alias("create CF and perform initial aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(this::checkInitialCalculationValues); } @@ -656,7 +660,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll LatestValuesAggregationCalculatedFieldConfiguration configuration = new LatestValuesAggregationCalculatedFieldConfiguration(); configuration.setRelation(relation); configuration.setArguments(inputs); - configuration.setDeduplicationIntervalMillis(deduplicationInterval); + configuration.setDeduplicationIntervalInSec(deduplicationInterval); configuration.setMetrics(metrics); configuration.setOutput(output); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java index f83a812529..6a475daae3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java @@ -40,5 +40,6 @@ public class SystemParams { long maxDataPointsPerRollingArg; int minAllowedScheduledUpdateIntervalInSecForCF; int maxRelationLevelPerCfArgument; + long minAllowedDeduplicationIntervalInSecForCF; TrendzSettings trendzSettings; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java index 721930e676..5f8fb65265 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java @@ -29,7 +29,7 @@ public class LatestValuesAggregationCalculatedFieldConfiguration implements Argu private RelationPathLevel relation; private Map arguments; - private long deduplicationIntervalMillis; + private long deduplicationIntervalInSec; private Map metrics; private Output output; private boolean useLatestTs; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index c6bd9a7f38..9f124fd9e4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -184,6 +184,8 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private long maxStateSizeInKBytes = 32; @Schema(example = "2") private long maxSingleValueArgumentSizeInKBytes = 2; + @Schema(example = "3600") + private long minAllowedDeduplicationIntervalInSecForCF = 3600; @Override public long getProfileThreshold(ApiUsageRecordKey key) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java index 67ed8f29b0..319a4fbe8b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.configuration.ArgumentsBasedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScheduledUpdateSupportedCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.cf.CalculatedFieldDao; @@ -46,6 +47,7 @@ public class CalculatedFieldDataValidator extends DataValidator validateCalculatedFieldConfiguration(calculatedField); validateSchedulingConfiguration(tenantId, calculatedField); validateRelationQuerySourceArguments(tenantId, calculatedField); + validateAggregationConfiguration(tenantId, calculatedField); } @Override @@ -87,7 +89,7 @@ public class CalculatedFieldDataValidator extends DataValidator private void validateSchedulingConfiguration(TenantId tenantId, CalculatedField calculatedField) { if (!(calculatedField.getConfiguration() instanceof ScheduledUpdateSupportedCalculatedFieldConfiguration scheduledUpdateCfg) - || !scheduledUpdateCfg.isScheduledUpdateEnabled()) { + || !scheduledUpdateCfg.isScheduledUpdateEnabled()) { return; } long minAllowedScheduledUpdateInterval = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMinAllowedScheduledUpdateIntervalInSecForCF); @@ -110,6 +112,17 @@ public class CalculatedFieldDataValidator extends DataValidator wrapAsDataValidation(() -> relationQueryDynamicSourceConfiguration.validateMaxRelationLevel(argumentName, maxRelationLevel))); } + private void validateAggregationConfiguration(TenantId tenantId, CalculatedField calculatedField) { + if (!(calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfiguration)) { + return; + } + long minAllowedDeduplicationInterval = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMinAllowedDeduplicationIntervalInSecForCF); + if (aggConfiguration.getDeduplicationIntervalInSec() < minAllowedDeduplicationInterval) { + throw new IllegalArgumentException("Deduplication interval is less than configured " + + "minimum allowed interval in tenant profile: " + minAllowedDeduplicationInterval); + } + } + private static void wrapAsDataValidation(Runnable validation) { try { validation.run(); From 55fb64ef13af8c7cd84033ce31639087f4cf8ac7 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 20 Oct 2025 11:20:38 +0300 Subject: [PATCH 54/88] fixed delete attributes --- ...CalculatedFieldEntityMessageProcessor.java | 49 +++++++----- ...alculatedFieldManagerMessageProcessor.java | 30 ++++---- ...tValuesAggregationCalculatedFieldTest.java | 75 +++++++++++++++++++ 3 files changed, 119 insertions(+), 35 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 58113deb08..e0faf46185 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -551,19 +551,19 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { - return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getRelatedEntityArguments(), data); + return mapToArguments(entityId, ctx.getMainEntityArguments(), Collections.emptyMap(), data); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { return mapToArguments(entityId, ctx.getLinkedAndDynamicArgs(entityId), ctx.getRelatedEntityArguments(), data); } - private Map mapToArguments(EntityId originator, Map argNames, Map aggArgNames, List data) { + private Map mapToArguments(EntityId originator, Map argNames, Map relatedEntityArgs, List data) { Map arguments = new HashMap<>(); - if (!aggArgNames.isEmpty()) { + if (!relatedEntityArgs.isEmpty()) { for (TsKvProto item : data) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); - String argName = aggArgNames.get(key); + String argName = relatedEntityArgs.get(key); if (argName != null) { arguments.put(argName, new AggSingleEntityArgumentEntry(originator, item)); } @@ -587,17 +587,17 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List attrDataList) { - return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), ctx.getRelatedEntityArguments(), scope, attrDataList); + return mapToArguments(entityId, ctx.getMainEntityArguments(), ctx.getMainEntityGeofencingArgumentNames(), Collections.emptyMap(), scope, attrDataList); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { var argNames = ctx.getLinkedAndDynamicArgs(entityId); List geofencingArgumentNames = ctx.getLinkedEntityAndCurrentOwnerGeofencingArgumentNames(); - Map aggregationInputs = ctx.getRelatedEntityArguments(); - return mapToArguments(entityId, argNames, geofencingArgumentNames, aggregationInputs, scope, attrDataList); + Map relatedEntityArgs = ctx.getRelatedEntityArguments(); + return mapToArguments(entityId, argNames, geofencingArgumentNames, relatedEntityArgs, scope, attrDataList); } - private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, Map aggArgNames, AttributeScopeProto scope, List attrDataList) { + private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, Map relatedEntityArgs, AttributeScopeProto scope, List attrDataList) { Map arguments = new HashMap<>(); if (!argNames.isEmpty()) { for (AttributeValueProto item : attrDataList) { @@ -613,10 +613,10 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM arguments.put(argName, new SingleValueArgumentEntry(item)); } } - if (!aggArgNames.isEmpty()) { + if (!relatedEntityArgs.isEmpty()) { for (AttributeValueProto item : attrDataList) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); - String argName = aggArgNames.get(key); + String argName = relatedEntityArgs.get(key); if (argName != null) { arguments.put(argName, new AggSingleEntityArgumentEntry(entityId, item)); } @@ -627,26 +627,40 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private Map mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List removedAttrKeys) { var argNames = ctx.getLinkedAndDynamicArgs(entityId); - if (argNames.isEmpty()) { + Map relatedEntityArguments = ctx.getRelatedEntityArguments(); + if (argNames.isEmpty() && relatedEntityArguments.isEmpty()) { return Collections.emptyMap(); } List geofencingArgumentNames = ctx.getLinkedEntityAndCurrentOwnerGeofencingArgumentNames(); - List relatedArgumentNames = ctx.getRelatedEntityArgumentNames(); - return mapToArgumentsWithDefaultValue(entityId, argNames, ctx.getArguments(), geofencingArgumentNames, relatedArgumentNames, scope, removedAttrKeys); + return mapToArgumentsWithDefaultValue(entityId, argNames, ctx.getArguments(), geofencingArgumentNames, relatedEntityArguments, scope, removedAttrKeys); } private Map mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, AttributeScopeProto scope, List removedAttrKeys) { - return mapToArgumentsWithDefaultValue(null, ctx.getMainEntityArguments(), ctx.getArguments(), ctx.getMainEntityGeofencingArgumentNames(), new ArrayList<>(), scope, removedAttrKeys); + return mapToArgumentsWithDefaultValue(null, ctx.getMainEntityArguments(), ctx.getArguments(), ctx.getMainEntityGeofencingArgumentNames(), Collections.emptyMap(), scope, removedAttrKeys); } private Map mapToArgumentsWithDefaultValue(EntityId msgEntityId, Map argNames, Map configArguments, List geofencingArgNames, - List relatedEntityArgNames, + Map relatedEntityArgs, AttributeScopeProto scope, List removedAttrKeys) { Map arguments = new HashMap<>(); + if (!relatedEntityArgs.isEmpty()) { + for (String removedKey : removedAttrKeys) { + ReferencedEntityKey key = new ReferencedEntityKey(removedKey, ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); + if (relatedEntityArgs.containsKey(key)) { + String argName = relatedEntityArgs.get(key); + Argument argument = configArguments.get(argName); + String defaultValue = (argument != null) ? argument.getDefaultValue() : null; + SingleValueArgumentEntry argumentEntry = StringUtils.isNotEmpty(defaultValue) + ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) + : new SingleValueArgumentEntry(); + arguments.put(argName, new AggSingleEntityArgumentEntry(msgEntityId, argumentEntry)); + } + } + } for (String removedKey : removedAttrKeys) { ReferencedEntityKey key = new ReferencedEntityKey(removedKey, ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); String argName = argNames.get(key); @@ -662,12 +676,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM SingleValueArgumentEntry argumentEntry = StringUtils.isNotEmpty(defaultValue) ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) : new SingleValueArgumentEntry(); - if (relatedEntityArgNames.contains(argName)) { - arguments.put(argName, new AggSingleEntityArgumentEntry(msgEntityId, argumentEntry)); - continue; - } arguments.put(argName, argumentEntry); - } return arguments; } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index af5a672157..24fd5d320b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -41,9 +41,9 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationPathLevel; -import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.msg.CalculatedFieldStatePartitionRestoreMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldCacheInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; @@ -520,22 +520,22 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware List result = new ArrayList<>(); if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration configuration) { RelationPathLevel relation = configuration.getRelation(); - switch (relation.direction()) { - case FROM -> { - List byToAndType = relationService.findByToAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); - if (byToAndType != null && !byToAndType.isEmpty()) { - EntityRelation entityRelation = byToAndType.get(0); // only one supported + EntitySearchDirection inverseDirection = switch (relation.direction()) { + case FROM -> EntitySearchDirection.TO; + case TO -> EntitySearchDirection.FROM; + }; + RelationPathLevel inverseRelation = new RelationPathLevel(inverseDirection, relation.relationType()); + List byRelationPathQuery = relationService.findByRelationPathQuery(tenantId, new EntityRelationPathQuery(entityId, List.of(inverseRelation))); + if (byRelationPathQuery != null && !byRelationPathQuery.isEmpty()) { + switch (relation.direction()) { + case FROM -> { + EntityRelation entityRelation = byRelationPathQuery.get(0); // only one supported result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getFrom())); } - } - case TO -> { - List byFromAndType = relationService.findByFromAndType(tenantId, entityId, relation.relationType(), RelationTypeGroup.COMMON); - if (byFromAndType != null && !byFromAndType.isEmpty()) { - for (EntityRelation entityRelation : byFromAndType) { - if (entityRelation.getTo().equals(cf.getEntityId())) { - result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getTo())); - } - } + case TO -> { + byRelationPathQuery.stream() + .filter(entityRelation -> entityRelation.getTo().equals(cf.getEntityId())) + .forEach(entityRelation -> result.add(new CalculatedFieldEntityCtxId(tenantId, cf.getCfId(), entityRelation.getTo()))); } } } diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java index e8c244b57e..eb11a329be 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java @@ -383,6 +383,44 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll }); } + @Test + public void testDeleteAttr_checkAggregationWithDefault() throws Exception { + Asset asset2 = createAsset("Asset 2", assetProfile.getId()); + Device device3 = createDevice("Device 3", "1234567890333"); + Device device4 = createDevice("Device 4", "1234567890444"); + + createEntityRelation(asset2.getId(), device3.getId(), "Contains"); + createEntityRelation(asset2.getId(), device4.getId(), "Contains"); + + postAttributes(device3.getId(), AttributeScope.SERVER_SCOPE, "{\"occupied\":true}"); + postAttributes(device4.getId(), AttributeScope.SERVER_SCOPE, "{\"occupied\":true}"); + + createOccupancyCFWithAttr(asset2.getId()); + + await().alias("create CF and perform aggregation").atMost(deduplicationInterval, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "0", + "occupiedSpaces", "2", + "totalSpaces", "2" + )); + }); + + doDelete("/api/plugins/telemetry/DEVICE/" + device3.getUuidId() + "/SERVER_SCOPE?keys=occupied", String.class); + doDelete("/api/plugins/telemetry/DEVICE/" + device4.getUuidId() + "/SERVER_SCOPE?keys=occupied", String.class); + + await().alias("delete attribute and perform aggregation with default values").atMost(deduplicationInterval * 2, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + verifyTelemetry(asset2.getId(), Map.of( + "freeSpaces", "2", + "occupiedSpaces", "0", + "totalSpaces", "2" + )); + }); + } + @Test public void testCreateRelation_checkAggregation() throws Exception { createOccupancyCF(asset.getId()); @@ -646,6 +684,43 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll output); } + private CalculatedField createOccupancyCFWithAttr(EntityId entityId) { + Map arguments = new HashMap<>(); + Argument argument = new Argument(); + argument.setRefEntityKey(new ReferencedEntityKey("occupied", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + argument.setDefaultValue("false"); + arguments.put("oc", argument); + + Map aggMetrics = new HashMap<>(); + + AggMetric freeSpaces = new AggMetric(); + freeSpaces.setFunction(AggFunction.COUNT); + freeSpaces.setFilter("return oc == false;"); + freeSpaces.setInput(new AggKeyInput("oc")); + aggMetrics.put("freeSpaces", freeSpaces); + + AggMetric occupiedSpaces = new AggMetric(); + occupiedSpaces.setFunction(AggFunction.COUNT); + occupiedSpaces.setFilter("return oc == true;"); + occupiedSpaces.setInput(new AggKeyInput("oc")); + aggMetrics.put("occupiedSpaces", occupiedSpaces); + + AggMetric totalSpaces = new AggMetric(); + totalSpaces.setFunction(AggFunction.COUNT); + totalSpaces.setInput(new AggFunctionInput("return 1;")); + aggMetrics.put("totalSpaces", totalSpaces); + + Output output = new Output(); + output.setType(OutputType.TIME_SERIES); + output.setDecimalsByDefault(0); + + return createAggCf("Occupied spaces", entityId, + new RelationPathLevel(EntitySearchDirection.FROM, "Contains"), + arguments, + aggMetrics, + output); + } + private CalculatedField createAggCf(String name, EntityId entityId, RelationPathLevel relation, From cefc6925c196edec8989f3c42026792c9dc39f1c Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 20 Oct 2025 17:02:04 +0300 Subject: [PATCH 55/88] fixed arguments in ctx when the same keys defined --- ...CalculatedFieldEntityMessageProcessor.java | 44 ++++++++-------- .../cf/ctx/state/CalculatedFieldCtx.java | 19 ++++--- .../cf/CalculatedFieldIntegrationTest.java | 50 +++++++++++++++++++ 3 files changed, 84 insertions(+), 29 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 35539834c3..a905738a2f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -346,21 +346,22 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return mapToArguments(argNames, data); } - private Map mapToArguments(Map argNames, List data) { - if (argNames.isEmpty()) { + private Map mapToArguments(Map> args, List data) { + if (args.isEmpty()) { return Collections.emptyMap(); } Map arguments = new HashMap<>(); for (TsKvProto item : data) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); - String argName = argNames.get(key); - if (argName != null) { - arguments.put(argName, new SingleValueArgumentEntry(item)); + Set argNames = args.get(key); + if (argNames != null) { + argNames.forEach(argName -> arguments.put(argName, new SingleValueArgumentEntry(item))); } + key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null); - argName = argNames.get(key); - if (argName != null) { - arguments.put(argName, new SingleValueArgumentEntry(item)); + argNames = args.get(key); + if (argNames != null) { + argNames.forEach(argName -> arguments.put(argName, new SingleValueArgumentEntry(item))); } } return arguments; @@ -378,13 +379,13 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return mapToArguments(argNames, scope, attrDataList); } - private Map mapToArguments(Map argNames, AttributeScopeProto scope, List attrDataList) { + private Map mapToArguments(Map> args, AttributeScopeProto scope, List attrDataList) { Map arguments = new HashMap<>(); for (AttributeValueProto item : attrDataList) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); - String argName = argNames.get(key); - if (argName != null) { - arguments.put(argName, new SingleValueArgumentEntry(item)); + Set argNames = args.get(key); + if (argNames != null) { + argNames.forEach(argName -> arguments.put(argName, new SingleValueArgumentEntry(item))); } } return arguments; @@ -402,18 +403,19 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return mapToArgumentsWithDefaultValue(ctx.getMainEntityArguments(), ctx.getArguments(), scope, removedAttrKeys); } - private Map mapToArgumentsWithDefaultValue(Map argNames, Map configArguments, AttributeScopeProto scope, List removedAttrKeys) { + private Map mapToArgumentsWithDefaultValue(Map> args, Map configArguments, AttributeScopeProto scope, List removedAttrKeys) { Map arguments = new HashMap<>(); for (String removedKey : removedAttrKeys) { ReferencedEntityKey key = new ReferencedEntityKey(removedKey, ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); - String argName = argNames.get(key); - if (argName != null) { - Argument argument = configArguments.get(argName); - String defaultValue = (argument != null) ? argument.getDefaultValue() : null; - arguments.put(argName, StringUtils.isNotEmpty(defaultValue) - ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) - : new SingleValueArgumentEntry()); - + Set argNames = args.get(key); + if (argNames != null) { + argNames.forEach(argName -> { + Argument argument = configArguments.get(argName); + String defaultValue = (argument != null) ? argument.getDefaultValue() : null; + arguments.put(argName, StringUtils.isNotEmpty(defaultValue) + ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) + : new SingleValueArgumentEntry()); + }); } } return arguments; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index c9eaaef19a..9cb68bbc27 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -42,8 +42,10 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import static org.thingsboard.common.util.ExpressionFunctionsUtil.userDefinedFunctions; @@ -57,8 +59,8 @@ public class CalculatedFieldCtx { private EntityId entityId; private CalculatedFieldType cfType; private final Map arguments; - private final Map mainEntityArguments; - private final Map> linkedEntityArguments; + private final Map> mainEntityArguments; + private final Map>> linkedEntityArguments; private final List argNames; private Output output; private String expression; @@ -88,9 +90,10 @@ public class CalculatedFieldCtx { var refId = entry.getValue().getRefEntityId(); var refKey = entry.getValue().getRefEntityKey(); if (refId == null || refId.equals(calculatedField.getEntityId())) { - mainEntityArguments.put(refKey, entry.getKey()); + mainEntityArguments.computeIfAbsent(refKey, key -> new HashSet<>()).add(entry.getKey()); } else { - linkedEntityArguments.computeIfAbsent(refId, key -> new HashMap<>()).put(refKey, entry.getKey()); + linkedEntityArguments.computeIfAbsent(refId, key -> new HashMap<>()) + .computeIfAbsent(refKey, key -> new HashSet<>()).add(entry.getKey()); } } this.argNames = new ArrayList<>(arguments.keySet()); @@ -182,7 +185,7 @@ public class CalculatedFieldCtx { return map != null && matchesTimeSeries(map, values); } - private boolean matchesAttributes(Map argMap, List values, AttributeScope scope) { + private boolean matchesAttributes(Map> argMap, List values, AttributeScope scope) { if (argMap.isEmpty() || values.isEmpty()) { return false; } @@ -196,7 +199,7 @@ public class CalculatedFieldCtx { return false; } - private boolean matchesTimeSeries(Map argMap, List values) { + private boolean matchesTimeSeries(Map> argMap, List values) { if (argMap.isEmpty() || values.isEmpty()) { return false; } @@ -225,7 +228,7 @@ public class CalculatedFieldCtx { return matchesTimeSeriesKeys(mainEntityArguments, keys); } - private boolean matchesAttributesKeys(Map argMap, List keys, AttributeScope scope) { + private boolean matchesAttributesKeys(Map> argMap, List keys, AttributeScope scope) { if (argMap.isEmpty() || keys.isEmpty()) { return false; } @@ -240,7 +243,7 @@ public class CalculatedFieldCtx { return false; } - private boolean matchesTimeSeriesKeys(Map argMap, List keys) { + private boolean matchesTimeSeriesKeys(Map> argMap, List keys) { if (argMap.isEmpty() || keys.isEmpty()) { return false; } diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index da4b5758cc..b500f95d45 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -659,6 +659,56 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes }); } + @Test + public void testCalculatedFieldWhenTheSameTelemetryKeysUsed() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"a\":5}")); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("a + b"); + calculatedField.setDebugSettings(DebugSettings.all()); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("a", ArgumentType.TS_LATEST, null); + Argument argumentA = new Argument(); + argumentA.setRefEntityKey(refEntityKey); + Argument argumentB = new Argument(); + argumentB.setRefEntityKey(refEntityKey); + config.setArguments(Map.of("a", argumentA, "b", argumentB)); + config.setExpression("a + b"); + + Output output = new Output(); + output.setName("c"); + output.setType(OutputType.TIME_SERIES); + output.setDecimalsByDefault(0); + config.setOutput(output); + + calculatedField.setConfiguration(config); + + doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + await().alias("create CF -> perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode c = getLatestTelemetry(testDevice.getId(), "c"); + assertThat(c).isNotNull(); + assertThat(c.get("c").get(0).get("value").asText()).isEqualTo("10"); + }); + + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"a\":10}")); + + await().alias("update telemetry -> recalculate state").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode c = getLatestTelemetry(testDevice.getId(), "c"); + assertThat(c).isNotNull(); + assertThat(c.get("c").get(0).get("value").asText()).isEqualTo("20"); + }); + } + private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); } From 5e8de6b955f432a244dbb5c4406bc970ee18c5a8 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 21 Oct 2025 10:33:20 +0300 Subject: [PATCH 56/88] renamed cf --- ...CalculatedFieldEntityMessageProcessor.java | 12 +++++----- ...alculatedFieldManagerMessageProcessor.java | 12 +++++----- ...tractCalculatedFieldProcessingService.java | 8 +++---- .../cf/DefaultCalculatedFieldCache.java | 6 ++--- .../DefaultCalculatedFieldQueueService.java | 4 ++-- .../cf/ctx/state/CalculatedFieldCtx.java | 22 +++++++++---------- ...itiesAggregationCalculatedFieldState.java} | 10 ++++----- .../utils/CalculatedFieldArgumentUtils.java | 4 ++-- .../server/utils/CalculatedFieldUtils.java | 10 ++++----- ...titiesAggregationCalculatedFieldTest.java} | 18 +++++++-------- .../common/data/cf/CalculatedFieldType.java | 2 +- .../CalculatedFieldConfiguration.java | 4 ++-- ...regationCalculatedFieldConfiguration.java} | 4 ++-- .../CalculatedFieldDataValidator.java | 4 ++-- 14 files changed, 60 insertions(+), 60 deletions(-) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/{LatestValuesAggregationCalculatedFieldState.java => RelaredEntitiesAggregationCalculatedFieldState.java} (94%) rename application/src/test/java/org/thingsboard/server/cf/{LatestValuesAggregationCalculatedFieldTest.java => RelatedEntitiesAggregationCalculatedFieldTest.java} (97%) rename common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/{LatestValuesAggregationCalculatedFieldConfiguration.java => RelatedEntitiesAggregationCalculatedFieldConfiguration.java} (91%) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 1252b8091b..5a1ada08c5 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -54,7 +54,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelaredEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -254,8 +254,8 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM Map fetchedArgs = fetchAggArguments(ctx, relatedEntityId); Map updatedArgs = state.update(fetchedArgs, ctx); - if (state instanceof LatestValuesAggregationCalculatedFieldState latestValuesState) { - latestValuesState.setLastMetricsEvalTs(-1); + if (state instanceof RelaredEntitiesAggregationCalculatedFieldState relatedEntitiesAggState) { + relatedEntitiesAggState.setLastMetricsEvalTs(-1); } state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); @@ -271,7 +271,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM msg.getCallback().onSuccess(); return; } - if (state instanceof LatestValuesAggregationCalculatedFieldState aggState) { + if (state instanceof RelaredEntitiesAggregationCalculatedFieldState aggState) { cleanupAggregationState(msg.getRelatedEntityId(), aggState); processStateIfReady(state, Collections.emptyMap(), state.getCtx(), Collections.emptyList(), null, null, msg.getCallback()); } else { @@ -279,7 +279,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } - private void cleanupAggregationState(EntityId relatedEntityId, LatestValuesAggregationCalculatedFieldState state) { + private void cleanupAggregationState(EntityId relatedEntityId, RelaredEntitiesAggregationCalculatedFieldState state) { state.getArguments().values().forEach(argEntry -> { AggArgumentEntry aggEntry = (AggArgumentEntry) argEntry; aggEntry.getAggInputs().remove(relatedEntityId); @@ -691,7 +691,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, deletedArguments); - if (CalculatedFieldType.LATEST_VALUES_AGGREGATION.equals(ctx.getCfType())) { + if (CalculatedFieldType.RELATED_ENTITIES_AGGREGATION.equals(ctx.getCfType())) { fetchedArgs = fetchedArgs.entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 24fd5d320b..5b4528370b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; @@ -346,7 +346,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware List matchingCfs = cfsByEntityIdAndProfile.stream() .filter(cf -> { - var config = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); + var config = (RelatedEntitiesAggregationCalculatedFieldConfiguration) cf.getCalculatedField().getConfiguration(); RelationPathLevel relation = config.getRelation(); return direction.equals(relation.direction()) && relationType.equals(relation.relationType()); }) @@ -377,7 +377,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } calculatedFields.put(cf.getId(), cfCtx); - if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + if (cf.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig) { aggCalculatedFields.put(cf.getId(), cfCtx); } // We use copy on write lists to safely pass the reference to another actor for the iteration. @@ -411,7 +411,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(newCfCtx).eventEntity(newCfCtx.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(newCf.getId(), newCfCtx); - if (newCf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + if (newCf.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration) { aggCalculatedFields.put(newCf.getId(), newCfCtx); } List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); @@ -518,7 +518,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private List findRelationsForCf(EntityId entityId, CalculatedFieldCtx cf) { List result = new ArrayList<>(); - if (cf.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration configuration) { + if (cf.getCalculatedField().getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration configuration) { RelationPathLevel relation = configuration.getRelation(); EntitySearchDirection inverseDirection = switch (relation.direction()) { case FROM -> EntitySearchDirection.TO; @@ -753,7 +753,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); } finally { calculatedFields.put(cf.getId(), cfCtx); - if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + if (cf.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration) { aggCalculatedFields.put(cf.getId(), cfCtx); } // We use copy on write lists to safely pass the reference to another actor for the iteration. diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 1c606cf840..3ddc8e1067 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -27,7 +27,7 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; @@ -98,7 +98,7 @@ public abstract class AbstractCalculatedFieldProcessingService { Map> argFutures = switch (ctx.getCfType()) { case GEOFENCING -> fetchGeofencingCalculatedFieldArguments(ctx, entityId, false, ts); case SIMPLE, SCRIPT, ALARM, PROPAGATION -> getBaseCalculatedFieldArguments(ctx, entityId, ts); - case LATEST_VALUES_AGGREGATION -> fetchAggArguments(ctx, entityId, ts); + case RELATED_ENTITIES_AGGREGATION -> fetchAggArguments(ctx, entityId, ts); }; if (ctx.getCfType() == PROPAGATION) { argFutures.put(PROPAGATION_CONFIG_ARGUMENT, fetchPropagationCalculatedFieldArgument(ctx, entityId)); @@ -190,7 +190,7 @@ public abstract class AbstractCalculatedFieldProcessingService { } protected Map> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { - LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig = (RelatedEntitiesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); ListenableFuture> relatedEntitiesFut = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getRelation()); @@ -202,7 +202,7 @@ public abstract class AbstractCalculatedFieldProcessingService { } protected ListenableFuture> fetchEntityAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { - LatestValuesAggregationCalculatedFieldConfiguration aggConfig = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig = (RelatedEntitiesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); Map> argsFutures = aggConfig.getArguments().entrySet().stream() .collect(Collectors.toMap( diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 918d71e326..1c755ee05a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -27,7 +27,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; @@ -83,7 +83,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { cfs.forEach(cf -> { if (cf != null) { calculatedFields.putIfAbsent(cf.getId(), cf); - if (cf.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + if (cf.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration) { aggCalculatedFields.put(cf.getId(), cf); } } @@ -200,7 +200,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new CopyOnWriteArrayList<>()).add(calculatedField); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); - if (configuration instanceof LatestValuesAggregationCalculatedFieldConfiguration) { + if (configuration instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration) { aggCalculatedFields.put(calculatedField.getId(), calculatedField); } calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 147882c3c2..84f49c7c9e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -27,7 +27,7 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -190,7 +190,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS List cfCtxs = calculatedFieldCache.getAggCalculatedFieldCtxsByFilter(relatedEntityFilter); for (CalculatedFieldCtx cfCtx : cfCtxs) { - if (cfCtx.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + if (cfCtx.getCalculatedField().getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig) { RelationPathLevel relation = aggConfig.getRelation(); EntitySearchDirection inverseDirection = switch (relation.direction()) { case FROM -> EntitySearchDirection.TO; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 89ed4e4401..46442847a0 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -44,7 +44,7 @@ import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.ScheduledUpdateSupportedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -139,7 +139,7 @@ public class CalculatedFieldCtx { var refId = entry.getValue().getRefEntityId(); var refKey = entry.getValue().getRefEntityKey(); if (refId == null) { - if (CalculatedFieldType.LATEST_VALUES_AGGREGATION.equals(cfType)) { + if (CalculatedFieldType.RELATED_ENTITIES_AGGREGATION.equals(cfType)) { relatedEntityArguments.put(refKey, entry.getKey()); continue; } @@ -185,7 +185,7 @@ public class CalculatedFieldCtx { this.scheduledUpdateIntervalMillis = scheduledConfig.isScheduledUpdateEnabled() ? TimeUnit.SECONDS.toMillis(scheduledConfig.getScheduledUpdateInterval()) : -1L; } this.requiresScheduledReevaluation = calculatedField.getConfiguration().requiresScheduledReevaluation(); - if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfig) { + if (calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig) { this.useLatestTs = aggConfig.isUseLatestTs(); } this.systemContext = systemContext; @@ -228,8 +228,8 @@ public class CalculatedFieldCtx { } initialized = true; } - case LATEST_VALUES_AGGREGATION -> { - LatestValuesAggregationCalculatedFieldConfiguration configuration = (LatestValuesAggregationCalculatedFieldConfiguration) calculatedField.getConfiguration(); + case RELATED_ENTITIES_AGGREGATION -> { + RelatedEntitiesAggregationCalculatedFieldConfiguration configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) calculatedField.getConfiguration(); configuration.getMetrics().forEach((key, metric) -> { if (metric.getInput() instanceof AggFunctionInput functionInput) { initTbelExpression(functionInput.getFunction()); @@ -594,8 +594,8 @@ public class CalculatedFieldCtx { if (scheduledUpdateIntervalMillis != other.scheduledUpdateIntervalMillis) { return true; } - if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig - && other.getCalculatedField().getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig + if (calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration thisConfig + && other.getCalculatedField().getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration otherConfig && (thisConfig.getDeduplicationIntervalInSec() != otherConfig.getDeduplicationIntervalInSec() || !thisConfig.getMetrics().equals(otherConfig.getMetrics()))) { return true; } @@ -617,7 +617,7 @@ public class CalculatedFieldCtx { if (hasGeofencingZoneGroupConfigurationChanges(other)) { return true; } - if (hasLatestValuesAggregationConfigurationChanges(other)) { + if (hasRelatedEntitiesAggregationConfigurationChanges(other)) { return true; } return false; @@ -631,9 +631,9 @@ public class CalculatedFieldCtx { return false; } - private boolean hasLatestValuesAggregationConfigurationChanges(CalculatedFieldCtx other) { - if (calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration thisConfig - && other.calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration otherConfig) { + private boolean hasRelatedEntitiesAggregationConfigurationChanges(CalculatedFieldCtx other) { + if (calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration thisConfig + && other.calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration otherConfig) { return !thisConfig.getArguments().equals(otherConfig.getArguments()) || !thisConfig.getRelation().equals(otherConfig.getRelation()); } return false; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelaredEntitiesAggregationCalculatedFieldState.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelaredEntitiesAggregationCalculatedFieldState.java index fdc045a0db..b53042b1fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/LatestValuesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelaredEntitiesAggregationCalculatedFieldState.java @@ -30,7 +30,7 @@ import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFuncti import org.thingsboard.server.common.data.cf.configuration.aggregation.AggInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.TelemetryCalculatedFieldResult; @@ -46,7 +46,7 @@ import java.util.Map.Entry; @Slf4j @Getter -public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedFieldState { +public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculatedFieldState { @Setter private long lastArgsRefreshTs = -1; @@ -57,14 +57,14 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF private final Map> inputs = new HashMap<>(); - public LatestValuesAggregationCalculatedFieldState(EntityId entityId) { + public RelaredEntitiesAggregationCalculatedFieldState(EntityId entityId) { super(entityId); } @Override public void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx) { super.setCtx(ctx, actorCtx); - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); + var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); metrics = configuration.getMetrics(); deduplicationInterval = configuration.getDeduplicationIntervalInSec(); } @@ -86,7 +86,7 @@ public class LatestValuesAggregationCalculatedFieldState extends BaseCalculatedF @Override public CalculatedFieldType getType() { - return CalculatedFieldType.LATEST_VALUES_AGGREGATION; + return CalculatedFieldType.RELATED_ENTITIES_AGGREGATION; } @Override diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index d30b4eaf93..e1763ffa2d 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -35,7 +35,7 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelaredEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalculatedFieldState; @@ -91,7 +91,7 @@ public class CalculatedFieldArgumentUtils { case GEOFENCING -> new GeofencingCalculatedFieldState(entityId); case ALARM -> new AlarmCalculatedFieldState(entityId); case PROPAGATION -> new PropagationCalculatedFieldState(entityId); - case LATEST_VALUES_AGGREGATION -> new LatestValuesAggregationCalculatedFieldState(entityId); + case RELATED_ENTITIES_AGGREGATION -> new RelaredEntitiesAggregationCalculatedFieldState(entityId); }; } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 6e65b4d5d9..67af13fdb3 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -49,7 +49,7 @@ import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.LatestValuesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelaredEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; @@ -120,7 +120,7 @@ public class CalculatedFieldUtils { alarmStateProto.setClearRuleState(toAlarmRuleStateProto(alarmState.getClearRuleState())); } } - if (state instanceof LatestValuesAggregationCalculatedFieldState aggState) { + if (state instanceof RelaredEntitiesAggregationCalculatedFieldState aggState) { aggBuilder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); builder.setLatestValuesAggregationState(aggBuilder.build()); } @@ -214,7 +214,7 @@ public class CalculatedFieldUtils { case GEOFENCING -> new GeofencingCalculatedFieldState(id.entityId()); case ALARM -> new AlarmCalculatedFieldState(id.entityId()); case PROPAGATION -> new PropagationCalculatedFieldState(id.entityId()); - case LATEST_VALUES_AGGREGATION -> new LatestValuesAggregationCalculatedFieldState(id.entityId()); + case RELATED_ENTITIES_AGGREGATION -> new RelaredEntitiesAggregationCalculatedFieldState(id.entityId()); }; proto.getSingleValueArgumentsList().forEach(argProto -> @@ -240,8 +240,8 @@ public class CalculatedFieldUtils { alarmState.setClearRuleState(fromAlarmRuleStateProto(alarmStateProto.getClearRuleState(), alarmState)); } } - case LATEST_VALUES_AGGREGATION -> { - LatestValuesAggregationCalculatedFieldState aggState = (LatestValuesAggregationCalculatedFieldState) state; + case RELATED_ENTITIES_AGGREGATION -> { + RelaredEntitiesAggregationCalculatedFieldState aggState = (RelaredEntitiesAggregationCalculatedFieldState) state; LatestValuesAggregationStateProto aggregationStateProto = proto.getLatestValuesAggregationState(); Map> arguments = new HashMap<>(); aggregationStateProto.getAggArgumentsList().forEach(argProto -> { diff --git a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java b/application/src/test/java/org/thingsboard/server/cf/RelatedEntitiesAggregationCalculatedFieldTest.java similarity index 97% rename from application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java rename to application/src/test/java/org/thingsboard/server/cf/RelatedEntitiesAggregationCalculatedFieldTest.java index eb11a329be..068c5f851d 100644 --- a/application/src/test/java/org/thingsboard/server/cf/LatestValuesAggregationCalculatedFieldTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/RelatedEntitiesAggregationCalculatedFieldTest.java @@ -39,7 +39,7 @@ import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFuncti import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunctionInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration; import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; @@ -66,7 +66,7 @@ import static org.thingsboard.server.cf.CalculatedFieldIntegrationTest.POLL_INTE @Slf4j @DaoSqlTest -public class LatestValuesAggregationCalculatedFieldTest extends AbstractControllerTest { +public class RelatedEntitiesAggregationCalculatedFieldTest extends AbstractControllerTest { private Tenant savedTenant; @@ -470,7 +470,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll createEntityRelation(asset.getId(), device3.getId(), "Has"); postTelemetry(device3.getId(), "{\"occupied\":true}"); - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); configuration.setRelation(new RelationPathLevel(EntitySearchDirection.FROM, "Has")); saveCalculatedField(cf); @@ -493,7 +493,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll postTelemetry(device1.getId(), "{\"occupiedStatus\":false}"); postTelemetry(device2.getId(), "{\"occupiedStatus\":false}"); - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); Argument argument = new Argument(); argument.setRefEntityKey(new ReferencedEntityKey("oc", ArgumentType.TS_LATEST, null)); argument.setDefaultValue("false"); @@ -523,7 +523,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); }); - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); AggMetric aggMetric = new AggMetric(); aggMetric.setInput(new AggKeyInput("temp")); aggMetric.setFilter("return temp < 100;"); @@ -559,7 +559,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); }); - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); Output output = new Output(); output.setType(OutputType.ATTRIBUTES); output.setScope(AttributeScope.SERVER_SCOPE); @@ -588,7 +588,7 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll verifyTelemetry(asset.getId(), Map.of("avgTemperature", "24")); }); - var configuration = (LatestValuesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); + var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) cf.getConfiguration(); configuration.setDeduplicationIntervalInSec(2 * deduplicationInterval); saveCalculatedField(cf); @@ -730,9 +730,9 @@ public class LatestValuesAggregationCalculatedFieldTest extends AbstractControll CalculatedField calculatedField = new CalculatedField(); calculatedField.setName(name); calculatedField.setEntityId(entityId); - calculatedField.setType(CalculatedFieldType.LATEST_VALUES_AGGREGATION); + calculatedField.setType(CalculatedFieldType.RELATED_ENTITIES_AGGREGATION); - LatestValuesAggregationCalculatedFieldConfiguration configuration = new LatestValuesAggregationCalculatedFieldConfiguration(); + RelatedEntitiesAggregationCalculatedFieldConfiguration configuration = new RelatedEntitiesAggregationCalculatedFieldConfiguration(); configuration.setRelation(relation); configuration.setArguments(inputs); configuration.setDeduplicationIntervalInSec(deduplicationInterval); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java index fe9ee7f9aa..4463c835db 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -26,7 +26,7 @@ public enum CalculatedFieldType { GEOFENCING, ALARM, PROPAGATION, - LATEST_VALUES_AGGREGATION; + RELATED_ENTITIES_AGGREGATION; public static final Set all = Collections.unmodifiableSet(EnumSet.allOf(CalculatedFieldType.class)); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 3072b7c546..ca0a3c1a54 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -42,7 +42,7 @@ import java.util.stream.Collectors; @Type(value = GeofencingCalculatedFieldConfiguration.class, name = "GEOFENCING"), @Type(value = AlarmCalculatedFieldConfiguration.class, name = "ALARM"), @Type(value = PropagationCalculatedFieldConfiguration.class, name = "PROPAGATION"), - @Type(value = LatestValuesAggregationCalculatedFieldConfiguration.class, name = "LATEST_VALUES_AGGREGATION") + @Type(value = RelatedEntitiesAggregationCalculatedFieldConfiguration.class, name = "RELATED_ENTITIES_AGGREGATION") }) @JsonIgnoreProperties(ignoreUnknown = true) public interface CalculatedFieldConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java similarity index 91% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java index 5f8fb65265..69e4ee7fdf 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/LatestValuesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.relation.RelationPathLevel; import java.util.Map; @Data -public class LatestValuesAggregationCalculatedFieldConfiguration implements ArgumentsBasedCalculatedFieldConfiguration { +public class RelatedEntitiesAggregationCalculatedFieldConfiguration implements ArgumentsBasedCalculatedFieldConfiguration { private RelationPathLevel relation; private Map arguments; @@ -36,7 +36,7 @@ public class LatestValuesAggregationCalculatedFieldConfiguration implements Argu @Override public CalculatedFieldType getType() { - return CalculatedFieldType.LATEST_VALUES_AGGREGATION; + return CalculatedFieldType.RELATED_ENTITIES_AGGREGATION; } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java index 319a4fbe8b..3ccb837b3f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -21,7 +21,7 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.configuration.ArgumentsBasedCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScheduledUpdateSupportedCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.cf.CalculatedFieldDao; @@ -113,7 +113,7 @@ public class CalculatedFieldDataValidator extends DataValidator } private void validateAggregationConfiguration(TenantId tenantId, CalculatedField calculatedField) { - if (!(calculatedField.getConfiguration() instanceof LatestValuesAggregationCalculatedFieldConfiguration aggConfiguration)) { + if (!(calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfiguration)) { return; } long minAllowedDeduplicationInterval = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMinAllowedDeduplicationIntervalInSecForCF); From 126d33a047690959ea7cebb2c80943d482680038 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Tue, 21 Oct 2025 12:04:20 +0300 Subject: [PATCH 57/88] fixed flaky test --- .../thingsboard/server/controller/DeviceControllerTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java index 2a2f7bc888..50099d7782 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java @@ -1638,7 +1638,8 @@ public class DeviceControllerTest extends AbstractControllerTest { Assert.assertEquals(deviceName, savedDevice.getName()); Assert.assertEquals(deviceType, savedDevice.getType()); - Optional retrieved = attributesService.find(tenantId, savedDevice.getId(), AttributeScope.SERVER_SCOPE, "attr").get(); + Optional retrieved = await().atMost(5, TimeUnit.SECONDS) + .until(() -> attributesService.find(tenantId, savedDevice.getId(), AttributeScope.SERVER_SCOPE, "attr").get(), Optional::isPresent); assertThat(retrieved.get().getJsonValue().get()).isEqualTo(deviceAttr); assertThat(retrieved.get().getStrValue()).isNotPresent(); } From 2778f79e5e8a1d9262450a33edcfbc77d64e88c2 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 21 Oct 2025 12:45:29 +0300 Subject: [PATCH 58/88] minor refactoring --- ...CalculatedFieldEntityMessageProcessor.java | 80 +++++++------------ ...alculatedFieldManagerMessageProcessor.java | 4 +- ...tractCalculatedFieldProcessingService.java | 14 ---- .../cf/CalculatedFieldProcessingService.java | 2 - ...faultCalculatedFieldProcessingService.java | 5 -- .../service/cf/ctx/state/ArgumentEntry.java | 6 +- .../cf/ctx/state/ArgumentEntryType.java | 2 +- .../AggSingleEntityArgumentEntry.java | 1 - ...itiesAggregationCalculatedFieldState.java} | 64 ++++++++------- ...java => RelatedEntitiesArgumentEntry.java} | 20 ++--- .../utils/CalculatedFieldArgumentUtils.java | 4 +- .../server/utils/CalculatedFieldUtils.java | 18 ++--- ... => RelatedEntitiesArgumentEntryTest.java} | 27 ++----- 13 files changed, 96 insertions(+), 151 deletions(-) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/{RelaredEntitiesAggregationCalculatedFieldState.java => RelatedEntitiesAggregationCalculatedFieldState.java} (84%) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/{AggArgumentEntry.java => RelatedEntitiesArgumentEntry.java} (72%) rename application/src/test/java/org/thingsboard/server/service/cf/ctx/state/{AggArgumentEntryTest.java => RelatedEntitiesArgumentEntryTest.java} (80%) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 5a1ada08c5..434f555325 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -52,9 +52,8 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.RelaredEntitiesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -227,17 +226,19 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); var state = states.get(ctx.getCfId()); try { - boolean justRestored = false; + Map updatedArgs = new HashMap<>(); if (state == null) { state = createState(ctx); - justRestored = true; + } else { + if (state instanceof RelatedEntitiesAggregationCalculatedFieldState relatedEntitiesAggState) { + Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, msg.getRelatedEntityId(), ctx.getArguments()); + updatedArgs = relatedEntitiesAggState.updateEntityData(toAggSingleEntityArguments(msg.getRelatedEntityId(), fetchedArgs)); + } + + state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); } if (state.isSizeOk()) { - Map updatedArgs = new HashMap<>(); - if (!justRestored) { - updatedArgs = updateAggregationState(msg.getRelatedEntityId(), state, ctx); - } - processStateIfReady(state, updatedArgs, ctx, new ArrayList<>(), null, null, callback); + processStateIfReady(state, updatedArgs, ctx, Collections.singletonList(ctx.getCfId()), null, null, callback); } else { throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(); } @@ -250,19 +251,6 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } - private Map updateAggregationState(EntityId relatedEntityId, CalculatedFieldState state, CalculatedFieldCtx ctx) { - Map fetchedArgs = fetchAggArguments(ctx, relatedEntityId); - Map updatedArgs = state.update(fetchedArgs, ctx); - - if (state instanceof RelaredEntitiesAggregationCalculatedFieldState relatedEntitiesAggState) { - relatedEntitiesAggState.setLastMetricsEvalTs(-1); - } - - state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); - - return updatedArgs; - } - private void handleRelationDelete(CalculatedFieldRelatedEntityMsg msg) throws CalculatedFieldException { CalculatedFieldCtx ctx = msg.getCalculatedField(); CalculatedFieldId cfId = ctx.getCfId(); @@ -271,34 +259,22 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM msg.getCallback().onSuccess(); return; } - if (state instanceof RelaredEntitiesAggregationCalculatedFieldState aggState) { - cleanupAggregationState(msg.getRelatedEntityId(), aggState); - processStateIfReady(state, Collections.emptyMap(), state.getCtx(), Collections.emptyList(), null, null, msg.getCallback()); + if (state instanceof RelatedEntitiesAggregationCalculatedFieldState aggState) { + aggState.cleanupEntityData(msg.getRelatedEntityId()); + + state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); + + if (state.isSizeOk()) { + processStateIfReady(state, Collections.emptyMap(), ctx, Collections.singletonList(ctx.getCfId()), null, null, msg.getCallback()); + } else { + throw new RuntimeException(ctx.getSizeExceedsLimitMessage()); + } } else { + // todo: log msg.getCallback().onSuccess(); } } - private void cleanupAggregationState(EntityId relatedEntityId, RelaredEntitiesAggregationCalculatedFieldState state) { - state.getArguments().values().forEach(argEntry -> { - AggArgumentEntry aggEntry = (AggArgumentEntry) argEntry; - aggEntry.getAggInputs().remove(relatedEntityId); - }); - state.getInputs().remove(relatedEntityId); - state.setLastMetricsEvalTs(-1); - } - - @SneakyThrows - private Map fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId) { - ListenableFuture> argumentsFuture = cfService.fetchAggEntityArguments(ctx, entityId); - // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. - // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. - // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, - // but this will significantly complicate the code. - return argumentsFuture.get(1, TimeUnit.MINUTES); - } - - public void process(EntityCalculatedFieldTelemetryMsg msg) throws CalculatedFieldException { log.trace("[{}] Processing CF telemetry msg: {}", msg.getEntityId(), msg); var proto = msg.getProto(); @@ -692,17 +668,21 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, deletedArguments); if (CalculatedFieldType.RELATED_ENTITIES_AGGREGATION.equals(ctx.getCfType())) { - fetchedArgs = fetchedArgs.entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - argEntry -> new AggSingleEntityArgumentEntry(entityId, argEntry.getValue()) - )); + fetchedArgs = toAggSingleEntityArguments(entityId, fetchedArgs); } fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); return fetchedArgs; } + private Map toAggSingleEntityArguments(EntityId relatedEntityId, Map fetchedArgs) { + return fetchedArgs.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + argEntry -> new AggSingleEntityArgumentEntry(relatedEntityId, argEntry.getValue()) + )); + } + private static List getCalculatedFieldIds(CalculatedFieldTelemetryMsgProto proto) { List cfIds = new LinkedList<>(); for (var cfId : proto.getPreviousCalculatedFieldsList()) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 5b4528370b..7f1c7b1925 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -469,8 +469,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); log.debug("Received telemetry msg from entity [{}]", entityId); - // 3 = 1 for CF processing + 1 for links processing + 1 for owner entity processing - MultipleTbCallback callback = new MultipleTbCallback(3, msg.getCallback()); + // 4 = 1 for CF processing + 1 for links processing + 1 for owner entity processing + 1 for aggregation processing + MultipleTbCallback callback = new MultipleTbCallback(4, msg.getCallback()); // process all cfs related to entity, or it's profile; var entityIdFields = getCalculatedFieldsByEntityId(entityId); var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 3ddc8e1067..0c0f2f23e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -201,20 +201,6 @@ public abstract class AbstractCalculatedFieldProcessingService { )); } - protected ListenableFuture> fetchEntityAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { - RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig = (RelatedEntitiesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); - - Map> argsFutures = aggConfig.getArguments().entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> fetchSingleAggArgumentEntry(ctx.getTenantId(), entityId, entry.getValue(), ts) - )); - - return Futures.whenAllComplete(argsFutures.values()) - .call(() -> resolveArgumentFutures(argsFutures), - MoreExecutors.directExecutor()); - } - private ListenableFuture> resolveGeofencingEntityIds(TenantId tenantId, EntityId entityId, Map.Entry entry) { Argument value = entry.getValue(); if (value.getRefEntityId() != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java index 52b3341151..a9139572b8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -33,8 +33,6 @@ public interface CalculatedFieldProcessingService { ListenableFuture> fetchArguments(CalculatedFieldCtx ctx, EntityId entityId); - ListenableFuture> fetchAggEntityArguments(CalculatedFieldCtx ctx, EntityId entityId); - Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId); Map fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map arguments); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index 9074c1036f..d7957dce9b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -91,11 +91,6 @@ public class DefaultCalculatedFieldProcessingService extends AbstractCalculatedF return super.fetchArguments(ctx, entityId, System.currentTimeMillis()); } - @Override - public ListenableFuture> fetchAggEntityArguments(CalculatedFieldCtx ctx, EntityId entityId) { - return super.fetchEntityAggArguments(ctx, entityId, System.currentTimeMillis()); - } - @Override public Map fetchDynamicArgsFromDb(CalculatedFieldCtx ctx, EntityId entityId) { return switch (ctx.getCfType()) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index c8f7dd0c3d..5bb16292be 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -22,7 +22,7 @@ import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry; @@ -40,7 +40,7 @@ import java.util.Map; @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING"), @JsonSubTypes.Type(value = PropagationArgumentEntry.class, name = "PROPAGATION"), - @JsonSubTypes.Type(value = AggArgumentEntry.class, name = "AGGREGATE_LATEST"), + @JsonSubTypes.Type(value = RelatedEntitiesArgumentEntry.class, name = "AGGREGATE_LATEST"), @JsonSubTypes.Type(value = AggSingleEntityArgumentEntry.class, name = "AGGREGATE_LATEST_SINGLE") }) public interface ArgumentEntry { @@ -77,7 +77,7 @@ public interface ArgumentEntry { } static ArgumentEntry createAggArgument(Map entityIdkvEntryMap) { - return new AggArgumentEntry(entityIdkvEntryMap, false); + return new RelatedEntitiesArgumentEntry(entityIdkvEntryMap, false); } static ArgumentEntry createAggSingleArgument(EntityId entityId, KvEntry kvEntry) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java index 9882b8181b..5c672cf04e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java @@ -16,5 +16,5 @@ package org.thingsboard.server.service.cf.ctx.state; public enum ArgumentEntryType { - SINGLE_VALUE, TS_ROLLING, GEOFENCING, PROPAGATION, AGGREGATE_LATEST, AGGREGATE_LATEST_SINGLE + SINGLE_VALUE, TS_ROLLING, GEOFENCING, PROPAGATION, RELATED_ENTITIES, AGGREGATE_LATEST_SINGLE } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java index 6430fe3f1c..b935256860 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java @@ -33,7 +33,6 @@ import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; public class AggSingleEntityArgumentEntry extends SingleValueArgumentEntry { private EntityId entityId; - private boolean deleted; public AggSingleEntityArgumentEntry(EntityId entityId, ArgumentEntry entry) { super(entry); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelaredEntitiesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java similarity index 84% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelaredEntitiesAggregationCalculatedFieldState.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java index b53042b1fe..c0baca836c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelaredEntitiesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -46,7 +45,7 @@ import java.util.Map.Entry; @Slf4j @Getter -public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculatedFieldState { +public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculatedFieldState { @Setter private long lastArgsRefreshTs = -1; @@ -55,9 +54,7 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat private long deduplicationInterval = -1; private Map metrics; - private final Map> inputs = new HashMap<>(); - - public RelaredEntitiesAggregationCalculatedFieldState(EntityId entityId) { + public RelatedEntitiesAggregationCalculatedFieldState(EntityId entityId) { super(entityId); } @@ -75,7 +72,6 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat lastArgsRefreshTs = -1; lastMetricsEvalTs = -1; metrics = null; - inputs.clear(); } @Override @@ -91,17 +87,8 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat @Override public Map update(Map argumentValues, CalculatedFieldCtx ctx) { - Map updatedArguments = super.update(argumentValues, ctx); lastArgsRefreshTs = System.currentTimeMillis(); - for (Map.Entry argEntry : arguments.entrySet()) { - String key = argEntry.getKey(); - AggArgumentEntry aggArgumentEntry = (AggArgumentEntry) argEntry.getValue(); - Map aggInputs = aggArgumentEntry.getAggInputs(); - aggInputs.forEach((entityId, argumentEntry) -> { - inputs.computeIfAbsent(entityId, k -> new HashMap<>()).put(key, argumentEntry); - }); - } - return updatedArguments; + return super.update(argumentValues, ctx); } @Override @@ -115,7 +102,7 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() .type(output.getType()) .scope(output.getScope()) - .result(createResultJson(ctx.isUseLatestTs(), aggResult)) + .result(toSimpleResult(ctx.isUseLatestTs(), aggResult)) .build()); } else { return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() @@ -124,20 +111,47 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat } } + public Map updateEntityData(Map fetchedArgs) { + lastMetricsEvalTs = -1; + return update(fetchedArgs, ctx); + } + + public void cleanupEntityData(EntityId relatedEntityId) { + arguments.values().forEach(argEntry -> { + RelatedEntitiesArgumentEntry aggEntry = (RelatedEntitiesArgumentEntry) argEntry; + aggEntry.getAggInputs().remove(relatedEntityId); + }); + lastMetricsEvalTs = -1; + lastArgsRefreshTs = System.currentTimeMillis(); + } + private boolean shouldRecalculate() { boolean intervalPassed = lastMetricsEvalTs <= System.currentTimeMillis() - deduplicationInterval; boolean argsUpdatedDuringInterval = lastArgsRefreshTs > lastMetricsEvalTs; return intervalPassed && argsUpdatedDuringInterval; } + private Map> prepareInputs() { + Map> inputs = new HashMap<>(); + for (Map.Entry argEntry : arguments.entrySet()) { + String key = argEntry.getKey(); + RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry.getValue(); + relatedEntitiesArgumentEntry.getAggInputs().forEach((entityId, argumentEntry) -> { + inputs.computeIfAbsent(entityId, k -> new HashMap<>()).put(key, argumentEntry); + }); + } + return inputs; + } + private ObjectNode aggregateMetrics(Output output) throws Exception { ObjectNode aggResult = JacksonUtil.newObjectNode(); + Map> inputs = prepareInputs(); for (Entry entry : metrics.entrySet()) { String metricKey = entry.getKey(); AggMetric metric = entry.getValue(); AggEntry aggMetricEntry = AggFunctionFactory.createAggFunction(metric.getFunction()); - aggregateMetric(metric, aggMetricEntry); + aggregateMetric(metric, aggMetricEntry, inputs); aggMetricEntry.result().ifPresent(result -> { aggResult.set(metricKey, JacksonUtil.valueToTree(formatResult(result, output.getDecimalsByDefault()))); }); @@ -145,7 +159,7 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat return aggResult; } - private void aggregateMetric(AggMetric metric, AggEntry aggEntry) throws Exception { + private void aggregateMetric(AggMetric metric, AggEntry aggEntry, Map> inputs) throws Exception { for (Map entityInputs : inputs.values()) { if (applyAggregation(metric.getFilter(), entityInputs)) { Object arg = resolveAggregationInput(metric.getInput(), entityInputs); @@ -183,16 +197,4 @@ public class RelaredEntitiesAggregationCalculatedFieldState extends BaseCalculat } } - protected JsonNode createResultJson(boolean useLatestTs, JsonNode result) { - long latestTs = getLatestTimestamp(); - if (useLatestTs && latestTs != -1) { - ObjectNode resultNode = JacksonUtil.newObjectNode(); - resultNode.put("ts", latestTs); - resultNode.set("values", result); - return resultNode; - } else { - return result; - } - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java similarity index 72% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java index e002aece2c..5385808923 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java @@ -27,7 +27,7 @@ import java.util.Map; @Data @AllArgsConstructor -public class AggArgumentEntry implements ArgumentEntry { +public class RelatedEntitiesArgumentEntry implements ArgumentEntry { private final Map aggInputs; @@ -35,7 +35,7 @@ public class AggArgumentEntry implements ArgumentEntry { @Override public ArgumentEntryType getType() { - return ArgumentEntryType.AGGREGATE_LATEST; + return ArgumentEntryType.RELATED_ENTITIES; } @Override @@ -45,19 +45,15 @@ public class AggArgumentEntry implements ArgumentEntry { @Override public boolean updateEntry(ArgumentEntry entry) { - if (entry instanceof AggArgumentEntry aggArgumentEntry) { - aggInputs.putAll(aggArgumentEntry.aggInputs); + if (entry instanceof RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry) { + aggInputs.putAll(relatedEntitiesArgumentEntry.aggInputs); return true; } else if (entry instanceof AggSingleEntityArgumentEntry aggSingleEntityArgumentEntry) { - if (aggSingleEntityArgumentEntry.isDeleted()) { - aggInputs.remove(aggSingleEntityArgumentEntry.getEntityId()); + ArgumentEntry argumentEntry = aggInputs.get(aggSingleEntityArgumentEntry.getEntityId()); + if (argumentEntry != null) { + argumentEntry.updateEntry(aggSingleEntityArgumentEntry); } else { - ArgumentEntry argumentEntry = aggInputs.get(aggSingleEntityArgumentEntry.getEntityId()); - if (argumentEntry != null) { - argumentEntry.updateEntry(aggSingleEntityArgumentEntry); - } else { - aggInputs.put(aggSingleEntityArgumentEntry.getEntityId(), aggSingleEntityArgumentEntry); - } + aggInputs.put(aggSingleEntityArgumentEntry.getEntityId(), aggSingleEntityArgumentEntry); } return true; } else { diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index e1763ffa2d..239ffedbc7 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -35,7 +35,7 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.RelaredEntitiesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalculatedFieldState; @@ -91,7 +91,7 @@ public class CalculatedFieldArgumentUtils { case GEOFENCING -> new GeofencingCalculatedFieldState(entityId); case ALARM -> new AlarmCalculatedFieldState(entityId); case PROPAGATION -> new PropagationCalculatedFieldState(entityId); - case RELATED_ENTITIES_AGGREGATION -> new RelaredEntitiesAggregationCalculatedFieldState(entityId); + case RELATED_ENTITIES_AGGREGATION -> new RelatedEntitiesAggregationCalculatedFieldState(entityId); }; } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 67af13fdb3..389fad8ff3 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -47,9 +47,9 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.RelaredEntitiesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; @@ -104,9 +104,9 @@ public class CalculatedFieldUtils { case SINGLE_VALUE -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry)); case TS_ROLLING -> builder.addRollingValueArguments(toRollingArgumentProto(argName, (TsRollingArgumentEntry) argEntry)); case GEOFENCING -> builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry)); - case AGGREGATE_LATEST -> { - AggArgumentEntry aggArgumentEntry = (AggArgumentEntry) argEntry; - aggArgumentEntry.getAggInputs() + case RELATED_ENTITIES -> { + RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry; + relatedEntitiesArgumentEntry.getAggInputs() .forEach((entityId, entry) -> aggBuilder.addAggArguments(toAggSingleArgumentProto(argName, entityId, entry))); } } @@ -120,7 +120,7 @@ public class CalculatedFieldUtils { alarmStateProto.setClearRuleState(toAlarmRuleStateProto(alarmState.getClearRuleState())); } } - if (state instanceof RelaredEntitiesAggregationCalculatedFieldState aggState) { + if (state instanceof RelatedEntitiesAggregationCalculatedFieldState aggState) { aggBuilder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); builder.setLatestValuesAggregationState(aggBuilder.build()); } @@ -214,7 +214,7 @@ public class CalculatedFieldUtils { case GEOFENCING -> new GeofencingCalculatedFieldState(id.entityId()); case ALARM -> new AlarmCalculatedFieldState(id.entityId()); case PROPAGATION -> new PropagationCalculatedFieldState(id.entityId()); - case RELATED_ENTITIES_AGGREGATION -> new RelaredEntitiesAggregationCalculatedFieldState(id.entityId()); + case RELATED_ENTITIES_AGGREGATION -> new RelatedEntitiesAggregationCalculatedFieldState(id.entityId()); }; proto.getSingleValueArgumentsList().forEach(argProto -> @@ -241,7 +241,7 @@ public class CalculatedFieldUtils { } } case RELATED_ENTITIES_AGGREGATION -> { - RelaredEntitiesAggregationCalculatedFieldState aggState = (RelaredEntitiesAggregationCalculatedFieldState) state; + RelatedEntitiesAggregationCalculatedFieldState aggState = (RelatedEntitiesAggregationCalculatedFieldState) state; LatestValuesAggregationStateProto aggregationStateProto = proto.getLatestValuesAggregationState(); Map> arguments = new HashMap<>(); aggregationStateProto.getAggArgumentsList().forEach(argProto -> { @@ -249,7 +249,7 @@ public class CalculatedFieldUtils { arguments.computeIfAbsent(argProto.getValue().getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); }); arguments.forEach((argName, entityInputs) -> { - aggState.getArguments().put(argName, new AggArgumentEntry(entityInputs, false)); + aggState.getArguments().put(argName, new RelatedEntitiesArgumentEntry(entityInputs, false)); }); aggState.setLastArgsRefreshTs(aggregationStateProto.getLastArgsUpdateTs()); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java similarity index 80% rename from application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java rename to application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java index c1c2d85c55..14d2801e0e 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java @@ -21,7 +21,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import java.util.HashMap; @@ -31,9 +31,9 @@ import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -public class AggArgumentEntryTest { +public class RelatedEntitiesArgumentEntryTest { - private AggArgumentEntry entry; + private RelatedEntitiesArgumentEntry entry; private final DeviceId device1 = new DeviceId(UUID.fromString("1984e5f4-9ff0-4187-84ae-e4438bba4c8a")); private final DeviceId device2 = new DeviceId(UUID.fromString("937fc062-1a9d-438f-aa22-55a93fc908b7")); @@ -46,7 +46,7 @@ public class AggArgumentEntryTest { aggInputs.put(device1, new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 12L), 1L))); aggInputs.put(device2, new AggSingleEntityArgumentEntry(device2, new BasicTsKvEntry(ts - 150, new LongDataEntry("key", 16L), 6L))); - entry = new AggArgumentEntry(aggInputs, false); + entry = new RelatedEntitiesArgumentEntry(aggInputs, false); } @Test @@ -61,17 +61,17 @@ public class AggArgumentEntryTest { DeviceId device3 = new DeviceId(UUID.randomUUID()); DeviceId device4 = new DeviceId(UUID.randomUUID()); - AggArgumentEntry aggArgumentEntry = new AggArgumentEntry(Map.of( + RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = new RelatedEntitiesArgumentEntry(Map.of( device3, new AggSingleEntityArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 16L), 13L)), device4, new AggSingleEntityArgumentEntry(device4, new BasicTsKvEntry(ts - 60, new LongDataEntry("key", 23L), 7L)) ), false); - assertThat(entry.updateEntry(aggArgumentEntry)).isTrue(); + assertThat(entry.updateEntry(relatedEntitiesArgumentEntry)).isTrue(); Map aggInputs = entry.getAggInputs(); assertThat(aggInputs.size()).isEqualTo(4); - assertThat(aggInputs.get(device3)).isEqualTo(aggArgumentEntry.getAggInputs().get(device3)); - assertThat(aggInputs.get(device4)).isEqualTo(aggArgumentEntry.getAggInputs().get(device4)); + assertThat(aggInputs.get(device3)).isEqualTo(relatedEntitiesArgumentEntry.getAggInputs().get(device3)); + assertThat(aggInputs.get(device4)).isEqualTo(relatedEntitiesArgumentEntry.getAggInputs().get(device4)); } @Test @@ -98,15 +98,4 @@ public class AggArgumentEntryTest { assertThat(aggInputs.get(device2)).isEqualTo(singleEntityArgumentEntry); } - @Test - void testUpdateEntryWhenDeletedAggSingleEntityArgumentEntryPassed() { - AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device2, true); - - assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); - - Map aggInputs = entry.getAggInputs(); - assertThat(aggInputs.size()).isEqualTo(1); - assertThat(aggInputs.get(device2)).isNull(); - } - } From 77f2250e0af94d361837dba8467142a5a354d1d8 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 21 Oct 2025 14:09:33 +0300 Subject: [PATCH 59/88] added helper method to collectionsUtil --- .../service/cf/ctx/state/CalculatedFieldCtx.java | 6 +++--- .../server/common/data/util/CollectionsUtil.java | 13 +++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 9cb68bbc27..9ef5a8c2a9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -42,7 +43,6 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -90,10 +90,10 @@ public class CalculatedFieldCtx { var refId = entry.getValue().getRefEntityId(); var refKey = entry.getValue().getRefEntityKey(); if (refId == null || refId.equals(calculatedField.getEntityId())) { - mainEntityArguments.computeIfAbsent(refKey, key -> new HashSet<>()).add(entry.getKey()); + mainEntityArguments.compute(refKey, (key, existingNames) -> CollectionsUtil.addToSet(existingNames, entry.getKey())); } else { linkedEntityArguments.computeIfAbsent(refId, key -> new HashMap<>()) - .computeIfAbsent(refKey, key -> new HashSet<>()).add(entry.getKey()); + .compute(refKey, (key, existingNames) -> CollectionsUtil.addToSet(existingNames, entry.getKey())); } } this.argNames = new ArrayList<>(arguments.keySet()); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java index 71c5256203..082be9b71f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java @@ -95,4 +95,17 @@ public class CollectionsUtil { return false; } + public static Set addToSet(Set existing, T value) { + if (existing == null || existing.isEmpty()) { + return Set.of(value); + } + if (existing.contains(value)) { + return existing; + } + Set newSet = new HashSet<>(existing.size() + 1); + newSet.addAll(existing); + newSet.add(value); + return (Set) Set.of(newSet.toArray()); + } + } From 7bf7e0f994e4d69c5ca2f1f8e7464021c9310f48 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 21 Oct 2025 15:33:02 +0300 Subject: [PATCH 60/88] removed unnecessary fetch arguments methods --- ...tractCalculatedFieldProcessingService.java | 77 ++++++------------- ...faultCalculatedFieldProcessingService.java | 3 - .../service/cf/ctx/state/ArgumentEntry.java | 8 +- .../utils/CalculatedFieldArgumentUtils.java | 9 --- 4 files changed, 27 insertions(+), 70 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index 0c0f2f23e9..cfd7c9af3e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -64,7 +64,6 @@ import static org.thingsboard.server.common.data.cf.configuration.geofencing.Ent import static org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates.ENTITY_ID_LONGITUDE_ARGUMENT_KEY; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultAttributeEntry; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.createDefaultKvEntry; -import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformAggSingleArgument; import static org.thingsboard.server.utils.CalculatedFieldArgumentUtils.transformSingleValueArgument; @Data @@ -98,7 +97,7 @@ public abstract class AbstractCalculatedFieldProcessingService { Map> argFutures = switch (ctx.getCfType()) { case GEOFENCING -> fetchGeofencingCalculatedFieldArguments(ctx, entityId, false, ts); case SIMPLE, SCRIPT, ALARM, PROPAGATION -> getBaseCalculatedFieldArguments(ctx, entityId, ts); - case RELATED_ENTITIES_AGGREGATION -> fetchAggArguments(ctx, entityId, ts); + case RELATED_ENTITIES_AGGREGATION -> fetchRelatedEntitiesAggArguments(ctx, entityId, ts); }; if (ctx.getCfType() == PROPAGATION) { argFutures.put(PROPAGATION_CONFIG_ARGUMENT, fetchPropagationCalculatedFieldArgument(ctx, entityId)); @@ -128,23 +127,6 @@ public abstract class AbstractCalculatedFieldProcessingService { return resolveOwnerArgument(tenantId, entityId); } - private ListenableFuture> resolveRelatedEntities(TenantId tenantId, EntityId entityId, RelationPathLevel relation) { - ListenableFuture> relationsFut = relationService.findByRelationPathQueryAsync(tenantId, new EntityRelationPathQuery(entityId, List.of(relation))); - - return Futures.transform(relationsFut, relations -> { - if (relations == null) { - return new ArrayList<>(); - } - - return switch (relation.direction()) { - case FROM -> relations.stream() - .map(EntityRelation::getTo) - .toList(); - case TO -> relations.isEmpty() ? List.of() : List.of(relations.get(0).getFrom()); - }; - }, calculatedFieldCallbackExecutor); - } - protected Map resolveArgumentFutures(Map> argFutures) { return argFutures.entrySet().stream() .collect(Collectors.toMap( @@ -189,7 +171,7 @@ public abstract class AbstractCalculatedFieldProcessingService { return argFutures; } - protected Map> fetchAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { + protected Map> fetchRelatedEntitiesAggArguments(CalculatedFieldCtx ctx, EntityId entityId, long ts) { RelatedEntitiesAggregationCalculatedFieldConfiguration aggConfig = (RelatedEntitiesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); ListenableFuture> relatedEntitiesFut = resolveRelatedEntities(ctx.getTenantId(), entityId, aggConfig.getRelation()); @@ -197,10 +179,27 @@ public abstract class AbstractCalculatedFieldProcessingService { return aggConfig.getArguments().entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, - entry -> Futures.transformAsync(relatedEntitiesFut, relatedEntities -> fetchAggArgumentEntry(ctx.getTenantId(), relatedEntities, entry.getValue(), ts), MoreExecutors.directExecutor()) + entry -> Futures.transformAsync(relatedEntitiesFut, relatedEntities -> fetchRelatedEntitiesArgumentEntry(ctx.getTenantId(), relatedEntities, entry.getValue(), ts), MoreExecutors.directExecutor()) )); } + private ListenableFuture> resolveRelatedEntities(TenantId tenantId, EntityId entityId, RelationPathLevel relation) { + ListenableFuture> relationsFut = relationService.findByRelationPathQueryAsync(tenantId, new EntityRelationPathQuery(entityId, List.of(relation))); + + return Futures.transform(relationsFut, relations -> { + if (relations == null) { + return new ArrayList<>(); + } + + return switch (relation.direction()) { + case FROM -> relations.stream() + .map(EntityRelation::getTo) + .toList(); + case TO -> relations.isEmpty() ? List.of() : List.of(relations.get(0).getFrom()); + }; + }, calculatedFieldCallbackExecutor); + } + private ListenableFuture> resolveGeofencingEntityIds(TenantId tenantId, EntityId entityId, Map.Entry entry) { Argument value = entry.getValue(); if (value.getRefEntityId() != null) { @@ -252,11 +251,11 @@ public abstract class AbstractCalculatedFieldProcessingService { .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))), MoreExecutors.directExecutor()); } - public ListenableFuture fetchAggArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) { + public ListenableFuture fetchRelatedEntitiesArgumentEntry(TenantId tenantId, List aggEntities, Argument argument, long startTs) { List>> futures = aggEntities.stream() .map(entityId -> { - ListenableFuture singleAggEntryFut = fetchSingleAggArgumentEntry(tenantId, entityId, argument, startTs); - return Futures.transform(singleAggEntryFut, singleAggEntry -> Map.entry(entityId, singleAggEntry), MoreExecutors.directExecutor()); + ListenableFuture argumentEntryFut = fetchArgumentValue(tenantId, entityId, argument, startTs); + return Futures.transform(argumentEntryFut, argumentEntry -> Map.entry(entityId, ArgumentEntry.createAggSingleArgument(entityId, argumentEntry)), MoreExecutors.directExecutor()); }) .toList(); @@ -321,34 +320,4 @@ public abstract class AbstractCalculatedFieldProcessingService { return new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, endTs, 0, limit, Aggregation.NONE); } - private ListenableFuture fetchSingleAggArgumentEntry(TenantId tenantId, EntityId entityId, Argument argument, long startTs) { - return switch (argument.getRefEntityKey().getType()) { - case TS_ROLLING -> throw new IllegalStateException("TS_ROLLING is not supported for aggregation"); - case ATTRIBUTE -> fetchAttributeAggEntry(tenantId, entityId, argument, startTs); - case TS_LATEST -> fetchTsLatestAggEntry(tenantId, entityId, argument, startTs); - }; - } - - private ListenableFuture fetchAttributeAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultLastUpdateTs) { - log.trace("[{}][{}] Fetching attribute for key {}", tenantId, entityId, argument.getRefEntityKey()); - var attributeOptFuture = attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()); - return Futures.transform(attributeOptFuture, attrOpt -> { - log.debug("[{}][{}] Fetched attribute for key {}: {}", tenantId, entityId, argument.getRefEntityKey(), attrOpt); - AttributeKvEntry attributeKvEntry = attrOpt.orElseGet(() -> new BaseAttributeKvEntry(createDefaultKvEntry(argument), defaultLastUpdateTs, SingleValueArgumentEntry.DEFAULT_VERSION)); - return transformAggSingleArgument(entityId, Optional.of(attributeKvEntry)); - }, calculatedFieldCallbackExecutor); - } - - private ListenableFuture fetchTsLatestAggEntry(TenantId tenantId, EntityId entityId, Argument argument, long defaultTs) { - String key = argument.getRefEntityKey().getKey(); - log.trace("[{}][{}] Fetching latest timeseries {}", tenantId, entityId, key); - return Futures.transform( - timeseriesService.findLatest(tenantId, entityId, key), - result -> { - log.debug("[{}][{}] Fetched latest timeseries {}: {}", tenantId, entityId, key, result); - Optional tsKvEntry = result.or(() -> Optional.of(new BasicTsKvEntry(defaultTs, createDefaultKvEntry(argument), SingleValueArgumentEntry.DEFAULT_VERSION))); - return transformAggSingleArgument(entityId, tsKvEntry); - }, calculatedFieldCallbackExecutor); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index d7957dce9b..52393d0ffe 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.service.cf; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; @@ -56,7 +54,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.stream.Collectors; import static org.thingsboard.server.common.data.cf.configuration.PropagationCalculatedFieldConfiguration.PROPAGATION_CONFIG_ARGUMENT; import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 5bb16292be..863d6f5b50 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -22,8 +22,8 @@ import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry; @@ -40,7 +40,7 @@ import java.util.Map; @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING"), @JsonSubTypes.Type(value = PropagationArgumentEntry.class, name = "PROPAGATION"), - @JsonSubTypes.Type(value = RelatedEntitiesArgumentEntry.class, name = "AGGREGATE_LATEST"), + @JsonSubTypes.Type(value = RelatedEntitiesArgumentEntry.class, name = "RELATED_ENTITIES"), @JsonSubTypes.Type(value = AggSingleEntityArgumentEntry.class, name = "AGGREGATE_LATEST_SINGLE") }) public interface ArgumentEntry { @@ -80,8 +80,8 @@ public interface ArgumentEntry { return new RelatedEntitiesArgumentEntry(entityIdkvEntryMap, false); } - static ArgumentEntry createAggSingleArgument(EntityId entityId, KvEntry kvEntry) { - return new AggSingleEntityArgumentEntry(entityId, kvEntry); + static ArgumentEntry createAggSingleArgument(EntityId entityId, ArgumentEntry argumentEntry) { + return new AggSingleEntityArgumentEntry(entityId, argumentEntry); } } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java index 239ffedbc7..0c0d401688 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldArgumentUtils.java @@ -34,7 +34,6 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -56,14 +55,6 @@ public class CalculatedFieldArgumentUtils { } } - public static ArgumentEntry transformAggSingleArgument(EntityId entityId, Optional kvEntry) { - if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { - return ArgumentEntry.createAggSingleArgument(entityId, kvEntry.get()); - } else { - return new AggSingleEntityArgumentEntry(); - } - } - public static KvEntry createDefaultKvEntry(Argument argument) { String key = argument.getRefEntityKey().getKey(); String defaultValue = argument.getDefaultValue(); From 44dcbb4359a5b857351e6b0edece410b0d889aaf Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Tue, 21 Oct 2025 16:20:36 +0300 Subject: [PATCH 61/88] lwm2m: bootstrap new: add toast --- ui-ngx/src/app/core/http/device.service.ts | 94 +++++++++---------- .../device-credentials-lwm2m.component.ts | 27 +++++- 2 files changed, 72 insertions(+), 49 deletions(-) diff --git a/ui-ngx/src/app/core/http/device.service.ts b/ui-ngx/src/app/core/http/device.service.ts index 88e5f8f7bf..251870ab64 100644 --- a/ui-ngx/src/app/core/http/device.service.ts +++ b/ui-ngx/src/app/core/http/device.service.ts @@ -35,7 +35,7 @@ import { AuthService } from '@core/auth/auth.service'; import { BulkImportRequest, BulkImportResult } from '@shared/import-export/import-export.models'; import { PersistentRpc, RpcStatus } from '@shared/models/rpc.models'; import { ResourcesService } from '@core/services/resources.service'; -import {map} from "rxjs/operators"; +import {map, switchMap} from "rxjs/operators"; @Injectable({ providedIn: 'root' @@ -221,53 +221,51 @@ export class DeviceService { return this.resourcesService.downloadResource(`/api/device-connectivity/gateway-launch/${deviceId}/docker-compose/download`); } - public rebootDevice(deviceId: string = '', isBootstrapServer: boolean): void { + public rebootDevice(deviceId: string = '', isBootstrapServer: boolean): Observable<{ result: string, msg: string }> { const urlApi = `/api/plugins/rpc/twoway/${deviceId}`; - // DiscoveryAll} - this.http.post(urlApi, { method: "DiscoverAll" }) - .pipe( - timeout(10000), // 10 sec - catchError(err => { - console.error('DiscoverAll timeout or error', err); - return throwError(() => err); - }) - ) - .subscribe({ - next: (response: any) => { - console.log('success: Discovery'); - console.log(response); - // result = 'CONTENT' - if (response.result && response.result.toUpperCase() === 'CONTENT') { - const resourceId = isBootstrapServer ? 9 : 8; - const rebutName = isBootstrapServer ? "Bootstrap-Request Trigger" : - "Registration Update Trigger"; - const resourcePath = `/1/0/${resourceId}`; - - // first rebootTrigger - this.rebootTrigger(resourcePath, urlApi).subscribe(responseReboot => { + const rebootName = isBootstrapServer ? 'Bootstrap-Request Trigger' : 'Registration Update Trigger'; + return this.http.post(urlApi, { method: 'DiscoverAll' }).pipe( + timeout(10000), + switchMap((response: any) => { + if (response.result && response.result.toUpperCase() === 'CONTENT') { + const resourceId = isBootstrapServer ? 9 : 8; + const resourcePath = `/1/0/${resourceId}`; + return this.rebootTrigger(resourcePath, urlApi).pipe( + map((responseReboot: any) => { if (responseReboot.result === 'CHANGED') { - console.info(`info: ${rebutName} success.`); + return { + result: 'SUCCESS', + msg: `\"${rebootName}\" - Started Successfully.` }; } else { - console.error(`error: ${rebutName} failed: ${responseReboot.toString()}`); + return { + result: 'ERROR', + msg: `\"${rebootName}\" failed:
${JSON.stringify(responseReboot, null, 2)}
` + } } - }); - } - else { - console.error(`error3: Bad registration device with id = ${deviceId} ❗ RPC result is not CONTENT`); - } - }, - error: (e) => { - console.error(`error4: Bad registration device with id = ${deviceId} ${e.toString()}`); - return throwError(() => new Error('Could not get JWT token from store.')); - // return throwError(() => e); - }, - complete: () => { - console.log('Discovery stream complete'); } - }); - } - - private rebootTrigger(resourcePath: string, urlApi: string): Observable<{ result: string;}> { - console.log(`Sending reboot command to ${resourcePath}`); + }), + catchError(err => + of({ + result: 'ERROR', + msg: `\"${rebootName}\" failed.
Error: ${err.message || err}` }) + ) + ); + } else { + return of({ + result: 'ERROR', + msg: `\"${rebootName}\" failed.
Bad registration device with id = ${deviceId}.
\"DiscoverAll\" - RPC result is not \"CONTENT\"` + }); + } + }), + catchError(err => + of({ + result: 'ERROR', + msg: `\"${rebootName}\" failed.
Bad registration device with id = ${deviceId}.
Error: ${err.message || err}` + }) + ) + ); + } + + private rebootTrigger(resourcePath: string, urlApi: string): Observable<{ result: string, msg?: string }> { return this.http.post(urlApi, { method: 'Execute', params: { id: resourcePath } @@ -278,12 +276,14 @@ export class DeviceService { if (res?.result?.toUpperCase() === 'CHANGED') { return { result: 'CHANGED' }; } else { - return {result: 'ERROR'} + return { + result:`${res?.result}`, + msg: `${res?.error} ` + } }; }), catchError(err => { - console.error(`Execute error5 for ${resourcePath}:`, err); - return of({ result: 'ERROR' }); + return throwError(() => err); }) ); } diff --git a/ui-ngx/src/app/modules/home/components/device/device-credentials-lwm2m.component.ts b/ui-ngx/src/app/modules/home/components/device/device-credentials-lwm2m.component.ts index 4ddc3a4852..7ace8c349a 100644 --- a/ui-ngx/src/app/modules/home/components/device/device-credentials-lwm2m.component.ts +++ b/ui-ngx/src/app/modules/home/components/device/device-credentials-lwm2m.component.ts @@ -37,6 +37,9 @@ import {takeUntil} from 'rxjs/operators'; import { isDefinedAndNotNull } from '@core/utils'; import {DeviceId} from "@shared/models/id/device-id"; import {DeviceService} from "@core/http/device.service"; +import {ActionNotificationShow} from "@core/notification/notification.actions"; +import {Store} from "@ngrx/store"; +import {AppState} from "@core/core.state"; @Component({ selector: 'tb-device-credentials-lwm2m', @@ -70,7 +73,8 @@ export class DeviceCredentialsLwm2mComponent implements ControlValueAccessor, Va @Input() deviceId: DeviceId; - constructor(private fb: UntypedFormBuilder, + constructor(protected store: Store, + private fb: UntypedFormBuilder, private deviceService: DeviceService) { this.lwm2mConfigFormGroup = this.initLwm2mConfigForm(); } @@ -120,7 +124,26 @@ export class DeviceCredentialsLwm2mComponent implements ControlValueAccessor, Va */ public rebootDevice(isBootstrapServer: boolean): void { - this.deviceService.rebootDevice(this.deviceId.id, isBootstrapServer); + this.deviceService.rebootDevice(this.deviceId.id, isBootstrapServer).subscribe(responseReboot => { + if (responseReboot.result === 'SUCCESS') { + this.store.dispatch(new ActionNotificationShow( + { + message: responseReboot.msg, + type: 'success', + duration: 1500, + verticalPosition: 'top', + horizontalPosition: 'left' + })); + } else { + this.store.dispatch(new ActionNotificationShow( + { + message: responseReboot.msg, + type: 'error', + verticalPosition: 'top', + horizontalPosition: 'left' + })); + } + }); } private initClientSecurityConfig(config: Lwm2mSecurityConfigModels): void { From 899dd9002bb045042ca439566eefa4efe55bd4d3 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 21 Oct 2025 17:00:55 +0300 Subject: [PATCH 62/88] removed single agg argument and updated old single value argument --- ...CalculatedFieldEntityMessageProcessor.java | 15 ++- ...tractCalculatedFieldProcessingService.java | 2 +- .../service/cf/ctx/state/ArgumentEntry.java | 12 +-- .../cf/ctx/state/ArgumentEntryType.java | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 15 +-- .../cf/ctx/state/CalculatedFieldCtx.java | 2 +- .../cf/ctx/state/CalculatedFieldState.java | 1 - .../ctx/state/SimpleCalculatedFieldState.java | 2 +- .../ctx/state/SingleValueArgumentEntry.java | 30 ++++++ .../AggSingleEntityArgumentEntry.java | 97 ----------------- ...titiesAggregationCalculatedFieldState.java | 16 +-- .../RelatedEntitiesArgumentEntry.java | 13 +-- .../state/aggregation/function/AggEntry.java | 15 ++- .../function/AggFunctionFactory.java | 33 ------ .../aggregation/function/AvgAggEntry.java | 6 +- .../aggregation/function/BaseAggEntry.java | 9 +- .../aggregation/function/CountAggEntry.java | 2 +- .../function/CountUniqueAggEntry.java | 2 +- .../aggregation/function/MaxAggEntry.java | 5 +- .../aggregation/function/MinAggEntry.java | 5 +- .../aggregation/function/SumAggEntry.java | 5 +- .../server/utils/CalculatedFieldUtils.java | 61 ++++------- .../AggSingleEntityArgumentEntryTest.java | 101 ------------------ .../RelatedEntitiesArgumentEntryTest.java | 17 ++- .../configuration/aggregation/AggInput.java | 2 + ...gregationCalculatedFieldConfiguration.java | 12 +-- common/proto/src/main/proto/queue.proto | 14 +-- .../thingsboard/script/api/tbel/TbUtils.java | 11 ++ .../script/api/tbel/TbelCfArg.java | 2 +- ... => TbelCfRelatedEntitiesAggregation.java} | 4 +- 30 files changed, 148 insertions(+), 365 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java delete mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java rename common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/{TbelCfLatestValuesAggregation.java => TbelCfRelatedEntitiesAggregation.java} (90%) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 434f555325..ca97dae51b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -52,7 +52,6 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; @@ -232,7 +231,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } else { if (state instanceof RelatedEntitiesAggregationCalculatedFieldState relatedEntitiesAggState) { Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, msg.getRelatedEntityId(), ctx.getArguments()); - updatedArgs = relatedEntitiesAggState.updateEntityData(toAggSingleEntityArguments(msg.getRelatedEntityId(), fetchedArgs)); + updatedArgs = relatedEntitiesAggState.updateEntityData(setEntityIdToSingleEntityArguments(msg.getRelatedEntityId(), fetchedArgs)); } state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); @@ -544,7 +543,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); String argName = relatedEntityArgs.get(key); if (argName != null) { - arguments.put(argName, new AggSingleEntityArgumentEntry(originator, item)); + arguments.put(argName, new SingleValueArgumentEntry(originator, item)); } } } @@ -597,7 +596,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); String argName = relatedEntityArgs.get(key); if (argName != null) { - arguments.put(argName, new AggSingleEntityArgumentEntry(entityId, item)); + arguments.put(argName, new SingleValueArgumentEntry(entityId, item)); } } } @@ -636,7 +635,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM SingleValueArgumentEntry argumentEntry = StringUtils.isNotEmpty(defaultValue) ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) : new SingleValueArgumentEntry(); - arguments.put(argName, new AggSingleEntityArgumentEntry(msgEntityId, argumentEntry)); + arguments.put(argName, new SingleValueArgumentEntry(msgEntityId, argumentEntry)); } } } @@ -668,18 +667,18 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, deletedArguments); if (CalculatedFieldType.RELATED_ENTITIES_AGGREGATION.equals(ctx.getCfType())) { - fetchedArgs = toAggSingleEntityArguments(entityId, fetchedArgs); + fetchedArgs = setEntityIdToSingleEntityArguments(entityId, fetchedArgs); } fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); return fetchedArgs; } - private Map toAggSingleEntityArguments(EntityId relatedEntityId, Map fetchedArgs) { + private Map setEntityIdToSingleEntityArguments(EntityId relatedEntityId, Map fetchedArgs) { return fetchedArgs.entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, - argEntry -> new AggSingleEntityArgumentEntry(relatedEntityId, argEntry.getValue()) + argEntry -> new SingleValueArgumentEntry(relatedEntityId, argEntry.getValue()) )); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java index cfd7c9af3e..d4bf0d52be 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldProcessingService.java @@ -255,7 +255,7 @@ public abstract class AbstractCalculatedFieldProcessingService { List>> futures = aggEntities.stream() .map(entityId -> { ListenableFuture argumentEntryFut = fetchArgumentValue(tenantId, entityId, argument, startTs); - return Futures.transform(argumentEntryFut, argumentEntry -> Map.entry(entityId, ArgumentEntry.createAggSingleArgument(entityId, argumentEntry)), MoreExecutors.directExecutor()); + return Futures.transform(argumentEntryFut, argumentEntry -> Map.entry(entityId, ArgumentEntry.createSingleValueArgument(entityId, argumentEntry)), MoreExecutors.directExecutor()); }) .toList(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 863d6f5b50..b331c11a47 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -22,7 +22,6 @@ import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationArgumentEntry; @@ -40,8 +39,7 @@ import java.util.Map; @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = GeofencingArgumentEntry.class, name = "GEOFENCING"), @JsonSubTypes.Type(value = PropagationArgumentEntry.class, name = "PROPAGATION"), - @JsonSubTypes.Type(value = RelatedEntitiesArgumentEntry.class, name = "RELATED_ENTITIES"), - @JsonSubTypes.Type(value = AggSingleEntityArgumentEntry.class, name = "AGGREGATE_LATEST_SINGLE") + @JsonSubTypes.Type(value = RelatedEntitiesArgumentEntry.class, name = "RELATED_ENTITIES") }) public interface ArgumentEntry { @@ -64,6 +62,10 @@ public interface ArgumentEntry { return new SingleValueArgumentEntry(kvEntry); } + static ArgumentEntry createSingleValueArgument(EntityId entityId, ArgumentEntry argumentEntry) { + return new SingleValueArgumentEntry(entityId, argumentEntry); + } + static ArgumentEntry createTsRollingArgument(List kvEntries, int limit, long timeWindow) { return new TsRollingArgumentEntry(kvEntries, limit, timeWindow); } @@ -80,8 +82,4 @@ public interface ArgumentEntry { return new RelatedEntitiesArgumentEntry(entityIdkvEntryMap, false); } - static ArgumentEntry createAggSingleArgument(EntityId entityId, ArgumentEntry argumentEntry) { - return new AggSingleEntityArgumentEntry(entityId, argumentEntry); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java index 5c672cf04e..427df2bf5b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java @@ -16,5 +16,5 @@ package org.thingsboard.server.service.cf.ctx.state; public enum ArgumentEntryType { - SINGLE_VALUE, TS_ROLLING, GEOFENCING, PROPAGATION, RELATED_ENTITIES, AGGREGATE_LATEST_SINGLE + SINGLE_VALUE, TS_ROLLING, GEOFENCING, PROPAGATION, RELATED_ENTITIES } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 91c331f88e..d48ed9c268 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -18,13 +18,12 @@ package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Getter; import lombok.Setter; -import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.actors.TbActorRef; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.utils.CalculatedFieldUtils; import java.io.Closeable; @@ -76,7 +75,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, ArgumentEntry existingEntry = arguments.get(key); boolean entryUpdated; - if (existingEntry == null || !(newEntry instanceof AggSingleEntityArgumentEntry) && newEntry.isForceResetPrevious()) { + if (existingEntry == null || !ctx.getCfType().equals(CalculatedFieldType.RELATED_ENTITIES_AGGREGATION) && newEntry.isForceResetPrevious()) { validateNewEntry(key, newEntry); arguments.put(key, newEntry); entryUpdated = true; @@ -152,14 +151,4 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, this.latestTimestamp = Math.max(this.latestTimestamp, newTs); } - protected Object formatResult(double result, Integer decimals) { - if (decimals == null) { - return result; - } - if (decimals.equals(0)) { - return TbUtils.toInt(result); - } - return TbUtils.toFixed(result, decimals); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 46442847a0..aab858d85d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -634,7 +634,7 @@ public class CalculatedFieldCtx { private boolean hasRelatedEntitiesAggregationConfigurationChanges(CalculatedFieldCtx other) { if (calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration thisConfig && other.calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration otherConfig) { - return !thisConfig.getArguments().equals(otherConfig.getArguments()) || !thisConfig.getRelation().equals(otherConfig.getRelation()); + return !thisConfig.getRelation().equals(otherConfig.getRelation()); } return false; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 8e202308d0..2b3ba19528 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -33,7 +33,6 @@ import org.thingsboard.server.service.cf.ctx.state.propagation.PropagationCalcul import java.io.Closeable; import java.util.Map; -import java.util.concurrent.ExecutionException; import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArgumentProto; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index a268577d4b..ab0ed26dfe 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -52,7 +52,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { double expressionResult = ctx.evaluateSimpleExpression(expression.get(), this); Output output = ctx.getOutput(); - Object result = formatResult(expressionResult, output.getDecimalsByDefault()); + Object result = TbUtils.roundResult(expressionResult, output.getDecimalsByDefault()); JsonNode outputResult = createResultJson(ctx.isUseLatestTs(), output.getName(), result); return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index e81201c961..67fb385917 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -20,9 +20,11 @@ import com.fasterxml.jackson.core.type.TypeReference; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import org.springframework.lang.Nullable; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; @@ -37,6 +39,9 @@ import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; @AllArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { + @Nullable + protected EntityId entityId; + protected long ts; protected BasicKvEntry kvEntryValue; protected Long version; @@ -45,6 +50,11 @@ public class SingleValueArgumentEntry implements ArgumentEntry { public static final Long DEFAULT_VERSION = -1L; + public SingleValueArgumentEntry(EntityId entityId, ArgumentEntry entry) { + this(entry); + this.entityId = entityId; + } + public SingleValueArgumentEntry(ArgumentEntry entry) { if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { this.ts = singleValueArgumentEntry.ts; @@ -54,6 +64,11 @@ public class SingleValueArgumentEntry implements ArgumentEntry { } } + public SingleValueArgumentEntry(EntityId entityId, TsKvProto entry) { + this(entry); + this.entityId = entityId; + } + public SingleValueArgumentEntry(TsKvProto entry) { this.ts = entry.getTs(); if (entry.hasVersion()) { @@ -62,6 +77,11 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.kvEntryValue = ProtoUtils.fromProto(entry.getKv()); } + public SingleValueArgumentEntry(EntityId entityId, AttributeValueProto entry) { + this(entry); + this.entityId = entityId; + } + public SingleValueArgumentEntry(AttributeValueProto entry) { this.ts = entry.getLastUpdateTs(); if (entry.hasVersion()) { @@ -70,6 +90,11 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.kvEntryValue = ProtoUtils.basicKvEntryFromProto(entry); } + public SingleValueArgumentEntry(EntityId entityId, KvEntry entry) { + this(entry); + this.entityId = entityId; + } + public SingleValueArgumentEntry(KvEntry entry) { if (entry instanceof TsKvEntry tsKvEntry) { this.ts = tsKvEntry.getTs(); @@ -81,6 +106,11 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.kvEntryValue = ProtoUtils.basicKvEntryFromKvEntry(entry); } + public SingleValueArgumentEntry(EntityId entityId, long ts, BasicKvEntry kvEntryValue, Long version) { + this(ts, kvEntryValue, version); + this.entityId = entityId; + } + public SingleValueArgumentEntry(long ts, BasicKvEntry kvEntryValue, Long version) { this.ts = ts; this.kvEntryValue = kvEntryValue; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java deleted file mode 100644 index b935256860..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/AggSingleEntityArgumentEntry.java +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf.ctx.state.aggregation; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.kv.BasicKvEntry; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; -import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; -import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; - -@Data -@NoArgsConstructor -@AllArgsConstructor -public class AggSingleEntityArgumentEntry extends SingleValueArgumentEntry { - - private EntityId entityId; - - public AggSingleEntityArgumentEntry(EntityId entityId, ArgumentEntry entry) { - super(entry); - this.entityId = entityId; - } - - public AggSingleEntityArgumentEntry(EntityId entityId, TsKvProto entry) { - super(entry); - this.entityId = entityId; - } - - public AggSingleEntityArgumentEntry(EntityId entityId, AttributeValueProto entry) { - super(entry); - this.entityId = entityId; - } - - public AggSingleEntityArgumentEntry(EntityId entityId, KvEntry entry) { - super(entry); - this.entityId = entityId; - } - - public AggSingleEntityArgumentEntry(EntityId entityId, long ts, BasicKvEntry kvEntryValue, Long version) { - super(ts, kvEntryValue, version); - this.entityId = entityId; - } - - @Override - public boolean updateEntry(ArgumentEntry entry) { - if (entry instanceof AggSingleEntityArgumentEntry aggSingleEntityEntry) { - if (aggSingleEntityEntry.isForceResetPrevious()) { - return applyNewEntry(aggSingleEntityEntry); - } - - if (aggSingleEntityEntry.getTs() < this.ts) { - if (!isDefaultValue()) { - return false; - } - } - - Long newVersion = aggSingleEntityEntry.getVersion(); - if (newVersion == null || this.version == null || newVersion > this.version) { - return applyNewEntry(aggSingleEntityEntry); - } - } else { - throw new IllegalArgumentException("Unsupported argument entry type for aggregation single entity argument entry: " + entry.getType()); - } - return false; - } - - private boolean applyNewEntry(AggSingleEntityArgumentEntry entry) { - this.ts = entry.getTs(); - this.version = entry.getVersion(); - this.kvEntryValue = entry.getKvEntryValue(); - this.entityId = entry.getEntityId(); - return true; - } - - @Override - public ArgumentEntryType getType() { - return ArgumentEntryType.AGGREGATE_LATEST_SINGLE; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java index c0baca836c..8e78824c7c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java @@ -37,7 +37,6 @@ import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.BaseCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.aggregation.function.AggEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.function.AggFunctionFactory; import java.util.HashMap; import java.util.Map; @@ -150,10 +149,10 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat String metricKey = entry.getKey(); AggMetric metric = entry.getValue(); - AggEntry aggMetricEntry = AggFunctionFactory.createAggFunction(metric.getFunction()); + AggEntry aggMetricEntry = AggEntry.createAggFunction(metric.getFunction()); aggregateMetric(metric, aggMetricEntry, inputs); - aggMetricEntry.result().ifPresent(result -> { - aggResult.set(metricKey, JacksonUtil.valueToTree(formatResult(result, output.getDecimalsByDefault()))); + aggMetricEntry.result(output.getDecimalsByDefault()).ifPresent(result -> { + aggResult.set(metricKey, JacksonUtil.valueToTree(result)); }); } return aggResult; @@ -188,13 +187,4 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat } } - private Object formatResult(Object aggregationResult, Integer decimals) { - try { - double result = Double.parseDouble(aggregationResult.toString()); - return formatResult(result, decimals); - } catch (Exception e) { - throw new IllegalArgumentException("Aggregation result cannot be parsed: " + aggregationResult, e); - } - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java index 5385808923..45b7755af4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java @@ -18,10 +18,11 @@ package org.thingsboard.server.service.cf.ctx.state.aggregation; import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.script.api.tbel.TbelCfArg; -import org.thingsboard.script.api.tbel.TbelCfLatestValuesAggregation; +import org.thingsboard.script.api.tbel.TbelCfRelatedEntitiesAggregation; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import java.util.Map; @@ -48,12 +49,12 @@ public class RelatedEntitiesArgumentEntry implements ArgumentEntry { if (entry instanceof RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry) { aggInputs.putAll(relatedEntitiesArgumentEntry.aggInputs); return true; - } else if (entry instanceof AggSingleEntityArgumentEntry aggSingleEntityArgumentEntry) { - ArgumentEntry argumentEntry = aggInputs.get(aggSingleEntityArgumentEntry.getEntityId()); + } else if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + ArgumentEntry argumentEntry = aggInputs.get(singleValueArgumentEntry.getEntityId()); if (argumentEntry != null) { - argumentEntry.updateEntry(aggSingleEntityArgumentEntry); + argumentEntry.updateEntry(singleValueArgumentEntry); } else { - aggInputs.put(aggSingleEntityArgumentEntry.getEntityId(), aggSingleEntityArgumentEntry); + aggInputs.put(singleValueArgumentEntry.getEntityId(), singleValueArgumentEntry); } return true; } else { @@ -68,7 +69,7 @@ public class RelatedEntitiesArgumentEntry implements ArgumentEntry { @Override public TbelCfArg toTbelCfArg() { - return new TbelCfLatestValuesAggregation(aggInputs.values()); + return new TbelCfRelatedEntitiesAggregation(aggInputs.values()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java index 4239e94fec..c4b93fd91d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation.function; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; @@ -36,10 +37,22 @@ import java.util.Optional; }) public interface AggEntry { + @JsonIgnore AggFunction getType(); void update(Object value); - Optional result(); + Optional result(Integer precision); + + static AggEntry createAggFunction(AggFunction function) { + return switch (function) { + case MIN -> new MinAggEntry(); + case MAX -> new MaxAggEntry(); + case SUM -> new SumAggEntry(); + case AVG -> new AvgAggEntry(); + case COUNT -> new CountAggEntry(); + case COUNT_UNIQUE -> new CountUniqueAggEntry(); + }; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java deleted file mode 100644 index 5ccc355b1f..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AggFunctionFactory.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf.ctx.state.aggregation.function; - -import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; - -public class AggFunctionFactory { - - public static AggEntry createAggFunction(AggFunction function) { - return switch (function) { - case MIN -> new MinAggEntry(); - case MAX -> new MaxAggEntry(); - case SUM -> new SumAggEntry(); - case AVG -> new AvgAggEntry(); - case COUNT -> new CountAggEntry(); - case COUNT_UNIQUE -> new CountUniqueAggEntry(); - }; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java index afe6abb93e..e063ff2ea2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/AvgAggEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation.function; +import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; import java.math.BigDecimal; @@ -34,8 +35,9 @@ public class AvgAggEntry extends BaseAggEntry { } @Override - protected double prepareResult() { - return sum.divide(BigDecimal.valueOf(count), 10, RoundingMode.HALF_UP).doubleValue(); + protected Object prepareResult(Integer precision) { + double result = sum.divide(BigDecimal.valueOf(count), RoundingMode.HALF_UP).doubleValue(); + return TbUtils.roundResult(result, precision); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java index 0c12bd13c0..b320435e99 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java @@ -28,10 +28,10 @@ public abstract class BaseAggEntry implements AggEntry { } @Override - public Optional result() { + public Optional result(Integer precision) { if (hasResult) { hasResult = false; - return Optional.of(prepareResult()); + return Optional.of(prepareResult(precision)); } else { return Optional.empty(); } @@ -39,10 +39,13 @@ public abstract class BaseAggEntry implements AggEntry { protected abstract void doUpdate(double value); - protected abstract double prepareResult(); + protected abstract Object prepareResult(Integer precision); protected double extractDoubleValue(Object value) { try { + if (value instanceof Number) { + return ((Number) value).doubleValue(); + } return Double.parseDouble(value.toString()); } catch (Exception e) { throw new NumberFormatException("Cannot parse value " + value.toString()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java index 469048d62d..09116985d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountAggEntry.java @@ -29,7 +29,7 @@ public class CountAggEntry implements AggEntry { } @Override - public Optional result() { + public Optional result(Integer precision) { return Optional.of(count); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java index b8b0b92470..efb4a58c90 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java @@ -34,7 +34,7 @@ public class CountUniqueAggEntry implements AggEntry { } @Override - public Optional result() { + public Optional result(Integer precision) { return Optional.of(items.size()); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java index 6e4235b72f..ddc47daf33 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation.function; +import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; public class MaxAggEntry extends BaseAggEntry { @@ -29,8 +30,8 @@ public class MaxAggEntry extends BaseAggEntry { } @Override - protected double prepareResult() { - return max; + protected Object prepareResult(Integer precision) { + return TbUtils.roundResult(max, precision); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java index eeacc3a7d9..e517ad305f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MinAggEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation.function; +import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; public class MinAggEntry extends BaseAggEntry { @@ -29,8 +30,8 @@ public class MinAggEntry extends BaseAggEntry { } @Override - protected double prepareResult() { - return min; + protected Object prepareResult(Integer precision) { + return TbUtils.roundResult(min, precision); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java index b90817d784..fe29d27b7e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/SumAggEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state.aggregation.function; +import org.thingsboard.script.api.tbel.TbUtils; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; import java.math.BigDecimal; @@ -31,8 +32,8 @@ public class SumAggEntry extends BaseAggEntry { } @Override - protected double prepareResult() { - return sum.doubleValue(); + protected Object prepareResult(Integer precision) { + return TbUtils.roundResult(sum.doubleValue(), precision); } @Override diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index 389fad8ff3..d75ec8a70a 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.util.KvProtoUtil; import org.thingsboard.server.common.util.ProtoUtils; -import org.thingsboard.server.gen.transport.TransportProtos.AggSingleArgumentEntryProto; import org.thingsboard.server.gen.transport.TransportProtos.AlarmRuleStateProto; import org.thingsboard.server.gen.transport.TransportProtos.AlarmStateProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; @@ -35,7 +34,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdPro import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingZoneProto; -import org.thingsboard.server.gen.transport.TransportProtos.LatestValuesAggregationStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.RelatedEntitiesAggregationStateProto; import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.TsDoubleValProto; import org.thingsboard.server.gen.transport.TransportProtos.TsRollingArgumentProto; @@ -47,9 +46,8 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmRuleState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; @@ -98,7 +96,7 @@ public class CalculatedFieldUtils { .setId(toProto(stateId)) .setType(state.getType().name()); - LatestValuesAggregationStateProto.Builder aggBuilder = LatestValuesAggregationStateProto.newBuilder(); + RelatedEntitiesAggregationStateProto.Builder aggBuilder = RelatedEntitiesAggregationStateProto.newBuilder(); state.getArguments().forEach((argName, argEntry) -> { switch (argEntry.getType()) { case SINGLE_VALUE -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry)); @@ -107,7 +105,7 @@ public class CalculatedFieldUtils { case RELATED_ENTITIES -> { RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry; relatedEntitiesArgumentEntry.getAggInputs() - .forEach((entityId, entry) -> aggBuilder.addAggArguments(toAggSingleArgumentProto(argName, entityId, entry))); + .forEach((entityId, entry) -> aggBuilder.addAggArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) entry))); } } }); @@ -122,7 +120,7 @@ public class CalculatedFieldUtils { } if (state instanceof RelatedEntitiesAggregationCalculatedFieldState aggState) { aggBuilder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); - builder.setLatestValuesAggregationState(aggBuilder.build()); + builder.setRelatedEntitiesAggregationState(aggBuilder.build()); } return builder.build(); } @@ -145,17 +143,6 @@ public class CalculatedFieldUtils { return ruleState; } - public static AggSingleArgumentEntryProto toAggSingleArgumentProto(String argName, EntityId entityId, ArgumentEntry argumentEntry) { - AggSingleArgumentEntryProto.Builder builder = AggSingleArgumentEntryProto.newBuilder() - .setEntityId(ProtoUtils.toProto(entityId)); - - if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - builder.setValue(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); - } - - return builder.build(); - } - public static SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() .setArgName(argName); @@ -166,6 +153,10 @@ public class CalculatedFieldUtils { Optional.ofNullable(entry.getVersion()).ifPresent(builder::setVersion); + if (entry.getEntityId() != null) { + builder.setEntityId(ProtoUtils.toProto(entry.getEntityId())); + } + return builder.build(); } @@ -242,11 +233,11 @@ public class CalculatedFieldUtils { } case RELATED_ENTITIES_AGGREGATION -> { RelatedEntitiesAggregationCalculatedFieldState aggState = (RelatedEntitiesAggregationCalculatedFieldState) state; - LatestValuesAggregationStateProto aggregationStateProto = proto.getLatestValuesAggregationState(); + RelatedEntitiesAggregationStateProto aggregationStateProto = proto.getRelatedEntitiesAggregationState(); Map> arguments = new HashMap<>(); aggregationStateProto.getAggArgumentsList().forEach(argProto -> { - AggSingleEntityArgumentEntry entry = fromAggSingleValueArgumentProto(argProto); - arguments.computeIfAbsent(argProto.getValue().getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); + SingleValueArgumentEntry entry = fromSingleValueArgumentProto(argProto); + arguments.computeIfAbsent(argProto.getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); }); arguments.forEach((argName, entityInputs) -> { aggState.getArguments().put(argName, new RelatedEntitiesArgumentEntry(entityInputs, false)); @@ -258,31 +249,19 @@ public class CalculatedFieldUtils { return state; } - public static AggSingleEntityArgumentEntry fromAggSingleValueArgumentProto(AggSingleArgumentEntryProto proto) { - if (!proto.hasValue()) { - return new AggSingleEntityArgumentEntry(); - } - EntityId entityId = ProtoUtils.fromProto(proto.getEntityId()); - SingleValueArgumentProto singleValueArgument = proto.getValue(); - TsValueProto tsValueProto = singleValueArgument.getValue(); - return new AggSingleEntityArgumentEntry( - entityId, - tsValueProto.getTs(), - (BasicKvEntry) KvProtoUtil.fromTsValueProto(singleValueArgument.getArgName(), tsValueProto), - singleValueArgument.getVersion() - ); - } - public static SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { if (!proto.hasValue()) { return new SingleValueArgumentEntry(); } TsValueProto tsValueProto = proto.getValue(); - return new SingleValueArgumentEntry( - tsValueProto.getTs(), - (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto), - proto.getVersion() - ); + BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto); + long ts = tsValueProto.getTs(); + long version = proto.getVersion(); + if (proto.hasEntityId()) { + EntityId entityId = ProtoUtils.fromProto(proto.getEntityId()); + return new SingleValueArgumentEntry(entityId, ts, kvEntry, version); + } + return new SingleValueArgumentEntry(ts, kvEntry, version); } public static TsRollingArgumentEntry fromRollingArgumentProto(TsRollingArgumentProto proto) { diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java deleted file mode 100644 index 0401bb0156..0000000000 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/AggSingleEntityArgumentEntryTest.java +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf.ctx.state; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; - -import java.util.UUID; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -public class AggSingleEntityArgumentEntryTest { - - private AggSingleEntityArgumentEntry entry; - - private final DeviceId device1 = new DeviceId(UUID.fromString("1984e5f4-9ff0-4187-84ae-e4438bba4c8a")); - - private final long ts = System.currentTimeMillis(); - - @BeforeEach - void setUp() { - entry = new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 12L), 22L)); - } - - @Test - void testUpdateEntryWhenNotAggEntryPassed() { - assertThatThrownBy(() -> entry.updateEntry(new TsRollingArgumentEntry(5, 30000L))) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Unsupported argument entry type for aggregation single entity argument entry: " + ArgumentEntryType.TS_ROLLING); - } - - @Test - void testUpdateEntryWhenResetPrevious() { - AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 100L)); - singleEntityArgumentEntry.setForceResetPrevious(true); - - assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); - assertThat(entry.getTs()).isEqualTo(singleEntityArgumentEntry.getTs()); - assertThat(entry.getKvEntryValue()).isEqualTo(singleEntityArgumentEntry.getKvEntryValue()); - assertThat(entry.getVersion()).isEqualTo(singleEntityArgumentEntry.getVersion()); - } - - - @Test - void testUpdateEntryWithTheSameTsAndVersion() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 19L), 22L)))).isFalse(); - } - - @Test - void testUpdateEntryWithTheSameTsAndDifferentVersion() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 134L), 23L)))).isTrue(); - } - - @Test - void testUpdateEntryWhenNewVersionIsNull() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 56L), null)))).isTrue(); - assertThat(entry.getValue()).isEqualTo(56L); - assertThat(entry.getVersion()).isNull(); - } - - @Test - void testUpdateEntryWhenNewVersionIsGreaterThanCurrent() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 76L), 23L)))).isTrue(); - assertThat(entry.getValue()).isEqualTo(76L); - assertThat(entry.getVersion()).isEqualTo(23); - } - - @Test - void testUpdateEntryWhenNewVersionIsLessThanCurrent() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 11L), 20L)))).isFalse(); - } - - @Test - void testUpdateEntryWhenValueWasNotChanged() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 40, new LongDataEntry("key", 18L), 45L)))).isTrue(); - } - - @Test - void testUpdateEntryWithOldTs() { - assertThat(entry.updateEntry(new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 150, new LongDataEntry("key", 155L), 45L)))).isFalse(); - } - -} diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java index 14d2801e0e..61b45b83c9 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java @@ -22,7 +22,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleEntityArgumentEntry; import java.util.HashMap; import java.util.Map; @@ -43,8 +42,8 @@ public class RelatedEntitiesArgumentEntryTest { @BeforeEach void setUp() { Map aggInputs = new HashMap<>(); - aggInputs.put(device1, new AggSingleEntityArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 12L), 1L))); - aggInputs.put(device2, new AggSingleEntityArgumentEntry(device2, new BasicTsKvEntry(ts - 150, new LongDataEntry("key", 16L), 6L))); + aggInputs.put(device1, new SingleValueArgumentEntry(device1, new BasicTsKvEntry(ts - 100, new LongDataEntry("key", 12L), 1L))); + aggInputs.put(device2, new SingleValueArgumentEntry(device2, new BasicTsKvEntry(ts - 150, new LongDataEntry("key", 16L), 6L))); entry = new RelatedEntitiesArgumentEntry(aggInputs, false); } @@ -62,8 +61,8 @@ public class RelatedEntitiesArgumentEntryTest { DeviceId device4 = new DeviceId(UUID.randomUUID()); RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = new RelatedEntitiesArgumentEntry(Map.of( - device3, new AggSingleEntityArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 16L), 13L)), - device4, new AggSingleEntityArgumentEntry(device4, new BasicTsKvEntry(ts - 60, new LongDataEntry("key", 23L), 7L)) + device3, new SingleValueArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 16L), 13L)), + device4, new SingleValueArgumentEntry(device4, new BasicTsKvEntry(ts - 60, new LongDataEntry("key", 23L), 7L)) ), false); assertThat(entry.updateEntry(relatedEntitiesArgumentEntry)).isTrue(); @@ -75,10 +74,10 @@ public class RelatedEntitiesArgumentEntryTest { } @Test - void testUpdateEntryWhenAggSingleEntityArgumentEntryPassedAndNoEntriesById() { + void testUpdateEntryWhenSingleValueArgumentEntryPassedAndNoEntriesById() { DeviceId device3 = new DeviceId(UUID.randomUUID()); - AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 10L)); + SingleValueArgumentEntry singleEntityArgumentEntry = new SingleValueArgumentEntry(device3, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 10L)); assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); @@ -88,8 +87,8 @@ public class RelatedEntitiesArgumentEntryTest { } @Test - void testUpdateEntryWhenAggSingleEntityArgumentEntryPassedAndEntryByIdExist() { - AggSingleEntityArgumentEntry singleEntityArgumentEntry = new AggSingleEntityArgumentEntry(device2, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 10L)); + void testUpdateEntryWhenSingleValueArgumentEntryPassedAndEntryByIdExist() { + SingleValueArgumentEntry singleEntityArgumentEntry = new SingleValueArgumentEntry(device2, new BasicTsKvEntry(ts - 50, new LongDataEntry("key", 18L), 10L)); assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java index fac988d5d9..06929de81c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/AggInput.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -31,6 +32,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonIgnoreProperties(ignoreUnknown = true) public interface AggInput { + @JsonIgnore String getType(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java index 69e4ee7fdf..9d4c7bdaf6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; import lombok.Data; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; @@ -27,9 +30,12 @@ import java.util.Map; @Data public class RelatedEntitiesAggregationCalculatedFieldConfiguration implements ArgumentsBasedCalculatedFieldConfiguration { + @NotNull private RelationPathLevel relation; private Map arguments; private long deduplicationIntervalInSec; + @Valid + @NotEmpty private Map metrics; private Output output; private boolean useLatestTs; @@ -41,9 +47,6 @@ public class RelatedEntitiesAggregationCalculatedFieldConfiguration implements A @Override public void validate() { - if (relation == null) { - throw new IllegalArgumentException("Relation must be specified!"); - } relation.validate(); if (arguments.containsKey("ctx")) { throw new IllegalArgumentException("Argument name 'ctx' is reserved and cannot be used."); @@ -51,9 +54,6 @@ public class RelatedEntitiesAggregationCalculatedFieldConfiguration implements A if (arguments.values().stream().anyMatch(Argument::hasTsRollingArgument)) { throw new IllegalArgumentException("Calculated field with type: '" + getType() + "' doesn't support TS_ROLLING arguments."); } - if (metrics.isEmpty()) { - throw new IllegalArgumentException("Latest value aggregation calculated field must have at least one metric."); - } } } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index b59e01128b..2e3a2387e7 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -888,6 +888,7 @@ message SingleValueArgumentProto { string argName = 1; TsValueProto value = 2; int64 version = 3; + EntityIdProto entityId = 4; } message TsDoubleValProto { @@ -915,14 +916,9 @@ message GeofencingArgumentProto { repeated GeofencingZoneProto zones = 2; } -message AggSingleArgumentEntryProto { - EntityIdProto entityId = 1; - SingleValueArgumentProto value = 2; -} - -message LatestValuesAggregationStateProto { +message RelatedEntitiesAggregationStateProto { int64 lastArgsUpdateTs = 1; - repeated AggSingleArgumentEntryProto aggArguments = 2; + repeated SingleValueArgumentProto aggArguments = 2; } message CalculatedFieldStateProto { @@ -932,7 +928,7 @@ message CalculatedFieldStateProto { repeated TsRollingArgumentProto rollingValueArguments = 4; repeated GeofencingArgumentProto geofencingArguments = 5; AlarmStateProto alarmState = 6; - LatestValuesAggregationStateProto latestValuesAggregationState = 7; + RelatedEntitiesAggregationStateProto relatedEntitiesAggregationState = 7; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. @@ -1303,7 +1299,7 @@ message ComponentLifecycleMsgProto { int64 profileIdLSB = 12; optional string info = 13; bool ownerChanged = 100; - bool relationChanged = 15; + bool relationChanged = 14; } message EdgeEventMsgProto { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 072a17835d..3e677cf269 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -1186,6 +1186,17 @@ public class TbUtils { return BigDecimal.valueOf(value).setScale(0, RoundingMode.HALF_UP).intValue(); } + // todo: register method + public static Object roundResult(double value, Integer precision) { + if (precision == null) { + return value; + } + if (precision.equals(0)) { + return toInt(value); + } + return toFixed(value, precision); + } + public static boolean isNaN(double value) { return Double.isNaN(value); } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java index 856cc4bd39..62d6d3d002 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java @@ -29,7 +29,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = TbelCfTsRollingArg.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = TbelCfGeofencingArg.class, name = "GEOFENCING_CF_ARGUMENT_VALUE"), @JsonSubTypes.Type(value = TbelCfPropagationArg.class, name = "PROPAGATION_CF_ARGUMENT_VALUE"), - @JsonSubTypes.Type(value = TbelCfLatestValuesAggregation.class, name = "LATEST_VALUES_AGGREGATION") + @JsonSubTypes.Type(value = TbelCfRelatedEntitiesAggregation.class, name = "LATEST_VALUES_AGGREGATION") }) public interface TbelCfArg extends TbelCfObject { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java similarity index 90% rename from common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java rename to common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java index 4d1b42aa94..75c17f0a0e 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfLatestValuesAggregation.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java @@ -20,12 +20,12 @@ import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; @Data -public class TbelCfLatestValuesAggregation implements TbelCfArg { +public class TbelCfRelatedEntitiesAggregation implements TbelCfArg { private final Object value; @JsonCreator - public TbelCfLatestValuesAggregation( + public TbelCfRelatedEntitiesAggregation( @JsonProperty("value") Object value ) { this.value = value; From f3b85f395b466ebd14369543b77e75a73f0a385f Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Wed, 22 Oct 2025 10:03:47 +0300 Subject: [PATCH 63/88] lwm2m: bootstrap new: test short serverBsId = 0 --- .../validator/DeviceProfileDataValidatorTest.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidatorTest.java index 4c427544d4..f9d6c035dc 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidatorTest.java @@ -135,8 +135,13 @@ class DeviceProfileDataValidatorTest { } @Test - void testValidateDeviceProfile_Lwm2mShortServerId_Ok_BootstrapShortServerId_0_Error() { - verifyValidationError(123, 0, msgErrorBsRange); + void testValidateDeviceProfile_Lwm2mShortServerId_Ok_BootstrapShortServerId_validate_0_to_null_Ok() { + Integer shortServerId = 123; + Integer shortServerIdBs = 0; + DeviceProfile deviceProfile = getDeviceProfile(shortServerId, shortServerIdBs); + + validator.validateDataImpl(tenantId, deviceProfile); + verify(validator).validateString("Device profile name", deviceProfile.getName()); } @Test From d97d932cfbe3bc25705f73b6a1f57af15d1b4419 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Oct 2025 10:10:08 +0300 Subject: [PATCH 64/88] refactoring --- ...alculatedFieldManagerMessageProcessor.java | 33 +++++++------- .../cf/TelemetryCalculatedFieldResult.java | 8 ++-- .../ctx/state/BaseCalculatedFieldState.java | 18 +++++--- .../cf/ctx/state/CalculatedFieldCtx.java | 32 +++++--------- ...titiesAggregationCalculatedFieldState.java | 16 +++---- .../RelatedEntitiesArgumentEntry.java | 4 ++ .../aggregation/function/BaseAggEntry.java | 4 +- .../aggregation/function/MaxAggEntry.java | 2 +- .../queue/DefaultTbClusterService.java | 7 +-- .../server/utils/CalculatedFieldUtils.java | 43 ++++++++++--------- .../data/plugin/ComponentLifecycleEvent.java | 4 +- common/proto/src/main/proto/queue.proto | 9 ++-- .../script/api/tbel/TbelCfArg.java | 2 +- .../TbelCfRelatedEntitiesAggregation.java | 2 +- 14 files changed, 95 insertions(+), 89 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 7f1c7b1925..3d61825ebe 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -187,9 +187,18 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) throws CalculatedFieldException { + var event = msg.getData().getEvent(); + if (msg.getData().isRelationChanged()) { + log.debug("Processing relation [{}] event: ", msg.getData().getEvent()); + switch (event) { + case RELATION_UPDATED -> onRelationUpdated(msg.getData(), msg.getCallback()); + case RELATION_DELETED -> onRelationDeleted(msg.getData(), msg.getCallback()); + default -> msg.getCallback().onSuccess(); + } + return; + } log.debug("Processing entity lifecycle event: [{}] for entity: [{}]", msg.getData().getEvent(), msg.getData().getEntityId()); var entityType = msg.getData().getEntityId().getEntityType(); - var event = msg.getData().getEvent(); switch (entityType) { case CALCULATED_FIELD -> { switch (event) { @@ -280,26 +289,20 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } } else if (msg.isOwnerChanged()) { onEntityOwnerChanged(msg, callback); - } else if (msg.isRelationChanged()) { - onRelationUpdated(msg, callback); } else { callback.onSuccess(); } } private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { - if (msg.isRelationChanged()) { - onRelationDeleted(msg, callback); - } else { - switch (msg.getEntityId().getEntityType()) { - case DEVICE, ASSET -> entityProfileCache.removeEntityId(msg.getEntityId()); - case CUSTOMER -> ownerEntities.remove(msg.getEntityId()); - } - ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); - if (isMyPartition(msg.getEntityId(), callback)) { - log.debug("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); - getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); - } + switch (msg.getEntityId().getEntityType()) { + case DEVICE, ASSET -> entityProfileCache.removeEntityId(msg.getEntityId()); + case CUSTOMER -> ownerEntities.remove(msg.getEntityId()); + } + ownerEntities.values().forEach(entities -> entities.remove(msg.getEntityId())); + if (isMyPartition(msg.getEntityId(), callback)) { + log.debug("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); + getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/TelemetryCalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/TelemetryCalculatedFieldResult.java index 1ad666eac5..d59ec9cca9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/TelemetryCalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/TelemetryCalculatedFieldResult.java @@ -39,6 +39,8 @@ public final class TelemetryCalculatedFieldResult implements CalculatedFieldResu private final AttributeScope scope; private final JsonNode result; + public static TelemetryCalculatedFieldResult EMPTY = TelemetryCalculatedFieldResult.builder().result(null).build(); + @Override public TbMsg toTbMsg(EntityId entityId, List cfIds) { TbMsgType msgType = switch (type) { @@ -66,9 +68,9 @@ public final class TelemetryCalculatedFieldResult implements CalculatedFieldResu @Override public boolean isEmpty() { return result == null || result.isMissingNode() || result.isNull() || - (result.isObject() && result.isEmpty()) || - (result.isArray() && result.isEmpty()) || - (result.isTextual() && result.asText().isEmpty()); + (result.isObject() && result.isEmpty()) || + (result.isArray() && result.isEmpty()) || + (result.isTextual() && result.asText().isEmpty()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index d48ed9c268..d8f13e6a20 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -20,10 +20,10 @@ import lombok.Getter; import lombok.Setter; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.actors.TbActorRef; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesArgumentEntry; import org.thingsboard.server.utils.CalculatedFieldUtils; import java.io.Closeable; @@ -75,9 +75,13 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, ArgumentEntry existingEntry = arguments.get(key); boolean entryUpdated; - if (existingEntry == null || !ctx.getCfType().equals(CalculatedFieldType.RELATED_ENTITIES_AGGREGATION) && newEntry.isForceResetPrevious()) { + if (existingEntry == null || newEntry.isForceResetPrevious()) { validateNewEntry(key, newEntry); - arguments.put(key, newEntry); + if (existingEntry instanceof RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry) { + relatedEntitiesArgumentEntry.updateEntry(newEntry); + } else { + arguments.put(key, newEntry); + } entryUpdated = true; } else { entryUpdated = existingEntry.updateEntry(newEntry); @@ -110,7 +114,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, @Override public boolean isReady() { return arguments.keySet().containsAll(requiredArguments) && - arguments.values().stream().noneMatch(ArgumentEntry::isEmpty); + arguments.values().stream().noneMatch(ArgumentEntry::isEmpty); } @Override @@ -122,9 +126,11 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState, } @Override - public void close() {} + public void close() { + } - protected void validateNewEntry(String key, ArgumentEntry newEntry) {} + protected void validateNewEntry(String key, ArgumentEntry newEntry) { + } protected ObjectNode toSimpleResult(boolean useLatestTs, ObjectNode valuesNode) { if (!useLatestTs) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index aab858d85d..40e414920a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -263,33 +263,23 @@ public class CalculatedFieldCtx { } public ListenableFuture evaluateTbelExpression(String expression, CalculatedFieldState state) { - return evaluateTbelExpression(tbelExpressions.get(expression), state); + return evaluateTbelExpression(tbelExpressions.get(expression), state.getArguments(), state.getLatestTimestamp()); } public ListenableFuture evaluateTbelExpression(CalculatedFieldScriptEngine expression, CalculatedFieldState state) { - Map arguments = new LinkedHashMap<>(); - List args = new ArrayList<>(argNames.size() + 1); - args.add(new Object()); // first element is a ctx, but we will set it later; - for (String argName : argNames) { - var arg = toTbelArgument(argName, state); - arguments.put(argName, arg); - if (arg instanceof TbelCfSingleValueArg svArg) { - args.add(svArg.getValue()); - } else { - args.add(arg); - } - } - args.set(0, new TbelCfCtx(arguments, state.getLatestTimestamp())); - - return expression.executeScriptAsync(args.toArray()); + return evaluateTbelExpression(expression, state.getArguments(), state.getLatestTimestamp()); } public ListenableFuture evaluateTbelExpression(String expression, Map entries, long latestTimestamp) { + return evaluateTbelExpression(tbelExpressions.get(expression), entries, latestTimestamp); + } + + public ListenableFuture evaluateTbelExpression(CalculatedFieldScriptEngine expression, Map entries, long latestTimestamp) { Map arguments = new LinkedHashMap<>(); List args = new ArrayList<>(argNames.size() + 1); args.add(new Object()); // first element is a ctx, but we will set it later; for (String argName : argNames) { - var arg = entries.get(argName).toTbelCfArg(); + var arg = toTbelArgument(argName, entries); arguments.put(argName, arg); if (arg instanceof TbelCfSingleValueArg svArg) { args.add(svArg.getValue()); @@ -299,7 +289,7 @@ public class CalculatedFieldCtx { } args.set(0, new TbelCfCtx(arguments, latestTimestamp)); - return tbelExpressions.get(expression).executeScriptAsync(args.toArray()); + return expression.executeScriptAsync(args.toArray()); } public ScheduledFuture scheduleReevaluation(long delayMs, TbActorRef actorCtx) { @@ -308,8 +298,8 @@ public class CalculatedFieldCtx { return systemContext.scheduleMsgWithDelay(actorCtx, new CalculatedFieldReevaluateMsg(tenantId, this), delayMs); } - private TbelCfArg toTbelArgument(String key, CalculatedFieldState state) { - return state.getArguments().get(key).toTbelCfArg(); + private TbelCfArg toTbelArgument(String key, Map arguments) { + return arguments.get(key).toTbelCfArg(); } private void initTbelExpression(String expression) { @@ -658,7 +648,7 @@ public class CalculatedFieldCtx { yield true; } yield geofencingState.getLastDynamicArgumentsRefreshTs() < - System.currentTimeMillis() - scheduledUpdateIntervalMillis; + System.currentTimeMillis() - scheduledUpdateIntervalMillis; } default -> false; }; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java index 8e78824c7c..c8731b71f7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java @@ -42,6 +42,8 @@ import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; +import static java.util.concurrent.TimeUnit.SECONDS; + @Slf4j @Getter public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculatedFieldState { @@ -50,7 +52,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat private long lastArgsRefreshTs = -1; @Setter private long lastMetricsEvalTs = -1; - private long deduplicationInterval = -1; + private long deduplicationIntervalMs = -1; private Map metrics; public RelatedEntitiesAggregationCalculatedFieldState(EntityId entityId) { @@ -62,7 +64,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat super.setCtx(ctx, actorCtx); var configuration = (RelatedEntitiesAggregationCalculatedFieldConfiguration) ctx.getCalculatedField().getConfiguration(); metrics = configuration.getMetrics(); - deduplicationInterval = configuration.getDeduplicationIntervalInSec(); + deduplicationIntervalMs = SECONDS.toMillis(configuration.getDeduplicationIntervalInSec()); } @Override @@ -76,7 +78,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat @Override public void init() { super.init(); - ctx.scheduleReevaluation(deduplicationInterval, actorCtx); + ctx.scheduleReevaluation(deduplicationIntervalMs, actorCtx); } @Override @@ -97,16 +99,14 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat Output output = ctx.getOutput(); ObjectNode aggResult = aggregateMetrics(output); lastMetricsEvalTs = System.currentTimeMillis(); - ctx.scheduleReevaluation(deduplicationInterval, actorCtx); + ctx.scheduleReevaluation(deduplicationIntervalMs, actorCtx); return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() .type(output.getType()) .scope(output.getScope()) .result(toSimpleResult(ctx.isUseLatestTs(), aggResult)) .build()); } else { - return Futures.immediateFuture(TelemetryCalculatedFieldResult.builder() - .result(null) - .build()); + return Futures.immediateFuture(TelemetryCalculatedFieldResult.EMPTY); } } @@ -125,7 +125,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat } private boolean shouldRecalculate() { - boolean intervalPassed = lastMetricsEvalTs <= System.currentTimeMillis() - deduplicationInterval; + boolean intervalPassed = lastMetricsEvalTs <= System.currentTimeMillis() - deduplicationIntervalMs; boolean argsUpdatedDuringInterval = lastArgsRefreshTs > lastMetricsEvalTs; return intervalPassed && argsUpdatedDuringInterval; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java index 45b7755af4..5b97b1ed0a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java @@ -50,6 +50,10 @@ public class RelatedEntitiesArgumentEntry implements ArgumentEntry { aggInputs.putAll(relatedEntitiesArgumentEntry.aggInputs); return true; } else if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + if (entry.isForceResetPrevious()) { + aggInputs.put(singleValueArgumentEntry.getEntityId(), singleValueArgumentEntry); + return true; + } ArgumentEntry argumentEntry = aggInputs.get(singleValueArgumentEntry.getEntityId()); if (argumentEntry != null) { argumentEntry.updateEntry(singleValueArgumentEntry); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java index b320435e99..8ca523938d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/BaseAggEntry.java @@ -43,8 +43,8 @@ public abstract class BaseAggEntry implements AggEntry { protected double extractDoubleValue(Object value) { try { - if (value instanceof Number) { - return ((Number) value).doubleValue(); + if (value instanceof Number number) { + return number.doubleValue(); } return Double.parseDouble(value.toString()); } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java index ddc47daf33..6d734a5a08 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/MaxAggEntry.java @@ -20,7 +20,7 @@ import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFuncti public class MaxAggEntry extends BaseAggEntry { - private double max = -Double.MAX_VALUE; + private double max = Double.MIN_VALUE; @Override protected void doUpdate(double value) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 22faeaf61b..5b6596c9c6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -741,7 +741,7 @@ public class DefaultTbClusterService implements TbClusterService { .tenantId(tenantId) .entityId(entityRelation.getFrom()) .relationChanged(true) - .event(ComponentLifecycleEvent.UPDATED) + .event(ComponentLifecycleEvent.RELATION_UPDATED) .info(JacksonUtil.valueToTree(entityRelation)) .build(); broadcast(msg); @@ -753,7 +753,7 @@ public class DefaultTbClusterService implements TbClusterService { .tenantId(tenantId) .entityId(entityRelation.getFrom()) .relationChanged(true) - .event(ComponentLifecycleEvent.DELETED) + .event(ComponentLifecycleEvent.RELATION_DELETED) .info(JacksonUtil.valueToTree(entityRelation)) .build(); broadcast(msg); @@ -809,7 +809,8 @@ public class DefaultTbClusterService implements TbClusterService { private void pushDeviceUpdateMessage(TenantId tenantId, EdgeId edgeId, EntityId entityId, EdgeEventActionType action) { log.trace("{} Going to send edge update notification for device actor, device id {}, edge id {}", tenantId, entityId, edgeId); switch (action) { - case ASSIGNED_TO_EDGE -> pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); + case ASSIGNED_TO_EDGE -> + pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); case UNASSIGNED_FROM_EDGE -> { EdgeId relatedEdgeId = findRelatedEdgeIdIfAny(tenantId, entityId); pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), relatedEdgeId), null); diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index d75ec8a70a..ba3aa3fd53 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -34,7 +34,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdPro import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.GeofencingZoneProto; -import org.thingsboard.server.gen.transport.TransportProtos.RelatedEntitiesAggregationStateProto; import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.TsDoubleValProto; import org.thingsboard.server.gen.transport.TransportProtos.TsRollingArgumentProto; @@ -96,16 +95,18 @@ public class CalculatedFieldUtils { .setId(toProto(stateId)) .setType(state.getType().name()); - RelatedEntitiesAggregationStateProto.Builder aggBuilder = RelatedEntitiesAggregationStateProto.newBuilder(); state.getArguments().forEach((argName, argEntry) -> { switch (argEntry.getType()) { - case SINGLE_VALUE -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry)); - case TS_ROLLING -> builder.addRollingValueArguments(toRollingArgumentProto(argName, (TsRollingArgumentEntry) argEntry)); - case GEOFENCING -> builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry)); + case SINGLE_VALUE -> + builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) argEntry)); + case TS_ROLLING -> + builder.addRollingValueArguments(toRollingArgumentProto(argName, (TsRollingArgumentEntry) argEntry)); + case GEOFENCING -> + builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry)); case RELATED_ENTITIES -> { RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry; relatedEntitiesArgumentEntry.getAggInputs() - .forEach((entityId, entry) -> aggBuilder.addAggArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) entry))); + .forEach((entityId, entry) -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) entry))); } } }); @@ -119,8 +120,7 @@ public class CalculatedFieldUtils { } } if (state instanceof RelatedEntitiesAggregationCalculatedFieldState aggState) { - aggBuilder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); - builder.setRelatedEntitiesAggregationState(aggBuilder.build()); + builder.setLastArgsUpdateTs(aggState.getLastArgsRefreshTs()); } return builder.build(); } @@ -208,6 +208,20 @@ public class CalculatedFieldUtils { case RELATED_ENTITIES_AGGREGATION -> new RelatedEntitiesAggregationCalculatedFieldState(id.entityId()); }; + if (state instanceof RelatedEntitiesAggregationCalculatedFieldState relatedEntitiesAggState) { + Map> arguments = new HashMap<>(); + proto.getSingleValueArgumentsList().forEach(argProto -> { + SingleValueArgumentEntry entry = fromSingleValueArgumentProto(argProto); + arguments.computeIfAbsent(argProto.getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); + }); + arguments.forEach((argName, entityInputs) -> { + relatedEntitiesAggState.getArguments().put(argName, new RelatedEntitiesArgumentEntry(entityInputs, false)); + }); + relatedEntitiesAggState.setLastArgsRefreshTs(proto.getLastArgsUpdateTs()); + + return relatedEntitiesAggState; + } + proto.getSingleValueArgumentsList().forEach(argProto -> state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); @@ -231,19 +245,6 @@ public class CalculatedFieldUtils { alarmState.setClearRuleState(fromAlarmRuleStateProto(alarmStateProto.getClearRuleState(), alarmState)); } } - case RELATED_ENTITIES_AGGREGATION -> { - RelatedEntitiesAggregationCalculatedFieldState aggState = (RelatedEntitiesAggregationCalculatedFieldState) state; - RelatedEntitiesAggregationStateProto aggregationStateProto = proto.getRelatedEntitiesAggregationState(); - Map> arguments = new HashMap<>(); - aggregationStateProto.getAggArgumentsList().forEach(argProto -> { - SingleValueArgumentEntry entry = fromSingleValueArgumentProto(argProto); - arguments.computeIfAbsent(argProto.getArgName(), name -> new HashMap<>()).put(entry.getEntityId(), entry); - }); - arguments.forEach((argName, entityInputs) -> { - aggState.getArguments().put(argName, new RelatedEntitiesArgumentEntry(entityInputs, false)); - }); - aggState.setLastArgsRefreshTs(aggregationStateProto.getLastArgsUpdateTs()); - } } return state; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java index 5d13db2348..31cab71e0e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java @@ -32,7 +32,9 @@ public enum ComponentLifecycleEvent implements Serializable { STOPPED(5), DELETED(6), FAILED(7), - DEACTIVATED(8); + DEACTIVATED(8), + RELATION_UPDATED(9), + RELATION_DELETED(10); @Getter private final int protoNumber; // corresponds to ComponentLifecycleEvent proto diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 2e3a2387e7..bd441dd679 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -916,11 +916,6 @@ message GeofencingArgumentProto { repeated GeofencingZoneProto zones = 2; } -message RelatedEntitiesAggregationStateProto { - int64 lastArgsUpdateTs = 1; - repeated SingleValueArgumentProto aggArguments = 2; -} - message CalculatedFieldStateProto { CalculatedFieldEntityCtxIdProto id = 1; string type = 2; @@ -928,7 +923,7 @@ message CalculatedFieldStateProto { repeated TsRollingArgumentProto rollingValueArguments = 4; repeated GeofencingArgumentProto geofencingArguments = 5; AlarmStateProto alarmState = 6; - RelatedEntitiesAggregationStateProto relatedEntitiesAggregationState = 7; + int64 lastArgsUpdateTs = 7; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. @@ -1281,6 +1276,8 @@ enum ComponentLifecycleEvent { DELETED = 6; FAILED = 7; DEACTIVATED = 8; + RELATION_UPDATED = 9; + RELATION_DELETED = 10; } message ComponentLifecycleMsgProto { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java index 62d6d3d002..2fb12917ff 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java @@ -29,7 +29,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = TbelCfTsRollingArg.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = TbelCfGeofencingArg.class, name = "GEOFENCING_CF_ARGUMENT_VALUE"), @JsonSubTypes.Type(value = TbelCfPropagationArg.class, name = "PROPAGATION_CF_ARGUMENT_VALUE"), - @JsonSubTypes.Type(value = TbelCfRelatedEntitiesAggregation.class, name = "LATEST_VALUES_AGGREGATION") + @JsonSubTypes.Type(value = TbelCfRelatedEntitiesAggregation.class, name = "RELATED_ENTITIES_AGGREGATION") }) public interface TbelCfArg extends TbelCfObject { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java index 75c17f0a0e..3373aa2474 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java @@ -34,7 +34,7 @@ public class TbelCfRelatedEntitiesAggregation implements TbelCfArg { @Override public String getType() { - return "LATEST_VALUES_AGGREGATION"; + return "RELATED_ENTITIES_AGGREGATION"; } @Override From c46d2f041568d6a7bb2e851d2bd00d192744d576 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Oct 2025 10:49:07 +0300 Subject: [PATCH 65/88] added related entities argument entry implementation --- ...titiesAggregationCalculatedFieldState.java | 4 +-- .../RelatedEntitiesArgumentEntry.java | 25 ++++++++++++------- .../function/CountUniqueAggEntry.java | 1 - .../server/utils/CalculatedFieldUtils.java | 2 +- .../RelatedEntitiesArgumentEntryTest.java | 10 ++++---- .../script/api/tbel/TbelCfArg.java | 2 +- ...> TbelCfRelatedEntitiesArgumentValue.java} | 17 +++++++------ 7 files changed, 34 insertions(+), 27 deletions(-) rename common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/{TbelCfRelatedEntitiesAggregation.java => TbelCfRelatedEntitiesArgumentValue.java} (68%) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java index c8731b71f7..7e530b6809 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java @@ -118,7 +118,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat public void cleanupEntityData(EntityId relatedEntityId) { arguments.values().forEach(argEntry -> { RelatedEntitiesArgumentEntry aggEntry = (RelatedEntitiesArgumentEntry) argEntry; - aggEntry.getAggInputs().remove(relatedEntityId); + aggEntry.getEntityInputs().remove(relatedEntityId); }); lastMetricsEvalTs = -1; lastArgsRefreshTs = System.currentTimeMillis(); @@ -135,7 +135,7 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat for (Map.Entry argEntry : arguments.entrySet()) { String key = argEntry.getKey(); RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry.getValue(); - relatedEntitiesArgumentEntry.getAggInputs().forEach((entityId, argumentEntry) -> { + relatedEntitiesArgumentEntry.getEntityInputs().forEach((entityId, argumentEntry) -> { inputs.computeIfAbsent(entityId, k -> new HashMap<>()).put(key, argumentEntry); }); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java index 5b97b1ed0a..2abe78d243 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesArgumentEntry.java @@ -18,19 +18,21 @@ package org.thingsboard.server.service.cf.ctx.state.aggregation; import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.script.api.tbel.TbelCfArg; -import org.thingsboard.script.api.tbel.TbelCfRelatedEntitiesAggregation; +import org.thingsboard.script.api.tbel.TbelCfRelatedEntitiesArgumentValue; +import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntryType; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import java.util.Map; +import java.util.stream.Collectors; @Data @AllArgsConstructor public class RelatedEntitiesArgumentEntry implements ArgumentEntry { - private final Map aggInputs; + private final Map entityInputs; private boolean forceResetPrevious; @@ -41,24 +43,24 @@ public class RelatedEntitiesArgumentEntry implements ArgumentEntry { @Override public Object getValue() { - return aggInputs; + return entityInputs; } @Override public boolean updateEntry(ArgumentEntry entry) { if (entry instanceof RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry) { - aggInputs.putAll(relatedEntitiesArgumentEntry.aggInputs); + entityInputs.putAll(relatedEntitiesArgumentEntry.entityInputs); return true; } else if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { if (entry.isForceResetPrevious()) { - aggInputs.put(singleValueArgumentEntry.getEntityId(), singleValueArgumentEntry); + entityInputs.put(singleValueArgumentEntry.getEntityId(), singleValueArgumentEntry); return true; } - ArgumentEntry argumentEntry = aggInputs.get(singleValueArgumentEntry.getEntityId()); + ArgumentEntry argumentEntry = entityInputs.get(singleValueArgumentEntry.getEntityId()); if (argumentEntry != null) { argumentEntry.updateEntry(singleValueArgumentEntry); } else { - aggInputs.put(singleValueArgumentEntry.getEntityId(), singleValueArgumentEntry); + entityInputs.put(singleValueArgumentEntry.getEntityId(), singleValueArgumentEntry); } return true; } else { @@ -68,12 +70,17 @@ public class RelatedEntitiesArgumentEntry implements ArgumentEntry { @Override public boolean isEmpty() { - return aggInputs.isEmpty(); + return entityInputs.isEmpty(); } @Override public TbelCfArg toTbelCfArg() { - return new TbelCfRelatedEntitiesAggregation(aggInputs.values()); + var inputs = entityInputs.entrySet().stream() + .collect(Collectors.toMap( + e -> e.getKey().getId(), + e -> (TbelCfSingleValueArg) e.getValue().toTbelCfArg() + )); + return new TbelCfRelatedEntitiesArgumentValue(inputs); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java index efb4a58c90..a66cbaa6af 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/function/CountUniqueAggEntry.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.cf.ctx.state.aggregation.function; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction; -import java.util.HashSet; import java.util.Optional; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java index ba3aa3fd53..fd16245695 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -105,7 +105,7 @@ public class CalculatedFieldUtils { builder.addGeofencingArguments(toGeofencingArgumentProto(argName, (GeofencingArgumentEntry) argEntry)); case RELATED_ENTITIES -> { RelatedEntitiesArgumentEntry relatedEntitiesArgumentEntry = (RelatedEntitiesArgumentEntry) argEntry; - relatedEntitiesArgumentEntry.getAggInputs() + relatedEntitiesArgumentEntry.getEntityInputs() .forEach((entityId, entry) -> builder.addSingleValueArguments(toSingleValueArgumentProto(argName, (SingleValueArgumentEntry) entry))); } } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java index 61b45b83c9..cc60b249ac 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesArgumentEntryTest.java @@ -67,10 +67,10 @@ public class RelatedEntitiesArgumentEntryTest { assertThat(entry.updateEntry(relatedEntitiesArgumentEntry)).isTrue(); - Map aggInputs = entry.getAggInputs(); + Map aggInputs = entry.getEntityInputs(); assertThat(aggInputs.size()).isEqualTo(4); - assertThat(aggInputs.get(device3)).isEqualTo(relatedEntitiesArgumentEntry.getAggInputs().get(device3)); - assertThat(aggInputs.get(device4)).isEqualTo(relatedEntitiesArgumentEntry.getAggInputs().get(device4)); + assertThat(aggInputs.get(device3)).isEqualTo(relatedEntitiesArgumentEntry.getEntityInputs().get(device3)); + assertThat(aggInputs.get(device4)).isEqualTo(relatedEntitiesArgumentEntry.getEntityInputs().get(device4)); } @Test @@ -81,7 +81,7 @@ public class RelatedEntitiesArgumentEntryTest { assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); - Map aggInputs = entry.getAggInputs(); + Map aggInputs = entry.getEntityInputs(); assertThat(aggInputs.size()).isEqualTo(3); assertThat(aggInputs.get(device3)).isEqualTo(singleEntityArgumentEntry); } @@ -92,7 +92,7 @@ public class RelatedEntitiesArgumentEntryTest { assertThat(entry.updateEntry(singleEntityArgumentEntry)).isTrue(); - Map aggInputs = entry.getAggInputs(); + Map aggInputs = entry.getEntityInputs(); assertThat(aggInputs.size()).isEqualTo(2); assertThat(aggInputs.get(device2)).isEqualTo(singleEntityArgumentEntry); } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java index 2fb12917ff..4f2719fb75 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java @@ -29,7 +29,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = TbelCfTsRollingArg.class, name = "TS_ROLLING"), @JsonSubTypes.Type(value = TbelCfGeofencingArg.class, name = "GEOFENCING_CF_ARGUMENT_VALUE"), @JsonSubTypes.Type(value = TbelCfPropagationArg.class, name = "PROPAGATION_CF_ARGUMENT_VALUE"), - @JsonSubTypes.Type(value = TbelCfRelatedEntitiesAggregation.class, name = "RELATED_ENTITIES_AGGREGATION") + @JsonSubTypes.Type(value = TbelCfRelatedEntitiesArgumentValue.class, name = "RELATED_ENTITIES_ARGUMENT_VALUE") }) public interface TbelCfArg extends TbelCfObject { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesArgumentValue.java similarity index 68% rename from common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java rename to common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesArgumentValue.java index 3373aa2474..02d641d576 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesAggregation.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfRelatedEntitiesArgumentValue.java @@ -19,22 +19,23 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; +import java.util.Collections; +import java.util.Map; +import java.util.UUID; + @Data -public class TbelCfRelatedEntitiesAggregation implements TbelCfArg { +public class TbelCfRelatedEntitiesArgumentValue implements TbelCfArg { - private final Object value; + private final Map entityInputs; @JsonCreator - public TbelCfRelatedEntitiesAggregation( - @JsonProperty("value") Object value - ) { - this.value = value; + public TbelCfRelatedEntitiesArgumentValue(@JsonProperty("entityInputs") Map values) { + this.entityInputs = Collections.unmodifiableMap(values); } - @Override public String getType() { - return "RELATED_ENTITIES_AGGREGATION"; + return "RELATED_ENTITIES_ARGUMENT_VALUE"; } @Override From cd15206061d80f9998d95273613ffa3f9126347d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Oct 2025 12:12:09 +0300 Subject: [PATCH 66/88] relation changed processing --- ...alculatedFieldManagerMessageProcessor.java | 57 ++++++++----------- .../queue/DefaultTbClusterService.java | 2 - .../msg/plugin/ComponentLifecycleMsg.java | 6 +- .../server/common/util/ProtoUtils.java | 2 - common/proto/src/main/proto/queue.proto | 1 - 5 files changed, 26 insertions(+), 42 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 3d61825ebe..96f7e9aa80 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationPathQuery; import org.thingsboard.server.common.data.relation.EntitySearchDirection; @@ -77,6 +78,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; +import java.util.function.Function; import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; @@ -188,16 +190,12 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) throws CalculatedFieldException { var event = msg.getData().getEvent(); - if (msg.getData().isRelationChanged()) { - log.debug("Processing relation [{}] event: ", msg.getData().getEvent()); - switch (event) { - case RELATION_UPDATED -> onRelationUpdated(msg.getData(), msg.getCallback()); - case RELATION_DELETED -> onRelationDeleted(msg.getData(), msg.getCallback()); - default -> msg.getCallback().onSuccess(); - } + if (ComponentLifecycleEvent.RELATION_UPDATED.equals(event) || ComponentLifecycleEvent.RELATION_DELETED.equals(event)) { + log.debug("Processing relation [{}] event from entity: [{}]", event, msg.getData().getEntityId()); + onRelationChangedEvent(msg.getData(), msg.getCallback()); return; } - log.debug("Processing entity lifecycle event: [{}] for entity: [{}]", msg.getData().getEvent(), msg.getData().getEntityId()); + log.debug("Processing entity lifecycle event: [{}] for entity: [{}]", event, msg.getData().getEntityId()); var entityType = msg.getData().getEntityId().getEntityType(); switch (entityType) { case CALCULATED_FIELD -> { @@ -306,36 +304,29 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } } - private void onRelationUpdated(ComponentLifecycleMsg msg, TbCallback callback) { - try { - EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); - EntityId toId = entityRelation.getTo(); - EntityId fromId = entityRelation.getFrom(); - String relationType = entityRelation.getType(); - - MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); - processRelationByDirection(EntitySearchDirection.TO, relationType, toId, callbackForToAndFrom, (entityId, ctx, cb) -> initRelatedEntity(entityId, fromId, ctx, cb)); - processRelationByDirection(EntitySearchDirection.FROM, relationType, fromId, callbackForToAndFrom, (entityId, ctx, cb) -> initRelatedEntity(entityId, toId, ctx, cb)); - } catch (Exception e) { - callback.onSuccess(); - } - } + private void onRelationChangedEvent(ComponentLifecycleMsg msg, TbCallback callback) { + Function> relationAction = switch (msg.getEvent()) { + case RELATION_UPDATED -> relatedId -> (entityId, ctx, cb) -> initRelatedEntity(entityId, relatedId, ctx, cb); + case RELATION_DELETED -> relatedId -> (entityId, ctx, cb) -> deleteRelatedEntity(entityId, relatedId, ctx, cb); + default -> null; + }; - private void onRelationDeleted(ComponentLifecycleMsg msg, TbCallback callback) { - try { - EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); - EntityId toId = entityRelation.getTo(); - EntityId fromId = entityRelation.getFrom(); - String relationType = entityRelation.getType(); - - MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); - processRelationByDirection(EntitySearchDirection.TO, relationType, toId, callbackForToAndFrom, (entityId, ctx, cb) -> deleteRelatedEntity(entityId, fromId, ctx, cb)); - processRelationByDirection(EntitySearchDirection.FROM, relationType, fromId, callbackForToAndFrom, (entityId, ctx, cb) -> deleteRelatedEntity(entityId, toId, ctx, cb)); - } catch (Exception e) { + if (relationAction == null) { callback.onSuccess(); + return; } + + EntityRelation entityRelation = JacksonUtil.treeToValue(msg.getInfo(), EntityRelation.class); + EntityId toId = entityRelation.getTo(); + EntityId fromId = entityRelation.getFrom(); + String relationType = entityRelation.getType(); + + MultipleTbCallback callbackForToAndFrom = new MultipleTbCallback(2, callback); + processRelationByDirection(EntitySearchDirection.TO, relationType, toId, callbackForToAndFrom, relationAction.apply(fromId)); + processRelationByDirection(EntitySearchDirection.FROM, relationType, fromId, callbackForToAndFrom, relationAction.apply(toId)); } + private void processRelationByDirection(EntitySearchDirection direction, String relationType, EntityId mainId, diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 5b6596c9c6..dde24d358a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -740,7 +740,6 @@ public class DefaultTbClusterService implements TbClusterService { ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() .tenantId(tenantId) .entityId(entityRelation.getFrom()) - .relationChanged(true) .event(ComponentLifecycleEvent.RELATION_UPDATED) .info(JacksonUtil.valueToTree(entityRelation)) .build(); @@ -752,7 +751,6 @@ public class DefaultTbClusterService implements TbClusterService { ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() .tenantId(tenantId) .entityId(entityRelation.getFrom()) - .relationChanged(true) .event(ComponentLifecycleEvent.RELATION_DELETED) .info(JacksonUtil.valueToTree(entityRelation)) .build(); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index 38df529c96..23b9fe08e3 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -47,15 +47,14 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { private final EntityId oldProfileId; private final EntityId profileId; private final boolean ownerChanged; - private final boolean relationChanged; private final JsonNode info; public ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event) { - this(tenantId, entityId, event, null, null, null, null, false, false, null); + this(tenantId, entityId, event, null, null, null, null, false, null); } @Builder - private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId, boolean ownerChanged, boolean relationChanged, JsonNode info) { + private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId, boolean ownerChanged, JsonNode info) { this.tenantId = tenantId; this.entityId = entityId; this.event = event; @@ -64,7 +63,6 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { this.oldProfileId = oldProfileId; this.profileId = profileId; this.ownerChanged = ownerChanged; - this.relationChanged = relationChanged; this.info = info; } diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 83fb158efd..26a64c7f8a 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -130,7 +130,6 @@ public class ProtoUtils { builder.setOldProfileIdLSB(msg.getOldProfileId().getId().getLeastSignificantBits()); } builder.setOwnerChanged(msg.isOwnerChanged()); - builder.setRelationChanged(msg.isRelationChanged()); if (msg.getName() != null) { builder.setName(msg.getName()); } @@ -168,7 +167,6 @@ public class ProtoUtils { builder.oldProfileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB()))); } builder.ownerChanged(proto.getOwnerChanged()); - builder.relationChanged(proto.getRelationChanged()); if (proto.hasInfo()) { builder.info(JacksonUtil.toJsonNode(proto.getInfo())); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index bd441dd679..b060b47429 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1296,7 +1296,6 @@ message ComponentLifecycleMsgProto { int64 profileIdLSB = 12; optional string info = 13; bool ownerChanged = 100; - bool relationChanged = 14; } message EdgeEventMsgProto { From f48b8752d225e31f85b2acd745c8d5babc21bc0d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Oct 2025 14:33:12 +0300 Subject: [PATCH 67/88] fixed agg cfs filtration in queueu service --- .../service/cf/DefaultCalculatedFieldQueueService.java | 9 ++++++++- .../service/cf/ctx/state/CalculatedFieldState.java | 2 -- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java index 84f49c7c9e..c7e369a862 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -199,7 +199,14 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS RelationPathLevel inverseRelation = new RelationPathLevel(inverseDirection, relation.relationType()); List byRelationPathQuery = relationService.findByRelationPathQuery(tenantId, new EntityRelationPathQuery(entityId, List.of(inverseRelation))); if (!byRelationPathQuery.isEmpty()) { - return true; + EntityId cfEntityId = cfCtx.getEntityId(); + for (EntityRelation entityRelation : byRelationPathQuery) { + EntityId relatedId = (inverseDirection == EntitySearchDirection.FROM) ? entityRelation.getTo() : entityRelation.getFrom(); + if (cfEntityId.equals(relatedId) || cfEntityId.equals(calculatedFieldCache.getProfileId(tenantId, relatedId))) { + return true; + } + } + return false; } } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 2b3ba19528..34e9dad439 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -55,8 +55,6 @@ public interface CalculatedFieldState extends Closeable { long getLatestTimestamp(); - CalculatedFieldCtx getCtx(); - void setCtx(CalculatedFieldCtx ctx, TbActorRef actorCtx); void init(); From 25f6ab7eab222b5c91a224c5e1822b1dd9f68a67 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 22 Oct 2025 15:42:05 +0300 Subject: [PATCH 68/88] fixed TextContent preparation for Github AI modes --- .../server/service/ai/AiChatModelServiceImpl.java | 4 ++-- .../org/thingsboard/server/common/data/StringUtils.java | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java b/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java index c1829cbf84..899c7c4aba 100644 --- a/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java +++ b/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java @@ -32,7 +32,7 @@ import org.thingsboard.server.common.data.ai.model.chat.Langchain4jChatModelConf import java.util.List; import java.util.stream.Collectors; -import static org.thingsboard.server.common.data.StringUtils.escapeControlChars; +import static org.thingsboard.server.common.data.StringUtils.escapeJson; @Service @RequiredArgsConstructor @@ -74,7 +74,7 @@ class AiChatModelServiceImpl implements AiChatModelService { private Content prepareContent(Content content) { if (content instanceof TextContent txt) { - return new TextContent(escapeControlChars(txt.text())); + return new TextContent(escapeJson(txt.text())); } return content; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java index 2b8b631027..1ae5567c71 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java @@ -275,8 +275,12 @@ public class StringUtils { return result; } - public static String escapeControlChars(String text) { + public static String escapeJson(String text) { return text + .replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\b", "\\b") + .replace("\f", "\\f") .replace("\n", "\\n") .replace("\r", "\\r") .replace("\t", "\\t"); From 269794ec27d9faa2452c3761e11908968f25f95b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Oct 2025 15:54:33 +0300 Subject: [PATCH 69/88] added json subtype --- .../server/service/cf/ctx/state/CalculatedFieldState.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 34e9dad439..9598cc2b49 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.aggregation.RelatedEntitiesAggregationCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.alarm.AlarmCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.geofencing.GeofencingCalculatedFieldState; @@ -42,7 +43,8 @@ import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArg @Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT"), @Type(value = GeofencingCalculatedFieldState.class, name = "GEOFENCING"), @Type(value = AlarmCalculatedFieldState.class, name = "ALARM"), - @Type(value = PropagationCalculatedFieldState.class, name = "PROPAGATION") + @Type(value = PropagationCalculatedFieldState.class, name = "PROPAGATION"), + @Type(value = RelatedEntitiesAggregationCalculatedFieldState.class, name = "RELATED_ENTITIES_AGGREGATION") }) public interface CalculatedFieldState extends Closeable { From db83eba4c40f7942f1b394bf88e94aa66033c3f3 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 22 Oct 2025 18:02:52 +0300 Subject: [PATCH 70/88] refactoring --- .../server/service/ai/AiChatModelServiceImpl.java | 5 ++--- .../thingsboard/server/common/data/StringUtils.java | 11 ----------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java b/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java index 899c7c4aba..639e2025fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java +++ b/application/src/main/java/org/thingsboard/server/service/ai/AiChatModelServiceImpl.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.ai; +import com.fasterxml.jackson.core.io.JsonStringEncoder; import com.google.common.util.concurrent.FluentFuture; import dev.langchain4j.data.message.ChatMessage; import dev.langchain4j.data.message.Content; @@ -32,8 +33,6 @@ import org.thingsboard.server.common.data.ai.model.chat.Langchain4jChatModelConf import java.util.List; import java.util.stream.Collectors; -import static org.thingsboard.server.common.data.StringUtils.escapeJson; - @Service @RequiredArgsConstructor class AiChatModelServiceImpl implements AiChatModelService { @@ -74,7 +73,7 @@ class AiChatModelServiceImpl implements AiChatModelService { private Content prepareContent(Content content) { if (content instanceof TextContent txt) { - return new TextContent(escapeJson(txt.text())); + return new TextContent(new String(JsonStringEncoder.getInstance().quoteAsString(txt.text()))); } return content; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java index 1ae5567c71..cbc881d72f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java @@ -275,15 +275,4 @@ public class StringUtils { return result; } - public static String escapeJson(String text) { - return text - .replace("\\", "\\\\") - .replace("\"", "\\\"") - .replace("\b", "\\b") - .replace("\f", "\\f") - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace("\t", "\\t"); - } - } From 6dba0b6fd21ce4a6d1a388a88a60915f2f26e10f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 23 Oct 2025 11:19:53 +0300 Subject: [PATCH 71/88] scheduling for agg cfs on restart --- .../CalculatedFieldManagerMessageProcessor.java | 5 ++++- .../RelatedEntitiesAggregationCalculatedFieldState.java | 6 ------ ...tedEntitiesAggregationCalculatedFieldConfiguration.java | 7 +++++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 40707e32f3..de3967d5b6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -652,7 +652,10 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private List getCalculatedFieldsByEntityIdAndProfile(EntityId entityId) { List cfsByEntityIdAndProfile = new ArrayList<>(); cfsByEntityIdAndProfile.addAll(getCalculatedFieldsByEntityId(entityId)); - cfsByEntityIdAndProfile.addAll(getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId))); + EntityId profileId = getProfileId(tenantId, entityId); + if (profileId != null) { + cfsByEntityIdAndProfile.addAll(getCalculatedFieldsByEntityId(profileId)); + } return cfsByEntityIdAndProfile; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java index 7e530b6809..655217263b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/RelatedEntitiesAggregationCalculatedFieldState.java @@ -75,12 +75,6 @@ public class RelatedEntitiesAggregationCalculatedFieldState extends BaseCalculat metrics = null; } - @Override - public void init() { - super.init(); - ctx.scheduleReevaluation(deduplicationIntervalMs, actorCtx); - } - @Override public CalculatedFieldType getType() { return CalculatedFieldType.RELATED_ENTITIES_AGGREGATION; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java index 9d4c7bdaf6..931cb919ec 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.cf.configuration.aggregation; +import com.fasterxml.jackson.annotation.JsonIgnore; import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; @@ -56,4 +57,10 @@ public class RelatedEntitiesAggregationCalculatedFieldConfiguration implements A } } + @JsonIgnore + @Override + public boolean requiresScheduledReevaluation() { + return true; + } + } From bb308919ce3d8ad297b79c4288bf93c8f3230289 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 23 Oct 2025 15:43:43 +0300 Subject: [PATCH 72/88] UI: Add new tenant profile configuration minAllowedDeduplicationIntervalInSecForCF --- ui-ngx/src/app/core/auth/auth.models.ts | 1 + ui-ngx/src/app/core/auth/auth.reducer.ts | 1 + ...ult-tenant-profile-configuration.component.html | 14 +++++++++++++- ...fault-tenant-profile-configuration.component.ts | 1 + ui-ngx/src/app/shared/models/tenant.model.ts | 2 ++ .../src/assets/locale/locale.constant-en_US.json | 3 +++ 6 files changed, 21 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/core/auth/auth.models.ts b/ui-ngx/src/app/core/auth/auth.models.ts index d6612f2427..6e4d324b5b 100644 --- a/ui-ngx/src/app/core/auth/auth.models.ts +++ b/ui-ngx/src/app/core/auth/auth.models.ts @@ -31,6 +31,7 @@ export interface SysParamsState { maxDebugModeDurationMinutes: number; maxDataPointsPerRollingArg: number; maxArgumentsPerCF: number; + minAllowedDeduplicationIntervalInSecForCF: number; minAllowedScheduledUpdateIntervalInSecForCF: number; maxRelationLevelPerCfArgument: number; ruleChainDebugPerTenantLimitsConfiguration?: string; diff --git a/ui-ngx/src/app/core/auth/auth.reducer.ts b/ui-ngx/src/app/core/auth/auth.reducer.ts index 8cfbc04197..777cf5308e 100644 --- a/ui-ngx/src/app/core/auth/auth.reducer.ts +++ b/ui-ngx/src/app/core/auth/auth.reducer.ts @@ -33,6 +33,7 @@ const emptyUserAuthState: AuthPayload = { mobileQrEnabled: false, maxResourceSize: 0, maxArgumentsPerCF: 0, + minAllowedDeduplicationIntervalInSecForCF: 0, minAllowedScheduledUpdateIntervalInSecForCF: 0, maxRelationLevelPerCfArgument: 0, maxDataPointsPerRollingArg: 0, diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html index d53cacbd83..8cfa8fd0ed 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html @@ -354,7 +354,19 @@ tenant-profile.relation-search-entity-limit-hint -
+ + tenant-profile.min-allowed-deduplication-interval + + + {{ 'tenant-profile.min-allowed-deduplication-interval-required' | translate}} + + + {{ 'tenant-profile.min-allowed-deduplication-interval-range' | translate}} + + + diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts index 9595def95e..0000d01995 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts @@ -116,6 +116,7 @@ export class DefaultTenantProfileConfigurationComponent implements ControlValueA maxCalculatedFieldsPerEntity: [0, [Validators.required, Validators.min(0)]], maxArgumentsPerCF: [0, [Validators.required, Validators.min(0)]], maxRelationLevelPerCfArgument: [1, [Validators.required, Validators.min(1)]], + minAllowedDeduplicationIntervalInSecForCF: [0, [Validators.required, Validators.min(0)]], maxRelatedEntitiesToReturnPerCfArgument: [1, [Validators.required, Validators.min(1)]], minAllowedScheduledUpdateIntervalInSecForCF: [0, [Validators.required, Validators.min(0)]], maxDataPointsPerRollingArg: [0, [Validators.required, Validators.min(0)]], diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 1ed1092207..ae7a0ae8b8 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -107,6 +107,7 @@ export interface DefaultTenantProfileConfiguration { maxCalculatedFieldsPerEntity: number; maxArgumentsPerCF: number; maxRelationLevelPerCfArgument: number; + minAllowedDeduplicationIntervalInSecForCF: number; maxRelatedEntitiesToReturnPerCfArgument: number; minAllowedScheduledUpdateIntervalInSecForCF: number; maxDataPointsPerRollingArg: number; @@ -174,6 +175,7 @@ export function createTenantProfileConfiguration(type: TenantProfileType): Tenan maxArgumentsPerCF: 10, maxDataPointsPerRollingArg: 1000, maxRelationLevelPerCfArgument: 10, + minAllowedDeduplicationIntervalInSecForCF: 3600, maxRelatedEntitiesToReturnPerCfArgument: 100, minAllowedScheduledUpdateIntervalInSecForCF: 0, maxStateSizeInKBytes: 32, diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 3ee738a8b0..331f3cdb09 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -5876,6 +5876,9 @@ "max-related-level-per-argument-required": "Relation level per 'Related entities' argument max number is required", "min-allowed-scheduled-update-interval": "Min allowed update interval for 'Related entities' arguments (seconds)", "min-allowed-scheduled-update-interval-range": "Min allowed update interval min number can't be negative", + "min-allowed-deduplication-interval": "Min allowed deduplication interval (seconds)", + "min-allowed-deduplication-interval-range": "Min allowed deduplication interval value can't be negative", + "min-allowed-deduplication-interval-required": "Min allowed deduplication interval is required", "min-allowed-scheduled-update-interval-required": "Min allowed update interval min number is required", "max-state-size": "State maximum size in KB", "max-state-size-range": "State maximum size in KB can't be negative", From 59d2fe8c7b40866cd97f58cff2162d9a2aa0e905 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 23 Oct 2025 19:06:55 +0300 Subject: [PATCH 73/88] UI: Add bacis config RELATED_ENTITIES_AGGREGATION cf --- .../calculated-field.module.ts | 4 + ...ulated-field-argument-panel.component.html | 54 +++--- ...lculated-field-argument-panel.component.ts | 11 ++ ...calculated-field-arguments-table.module.ts | 9 +- ...d-aggregation-arguments-table.component.ts | 70 ++++++++ .../calculated-field-dialog.component.html | 7 + .../calculated-field-dialog.component.scss | 3 + .../calculated-field-output.component.html | 85 +++++----- .../calculated-field-output.component.ts | 13 +- ...ities-aggregation-component.component.html | 75 +++++++++ ...ntities-aggregation-component.component.ts | 154 ++++++++++++++++++ ...d-entities-aggregation-component.module.ts | 45 +++++ .../components/time-unit-input.component.html | 6 +- .../shared/models/calculated-field.models.ts | 24 ++- .../assets/locale/locale.constant-en_US.json | 15 +- 15 files changed, 505 insertions(+), 70 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/related-aggregation-arguments-table.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts index e0db7a6eef..5e3a854aa2 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-field.module.ts @@ -38,6 +38,9 @@ import { import { PropagationConfigurationModule } from '@home/components/calculated-fields/components/propagation-configuration/propagation-configuration.module'; +import { + RelatedEntitiesAggregationComponentModule +} from '@home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module'; @NgModule({ declarations: [ @@ -52,6 +55,7 @@ import { EntityDebugSettingsButtonComponent, SimpleConfigurationModule, PropagationConfigurationModule, + RelatedEntitiesAggregationComponentModule, ], exports: [ CalculatedFieldDialogComponent, diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html index 77bdedb068..607b107094 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html @@ -18,6 +18,11 @@
{{ 'calculated-fields.argument-settings' | translate }}
+ @if (hint) { +
+ {{ hint | translate }} +
+ }
@if (!isOutputKey) { } -
-
{{ 'entity.entity-type' | translate }}
- - - @for (type of argumentEntityTypes; track type) { - {{ ArgumentEntityTypeTranslations.get(type) | translate }} + @if (!hiddenEntityTypes) { +
+
{{ 'entity.entity-type' | translate }}
+ + + @for (type of argumentEntityTypes; track type) { + {{ ArgumentEntityTypeTranslations.get(type) | translate }} + } + + @if (argumentType.touched && argumentType.hasError('required')) { + + warning + } - - @if (argumentType.touched && argumentType.hasError('required')) { - - warning - - } - -
+ +
+ } @if (ArgumentEntityTypeParamsMap.has(entityType)) {
{{ ArgumentEntityTypeParamsMap.get(entityType).title | translate }}
@@ -143,9 +150,18 @@ } @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Rolling) {
-
{{ 'calculated-fields.default-value' | translate }}
+
{{ 'calculated-fields.default-value' | translate }}
+ @if (argumentFormGroup.get('defaultValue').touched && argumentFormGroup.get('defaultValue').hasError('required')) { + + warning + + }
} @else { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts index 070ffb5c06..6f90d126e0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.ts @@ -52,6 +52,7 @@ import { Store } from '@ngrx/store'; import { EntityAutocompleteComponent } from '@shared/components/entity/entity-autocomplete.component'; import { NULL_UUID } from '@shared/models/id/has-uuid'; import { TenantId } from '@shared/models/id/tenant-id'; +import { deduplicationStrategiesHintTranslations } from '@home/components/rule-node/rule-node-config.models'; @Component({ selector: 'tb-calculated-field-argument-panel', @@ -68,6 +69,9 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI @Input() isScript: boolean; @Input() usedArgumentNames: string[]; @Input() isOutputKey = false; + @Input() hiddenEntityTypes = false; + @Input() defaultValueRequired = false; + @Input() hint: string; @Input() argumentEntityTypes = Object.values(ArgumentEntityType).filter(value => value !== ArgumentEntityType.RelationQuery) as ArgumentEntityType[]; @ViewChild('entityAutocomplete') entityAutocomplete: EntityAutocompleteComponent; @@ -146,6 +150,11 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI this.setInitialEntityType(); this.setWatchKeyChange(); + if (this.defaultValueRequired) { + this.argumentFormGroup.get('defaultValue').addValidators(Validators.required); + this.argumentFormGroup.get('defaultValue').updateValueAndValidity({onlySelf: true}); + } + this.argumentTypes = Object.values(ArgumentType) .filter(type => type !== ArgumentType.Rolling || this.isScript); } @@ -311,4 +320,6 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI this.entityNameSubject.next(null); } } + + protected readonly deduplicationStrategiesHintTranslations = deduplicationStrategiesHintTranslations; } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module.ts index 082001f052..cae0b92387 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module.ts @@ -26,6 +26,9 @@ import { import { PropagateArgumentsTableComponent } from '@home/components/calculated-fields/components/calculated-field-arguments/propagate-arguments-table.component'; +import { + RelatedAggregationArgumentsTableComponent +} from '@home/components/calculated-fields/components/calculated-field-arguments/related-aggregation-arguments-table.component'; @NgModule({ imports: [ @@ -35,11 +38,13 @@ import { declarations: [ CalculatedFieldArgumentPanelComponent, CalculatedFieldArgumentsTableComponent, - PropagateArgumentsTableComponent + PropagateArgumentsTableComponent, + RelatedAggregationArgumentsTableComponent ], exports: [ CalculatedFieldArgumentsTableComponent, - PropagateArgumentsTableComponent + PropagateArgumentsTableComponent, + RelatedAggregationArgumentsTableComponent ] }) export class CalculatedFieldArgumentsTableModule {} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/related-aggregation-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/related-aggregation-arguments-table.component.ts new file mode 100644 index 0000000000..7c9212d3c2 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/related-aggregation-arguments-table.component.ts @@ -0,0 +1,70 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { ChangeDetectorRef, Component, DestroyRef, forwardRef, Renderer2, ViewContainerRef, } from '@angular/core'; +import { FormBuilder, NG_VALIDATORS, NG_VALUE_ACCESSOR, } from '@angular/forms'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityService } from '@core/http/entity.service'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { + CalculatedFieldArgumentsTableComponent +} from '@home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.component'; +import { ArgumentEntityType } from '@shared/models/calculated-field.models'; + +@Component({ + selector: 'tb-related-aggregation-arguments-table', + templateUrl: './calculated-field-arguments-table.component.html', + styleUrls: [`calculated-field-arguments-table.component.scss`], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => RelatedAggregationArgumentsTableComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => RelatedAggregationArgumentsTableComponent), + multi: true + } + ], +}) +export class RelatedAggregationArgumentsTableComponent extends CalculatedFieldArgumentsTableComponent { + + constructor( + protected fb: FormBuilder, + protected popoverService: TbPopoverService, + protected viewContainerRef: ViewContainerRef, + protected cd: ChangeDetectorRef, + protected renderer: Renderer2, + protected entityService: EntityService, + protected destroyRef: DestroyRef, + protected store: Store + ) { + super(fb, popoverService, viewContainerRef, cd, renderer, entityService, destroyRef, store); + + this.argumentNameColumn = 'calculated-fields.argument-name'; + this.displayColumns = ['name', 'type', 'key', 'actions']; + this.panelAdditionalCtx = { + hiddenEntityTypes: true, + defaultValueRequired: true, + argumentEntityTypes: [ArgumentEntityType.Current], + hint: 'calculated-fields.hint.setting-arguments-aggregation' + }; + + this.isScript = false; + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 1d9dcc98f1..478d41f42b 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -75,6 +75,13 @@ [testScript]="onTestScript.bind(this)"> } + @case (CalculatedFieldType.RELATED_ENTITIES_AGGREGATION) { + + + } @default { @if (simpleMode) { -
- - - {{ - (outputForm.get('type').value === OutputType.Timeseries - ? 'calculated-fields.timeseries-key' - : 'calculated-fields.attribute-key') - | translate - }} - - - @if (outputForm.get('name').errors && outputForm.get('name').touched) { - - @if (outputForm.get('name').hasError('required')) { - {{ 'common.hint.key-required' | translate }} - } @else if (outputForm.get('name').hasError('pattern')) { - {{ 'common.hint.key-pattern' | translate }} - } @else if (outputForm.get('name').hasError('maxlength')) { - {{ 'common.hint.key-max-length' | translate }} - } - - } - - - {{ 'calculated-fields.decimals-by-default' | translate }} - - @if (outputForm.get('decimalsByDefault').errors && outputForm.get('decimalsByDefault').touched) { - {{ 'calculated-fields.hint.decimals-range' | translate }} - } - -
- - - - - - - - - + @if (hiddenName) { +
+ + {{ 'calculated-fields.decimals-by-default' | translate }} + + @if (outputForm.get('decimalsByDefault').errors && outputForm.get('decimalsByDefault').touched) { + {{ 'calculated-fields.hint.decimals-range' | translate }} + } + + +
+ } @else { +
+ + + {{ + (outputForm.get('type').value === OutputType.Timeseries + ? 'calculated-fields.timeseries-key' + : 'calculated-fields.attribute-key') + | translate + }} + + + @if (outputForm.get('name').errors && outputForm.get('name').touched) { + + @if (outputForm.get('name').hasError('required')) { + {{ 'common.hint.key-required' | translate }} + } @else if (outputForm.get('name').hasError('pattern')) { + {{ 'common.hint.key-pattern' | translate }} + } @else if (outputForm.get('name').hasError('maxlength')) { + {{ 'common.hint.key-max-length' | translate }} + } + + } + + + {{ 'calculated-fields.decimals-by-default' | translate }} + + @if (outputForm.get('decimalsByDefault').errors && outputForm.get('decimalsByDefault').touched) { + {{ 'calculated-fields.hint.decimals-range' | translate }} + } + +
+ + } }
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.ts index 9a95c921fa..464ca99bd2 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.ts @@ -35,6 +35,7 @@ import { digitsRegex, oneSpaceInsideRegex } from '@shared/models/regex.constants import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityType } from '@shared/models/entity-type.models'; +import { coerceBoolean } from '@shared/decorators/coercion'; @Component({ selector: 'tb-calculate-field-output', @@ -55,8 +56,13 @@ import { EntityType } from '@shared/models/entity-type.models'; export class CalculatedFieldOutputComponent implements ControlValueAccessor, Validator, OnInit, OnChanges { @Input() + @coerceBoolean() simpleMode = false; + @Input() + @coerceBoolean() + hiddenName = false; + @Input({required: true}) entityId: EntityId; @@ -137,11 +143,14 @@ export class CalculatedFieldOutputComponent implements ControlValueAccessor, Val } private updatedFormWithMode(): void { - if (this.simpleMode) { + if (this.simpleMode && !this.hiddenName) { this.outputForm.get('name').enable({emitEvent: false}); - this.outputForm.get('decimalsByDefault').enable({emitEvent: false}); } else { this.outputForm.get('name').disable({emitEvent: false}); + } + if (this.simpleMode) { + this.outputForm.get('decimalsByDefault').enable({emitEvent: false}); + } else { this.outputForm.get('decimalsByDefault').disable({emitEvent: false}); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html new file mode 100644 index 0000000000..1988141c2a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html @@ -0,0 +1,75 @@ + +
+
+
+ {{ 'calculated-fields.aggregation-path-related-entities' | translate }} +
+
+ + {{ 'calculated-fields.direction' | translate }} + + @for (direction of Directions; track direction) { + {{ PropagationDirectionTranslations.get(direction) | translate }} + } + + + + +
+
+
+
+ {{ 'calculated-fields.arguments' | translate }} +
+ +
+
+
+ {{ 'calculated-fields.metrics' | translate }} +
+ + +
+ +
+ +
+ calculated-fields.use-latest-timestamp +
+ +
+ +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.ts new file mode 100644 index 0000000000..d10372beea --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.ts @@ -0,0 +1,154 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input } from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { EntityId } from '@shared/models/id/entity-id'; +import { Observable, of } from 'rxjs'; +import { + CalculatedFieldOutput, + CalculatedFieldRelatedAggregationConfiguration, + CalculatedFieldType, + getCalculatedFieldArgumentsEditorCompleter, + getCalculatedFieldArgumentsHighlights, + OutputType, + PropagationDirectionTranslations +} from '@shared/models/calculated-field.models'; +import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; +import { map } from 'rxjs/operators'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { ScriptLanguage } from '@app/shared/models/rule-node.models'; +import { EntitySearchDirection } from '@shared/models/relation.models'; +import { getCurrentAuthState } from '@core/auth/auth.selectors'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; + +@Component({ + selector: 'tb-related-entities-aggregation-component', + templateUrl: './related-entities-aggregation-component.component.html', + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => RelatedEntitiesAggregationComponentComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => RelatedEntitiesAggregationComponentComponent), + multi: true + } + ], +}) +export class RelatedEntitiesAggregationComponentComponent implements ControlValueAccessor, Validator { + + @Input({required: true}) + entityId: EntityId; + + @Input({required: true}) + tenantId: string; + + @Input({required: true}) + entityName: string; + + relatedAggregationConfiguration = this.fb.group({ + relation: this.fb.group({ + direction: [EntitySearchDirection.FROM, Validators.required], + relationType: ['Contains', Validators.required], + }), + arguments: this.fb.control({}), + deduplicationIntervalInSec: [], + output: this.fb.control({ + scope: AttributeScope.SERVER_SCOPE, + type: OutputType.Timeseries, + }), + useLatestTs: [false] + }); + + readonly ScriptLanguage = ScriptLanguage; + readonly CalculatedFieldType = CalculatedFieldType; + readonly OutputType = OutputType; + readonly Directions = Object.values(EntitySearchDirection) as Array; + readonly PropagationDirectionTranslations = PropagationDirectionTranslations; + readonly minAllowedDeduplicationIntervalInSecForCF = getCurrentAuthState(this.store).minAllowedDeduplicationIntervalInSecForCF; + + + functionArgs$ = this.relatedAggregationConfiguration.get('arguments').valueChanges.pipe( + map(argumentsObj => ['ctx', ...Object.keys(argumentsObj)]) + ); + + argumentsEditorCompleter$ = this.relatedAggregationConfiguration.get('arguments').valueChanges.pipe( + map(argumentsObj => getCalculatedFieldArgumentsEditorCompleter(argumentsObj ?? {})) + ); + + argumentsHighlightRules$ = this.relatedAggregationConfiguration.get('arguments').valueChanges.pipe( + map(argumentsObj => getCalculatedFieldArgumentsHighlights(argumentsObj)) + ); + + private propagateChange: (config: CalculatedFieldRelatedAggregationConfiguration) => void = () => { }; + + constructor(private fb: FormBuilder, + private store: Store) { + + this.relatedAggregationConfiguration.valueChanges.pipe( + takeUntilDestroyed() + ).subscribe((value: CalculatedFieldRelatedAggregationConfiguration) => { + this.updatedModel(value); + }) + } + + validate(): ValidationErrors | null { + return this.relatedAggregationConfiguration.valid || this.relatedAggregationConfiguration.status === "DISABLED" ? null : {invalidPropagateConfig: false}; + } + + writeValue(value: CalculatedFieldRelatedAggregationConfiguration): void { + this.relatedAggregationConfiguration.patchValue(value, {emitEvent: false}); + setTimeout(() => { + this.relatedAggregationConfiguration.get('arguments').updateValueAndValidity({onlySelf: true}); + }); + } + + registerOnChange(fn: (config: CalculatedFieldRelatedAggregationConfiguration) => void): void { + this.propagateChange = fn; + } + + registerOnTouched(_: any): void { } + + setDisabledState(isDisabled: boolean): void { + if (isDisabled) { + this.relatedAggregationConfiguration.disable({emitEvent: false}); + } else { + this.relatedAggregationConfiguration.enable({emitEvent: false}); + } + } + + fetchOptions(searchText: string): Observable> { + const search = searchText ? searchText?.toLowerCase() : ''; + return of(['Contains', 'Manages']).pipe(map(name => name?.filter(option => option.toLowerCase().includes(search)))); + } + + private updatedModel(value: CalculatedFieldRelatedAggregationConfiguration): void { + value.type = CalculatedFieldType.RELATED_ENTITIES_AGGREGATION; + this.propagateChange(value); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts new file mode 100644 index 0000000000..a2c48c1896 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts @@ -0,0 +1,45 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { SharedModule } from '@shared/shared.module'; +import { + CalculatedFieldOutputModule +} from '@home/components/calculated-fields/components/output/calculated-field-output.module'; +import { + CalculatedFieldArgumentsTableModule +} from '@home/components/calculated-fields/components/calculated-field-arguments/calculated-field-arguments-table.module'; +import { + RelatedEntitiesAggregationComponentComponent +} from '@home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component'; + +@NgModule({ + imports: [ + CommonModule, + SharedModule, + CalculatedFieldOutputModule, + CalculatedFieldArgumentsTableModule, + ], + declarations: [ + RelatedEntitiesAggregationComponentComponent, + ], + exports: [ + RelatedEntitiesAggregationComponentComponent, + ] +}) +export class RelatedEntitiesAggregationComponentModule { +} diff --git a/ui-ngx/src/app/shared/components/time-unit-input.component.html b/ui-ngx/src/app/shared/components/time-unit-input.component.html index d5f6319576..53a444895b 100644 --- a/ui-ngx/src/app/shared/components/time-unit-input.component.html +++ b/ui-ngx/src/app/shared/components/time-unit-input.component.html @@ -18,7 +18,7 @@
+ subscriptSizing="dynamic"> @if (labelText && !inlineField) { {{ labelText }} } @@ -41,7 +41,9 @@ {{ hasError }} - @if (!inlineField) { diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 8294a776df..8ea6659bca 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -65,7 +65,8 @@ export enum CalculatedFieldType { SIMPLE = 'SIMPLE', SCRIPT = 'SCRIPT', GEOFENCING = 'GEOFENCING', - PROPAGATION = 'PROPAGATION' + PROPAGATION = 'PROPAGATION', + RELATED_ENTITIES_AGGREGATION = 'RELATED_ENTITIES_AGGREGATION' } export const CalculatedFieldTypeTranslations = new Map( @@ -74,6 +75,7 @@ export const CalculatedFieldTypeTranslations = new Map; + useLatestTs: boolean; output: CalculatedFieldSimpleOutput; } @@ -105,6 +109,15 @@ export interface CalculatedFieldGeofencingConfiguration { output: CalculatedFieldOutput; } +export interface CalculatedFieldRelatedAggregationConfiguration { + type: CalculatedFieldType.RELATED_ENTITIES_AGGREGATION; + relation: RelationPathLevel; + arguments: Record; + deduplicationIntervalInSec: number; + useLatestTs: boolean; + output: Omit; +} + interface BasePropagationConfiguration { type: CalculatedFieldType.PROPAGATION; direction: EntitySearchDirection; @@ -250,7 +263,7 @@ export interface CalculatedFieldGeofencing { export interface RefDynamicSourceConfiguration { type?: ArgumentEntityType.RelationQuery; - levels?: Array<{direction: EntitySearchDirection; relationType: string;}>; + levels?: Array; } export interface CalculatedFieldGeofencingValue extends CalculatedFieldGeofencing { @@ -317,6 +330,11 @@ export interface CalculatedFieldArgumentValueBase { type: ArgumentType; } +export interface RelationPathLevel { + direction: EntitySearchDirection; + relationType: string; +} + export interface CalculatedFieldAttributeArgumentValue extends CalculatedFieldArgumentValueBase { ts: number; value: ValueType; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 331f3cdb09..0333c5ed2e 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1054,7 +1054,8 @@ "simple": "Simple", "script": "Script", "geofencing" : "Geofencing", - "propagation": "Propagation" + "propagation": "Propagation", + "related-entities-aggregation": "Related entities aggregation" }, "arguments": "Arguments", "decimals-by-default": "Decimals by default", @@ -1088,6 +1089,7 @@ "shared-attributes": "Shared attributes", "attribute-key": "Attribute key", "default-value": "Default value", + "default-value-required": "Default value is required.", "limit": "Max values", "time-window": "Time window", "customer-name": "Customer name", @@ -1156,6 +1158,11 @@ "data-propagate": "Data to propagate", "output-key": "Output key", "copy-output-key": "Copy output key", + "aggregation-path-related-entities": "Aggregation path to related entities", + "metrics": "Metrics", + "deduplication-interval": "Deduplication interval", + "deduplication-interval-min": "Deduplication interval should be at least {{ sec }} second.", + "deduplication-interval-required": "Deduplication interval is required.", "hint": { "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with time series rolling type.", "arguments-propagate-arguments-with-rolling": "'Time series rolling' type is incompatible with 'Arguments only' propagation.", @@ -1202,7 +1209,11 @@ "zone-group-refresh-interval-required": "Zone groups refresh interval is required.", "zone-group-refresh-interval-min": "Zone group refresh interval should be at least {{ min }} second.", "propagation-path-related-entities": "Defines a direct, single-level path to a related entity based on the selected direction and relation type.", - "data-propagate": "Defines the data to be propagated from the arguments configured below. 'Arguments only' uses the retrieved data directly, while 'Expression result' calculates a new value from that data." + "data-propagate": "Defines the data to be propagated from the arguments configured below. 'Arguments only' uses the retrieved data directly, while 'Expression result' calculates a new value from that data.", + "aggregation-path-related-entities": "Defines a single-level aggregation path via direct relations with parent or child entities based on direction and relation type. Only relations between device, asset, customer, and tenant entities are supported.", + "arguments-aggregation": "Defines input parameters used for filtering and aggregation.", + "setting-arguments-aggregation": "Data will be fetched from related entities configured in aggregation path.", + "metrics": "Defines metrics aggregated based on the configured arguments." } }, "ai-models": { From 25cf30d7f1164fb2b1683cc5a59b14c9df2143b9 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 24 Oct 2025 15:28:41 +0300 Subject: [PATCH 74/88] fixed arguments in ctx when the same keys defined --- ...CalculatedFieldEntityMessageProcessor.java | 65 ++++++++++--------- .../cf/ctx/state/CalculatedFieldCtx.java | 19 +++--- .../cf/CalculatedFieldIntegrationTest.java | 50 ++++++++++++++ .../common/data/util/CollectionsUtil.java | 13 ++++ 4 files changed, 110 insertions(+), 37 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index f8b61a082f..c06d0b88c5 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -360,21 +360,25 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return mapToArguments(argNames, data); } - private Map mapToArguments(Map argNames, List data) { - if (argNames.isEmpty()) { + private Map mapToArguments(Map> args, List data) { + if (args.isEmpty()) { return Collections.emptyMap(); } Map arguments = new HashMap<>(); for (TsKvProto item : data) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); - String argName = argNames.get(key); - if (argName != null) { - arguments.put(argName, new SingleValueArgumentEntry(item)); + Set argNames = args.get(key); + if (argNames != null) { + argNames.forEach(argName -> { + arguments.put(argName, new SingleValueArgumentEntry(item)); + }); } key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null); - argName = argNames.get(key); - if (argName != null) { - arguments.put(argName, new SingleValueArgumentEntry(item)); + argNames = args.get(key); + if (argNames != null) { + argNames.forEach(argName -> { + arguments.put(argName, new SingleValueArgumentEntry(item)); + }); } } return arguments; @@ -393,19 +397,21 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return mapToArguments(entityId, argNames, geofencingArgumentNames, scope, attrDataList); } - private Map mapToArguments(EntityId entityId, Map argNames, List geofencingArgNames, AttributeScopeProto scope, List attrDataList) { + private Map mapToArguments(EntityId entityId, Map> args, List geofencingArgNames, AttributeScopeProto scope, List attrDataList) { Map arguments = new HashMap<>(); for (AttributeValueProto item : attrDataList) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); - String argName = argNames.get(key); - if (argName == null) { - continue; - } - if (geofencingArgNames.contains(argName)) { - arguments.put(argName, new GeofencingArgumentEntry(entityId, item)); + Set argNames = args.get(key); + if (argNames == null) { continue; } - arguments.put(argName, new SingleValueArgumentEntry(item)); + argNames.forEach(argName -> { + if (geofencingArgNames.contains(argName)) { + arguments.put(argName, new GeofencingArgumentEntry(entityId, item)); + } else { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } + }); } return arguments; } @@ -423,24 +429,25 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return mapToArgumentsWithDefaultValue(ctx.getMainEntityArguments(), ctx.getArguments(), ctx.getMainEntityGeofencingArgumentNames(), scope, removedAttrKeys); } - private Map mapToArgumentsWithDefaultValue(Map argNames, Map configArguments, List geofencingArgNames, AttributeScopeProto scope, List removedAttrKeys) { + private Map mapToArgumentsWithDefaultValue(Map> args, Map configArguments, List geofencingArgNames, AttributeScopeProto scope, List removedAttrKeys) { Map arguments = new HashMap<>(); for (String removedKey : removedAttrKeys) { ReferencedEntityKey key = new ReferencedEntityKey(removedKey, ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); - String argName = argNames.get(key); - if (argName == null) { + Set argNames = args.get(key); + if (argNames == null) { continue; } - if (geofencingArgNames.contains(argName)) { - arguments.put(argName, new GeofencingArgumentEntry()); - continue; - } - Argument argument = configArguments.get(argName); - String defaultValue = (argument != null) ? argument.getDefaultValue() : null; - arguments.put(argName, StringUtils.isNotEmpty(defaultValue) - ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) - : new SingleValueArgumentEntry()); - + argNames.forEach(argName -> { + if (geofencingArgNames.contains(argName)) { + arguments.put(argName, new GeofencingArgumentEntry()); + } else { + Argument argument = configArguments.get(argName); + String defaultValue = (argument != null) ? argument.getDefaultValue() : null; + arguments.put(argName, StringUtils.isNotEmpty(defaultValue) + ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) + : new SingleValueArgumentEntry()); + } + }); } return arguments; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 650aaaa169..64b09b94ec 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.usagerecord.ApiLimitService; @@ -49,6 +50,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import static org.thingsboard.common.util.ExpressionFunctionsUtil.userDefinedFunctions; @@ -63,8 +65,8 @@ public class CalculatedFieldCtx { private EntityId entityId; private CalculatedFieldType cfType; private final Map arguments; - private final Map mainEntityArguments; - private final Map> linkedEntityArguments; + private final Map> mainEntityArguments; + private final Map>> linkedEntityArguments; private final List argNames; private Output output; private String expression; @@ -110,9 +112,10 @@ public class CalculatedFieldCtx { continue; } if (refId == null || refId.equals(calculatedField.getEntityId())) { - mainEntityArguments.put(refKey, entry.getKey()); + mainEntityArguments.compute(refKey, (key, existingNames) -> CollectionsUtil.addToSet(existingNames, entry.getKey())); } else { - linkedEntityArguments.computeIfAbsent(refId, key -> new HashMap<>()).put(refKey, entry.getKey()); + linkedEntityArguments.computeIfAbsent(refId, key -> new HashMap<>()) + .compute(refKey, (key, existingNames) -> CollectionsUtil.addToSet(existingNames, entry.getKey())); } } this.argNames.addAll(arguments.keySet()); @@ -225,7 +228,7 @@ public class CalculatedFieldCtx { return map != null && matchesTimeSeries(map, values); } - private boolean matchesAttributes(Map argMap, List values, AttributeScope scope) { + private boolean matchesAttributes(Map> argMap, List values, AttributeScope scope) { if (argMap.isEmpty() || values.isEmpty()) { return false; } @@ -239,7 +242,7 @@ public class CalculatedFieldCtx { return false; } - private boolean matchesTimeSeries(Map argMap, List values) { + private boolean matchesTimeSeries(Map> argMap, List values) { if (argMap.isEmpty() || values.isEmpty()) { return false; } @@ -268,7 +271,7 @@ public class CalculatedFieldCtx { return matchesTimeSeriesKeys(mainEntityArguments, keys); } - private boolean matchesAttributesKeys(Map argMap, List keys, AttributeScope scope) { + private boolean matchesAttributesKeys(Map> argMap, List keys, AttributeScope scope) { if (argMap.isEmpty() || keys.isEmpty()) { return false; } @@ -283,7 +286,7 @@ public class CalculatedFieldCtx { return false; } - private boolean matchesTimeSeriesKeys(Map argMap, List keys) { + private boolean matchesTimeSeriesKeys(Map> argMap, List keys) { if (argMap.isEmpty() || keys.isEmpty()) { return false; } diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index 232bcf9445..a805242e85 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -1002,6 +1002,56 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes }); } + @Test + public void testCalculatedFieldWhenTheSameTelemetryKeysUsed() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"a\":5}")); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("a + b"); + calculatedField.setDebugSettings(DebugSettings.all()); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("a", ArgumentType.TS_LATEST, null); + Argument argumentA = new Argument(); + argumentA.setRefEntityKey(refEntityKey); + Argument argumentB = new Argument(); + argumentB.setRefEntityKey(refEntityKey); + config.setArguments(Map.of("a", argumentA, "b", argumentB)); + config.setExpression("a + b"); + + Output output = new Output(); + output.setName("c"); + output.setType(OutputType.TIME_SERIES); + output.setDecimalsByDefault(0); + config.setOutput(output); + + calculatedField.setConfiguration(config); + + doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + await().alias("create CF -> perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode c = getLatestTelemetry(testDevice.getId(), "c"); + assertThat(c).isNotNull(); + assertThat(c.get("c").get(0).get("value").asText()).isEqualTo("10"); + }); + + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"a\":10}")); + + await().alias("update telemetry -> recalculate state").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode c = getLatestTelemetry(testDevice.getId(), "c"); + assertThat(c).isNotNull(); + assertThat(c.get("c").get(0).get("value").asText()).isEqualTo("20"); + }); + } + private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java index 71c5256203..082be9b71f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java @@ -95,4 +95,17 @@ public class CollectionsUtil { return false; } + public static Set addToSet(Set existing, T value) { + if (existing == null || existing.isEmpty()) { + return Set.of(value); + } + if (existing.contains(value)) { + return existing; + } + Set newSet = new HashSet<>(existing.size() + 1); + newSet.addAll(existing); + newSet.add(value); + return (Set) Set.of(newSet.toArray()); + } + } From aab06c465d32a462fd340e349c42f8afbb5ce6ea Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 24 Oct 2025 16:48:34 +0300 Subject: [PATCH 75/88] UI: Add metrics config to RELATED_ENTITIES_AGGREGATION --- .../calculated-field-output.component.html | 25 +- ...culated-field-metrics-panel.component.html | 168 ++++++++++++ ...culated-field-metrics-panel.component.scss | 31 +++ ...alculated-field-metrics-panel.component.ts | 176 +++++++++++++ ...culated-field-metrics-table.component.html | 111 ++++++++ ...culated-field-metrics-table.component.scss | 76 ++++++ ...alculated-field-metrics-table.component.ts | 242 ++++++++++++++++++ ...ities-aggregation-component.component.html | 7 +- ...ntities-aggregation-component.component.ts | 5 +- ...d-entities-aggregation-component.module.ts | 8 + .../shared/models/calculated-field.models.ts | 49 ++++ .../assets/locale/locale.constant-en_US.json | 30 ++- 12 files changed, 909 insertions(+), 19 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.ts diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.html index fcc000bf98..1faeb6adf6 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/output/calculated-field-output.component.html @@ -44,13 +44,7 @@ @if (simpleMode) { @if (hiddenName) {
- - {{ 'calculated-fields.decimals-by-default' | translate }} - - @if (outputForm.get('decimalsByDefault').errors && outputForm.get('decimalsByDefault').touched) { - {{ 'calculated-fields.hint.decimals-range' | translate }} - } - +
} @else { @@ -77,15 +71,18 @@ } - - {{ 'calculated-fields.decimals-by-default' | translate }} - - @if (outputForm.get('decimalsByDefault').errors && outputForm.get('decimalsByDefault').touched) { - {{ 'calculated-fields.hint.decimals-range' | translate }} - } - +
} }
+ + + {{ 'calculated-fields.decimals-by-default' | translate }} + + @if (outputForm.get('decimalsByDefault').errors && outputForm.get('decimalsByDefault').touched) { + {{ 'calculated-fields.hint.decimals-range' | translate }} + } + + diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html new file mode 100644 index 0000000000..5c2985321a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html @@ -0,0 +1,168 @@ + +
+
+
{{ 'calculated-fields.metrics.metric-settings' | translate }}
+
+
+
{{ 'calculated-fields.metrics.metric-name' | translate }}
+ + + @if (metricForm.get('name').touched && metricForm.get('name').hasError('required')) { + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('duplicateName')) { + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('pattern')) { + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('maxlength')) { + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('forbiddenName')) { + + warning + + } + +
+
+
{{ 'calculated-fields.metrics.aggregation' | translate }}
+ + + @for (aggFunction of AggFunctions; track aggFunction) { + {{ AggFunctionTranslations.get(aggFunction) | translate }} + } + + +
+ +
+ + + + +
+ {{ 'calculated-fields.metrics.filter' | translate }} +
+ + + + + +
{{ 'api-usage.tbel' | translate }} +
+ + + +
+ +
+
{{ 'calculated-fields.metrics.value-source' | translate }}
+ + + @for (inputType of AggInputTypes; track inputType) { + {{ AggInputTypeTranslations.get(inputType) | translate }} + } + + +
+ @if (this.metricForm.get('input.type').value === AggInputType.key) { +
+
{{ 'calculated-fields.argument-name' | translate }}
+ + +
+ } @else { + +
{{ 'api-usage.tbel' | translate }} +
+ + } + +
+
+
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.scss new file mode 100644 index 0000000000..ae692b6f93 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.scss @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import '../../../../../../../scss/constants'; + +$panel-width: 520px; + +:host { + display: flex; + width: $panel-width; + max-width: 100%; + max-height: 80vh; + + .fixed-title-width { + @media #{$mat-xs} { + min-width: 120px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.ts new file mode 100644 index 0000000000..6fb296a161 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.ts @@ -0,0 +1,176 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit, output } from '@angular/core'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { FormBuilder, FormControl, ValidatorFn, Validators } from '@angular/forms'; +import { charsWithNumRegex } from '@shared/models/regex.constants'; +import { + AggFunction, + AggFunctionTranslations, + AggInputType, + AggInputTypeTranslations, + CalculatedFieldAggMetricValue +} from '@shared/models/calculated-field.models'; +import { delay, map } from 'rxjs/operators'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { EntityFilter } from '@shared/models/query/query.models'; +import { merge, Observable, of } from 'rxjs'; +import { ScriptLanguage } from '@shared/models/rule-node.models'; +import { TbEditorCompleter } from '@shared/models/ace/completion.models'; +import { AceHighlightRules } from '@shared/models/ace/ace.models'; + +interface CalculatedFieldAggMetricValuePanel extends CalculatedFieldAggMetricValue { + allowFilter: boolean; +} + +@Component({ + selector: 'tb-calculated-field-metrics-panel', + templateUrl: './calculated-field-metrics-panel.component.html', + styleUrls: ['./calculated-field-metrics-panel.component.scss'] +}) +export class CalculatedFieldMetricsPanelComponent implements OnInit { + + @Input() buttonTitle: string; + @Input() metric: CalculatedFieldAggMetricValue; + @Input() usedNames: string[]; + @Input() arguments: Array; + @Input() editorCompleter: TbEditorCompleter; + @Input() highlightRules: AceHighlightRules; + + metricDataApplied = output(); + filterExpanded = false; + functionArgs: Array + + metricForm = this.fb.group({ + name: ['', [Validators.required, this.uniqNameRequired(), this.forbiddenNameValidator(), Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], + function: [AggFunction.AVG], + allowFilter: [false], + filter: ['', Validators.required], + input: this.fb.group({ + type: [AggInputType.key], + key: ['', Validators.required], + function: ['', Validators.required], + }) + }); + + entityFilter: EntityFilter; + + readonly AggFunctions = Object.values(AggFunction) as AggFunction[]; + readonly AggFunctionTranslations = AggFunctionTranslations; + readonly ScriptLanguage = ScriptLanguage; + readonly AggInputType = AggInputType; + readonly AggInputTypes = Object.values(AggInputType) as AggInputType[]; + readonly AggInputTypeTranslations = AggInputTypeTranslations; + + constructor( + private fb: FormBuilder, + private popover: TbPopoverComponent + ) { + this.observeUpdatePosition(); + this.observeFilterAllowChange(); + this.observeInputTypeChange(); + } + + ngOnInit(): void { + const data: CalculatedFieldAggMetricValuePanel = { + ...this.metric, + allowFilter: !!this.metric.filter, + } + this.metricForm.patchValue(data, {emitEvent: false}); + + this.validateFilter(data.allowFilter); + this.validateInputTypeFilter(data.input?.type ?? AggInputType.key); + + this.functionArgs = ['ctx', ...this.arguments]; + } + + fetchOptions(searchText: string): Observable> { + const search = searchText ? searchText?.toLowerCase() : ''; + return of(this.arguments).pipe(map(name => name?.filter(option => option.toLowerCase().includes(search)))); + } + + private observeFilterAllowChange(): void { + this.metricForm.get('allowFilter').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(value => this.validateFilter(value)); + } + + private observeInputTypeChange(): void { + this.metricForm.get('input.type').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(value => this.validateInputTypeFilter(value)); + } + + private validateFilter(allowFilter = false): void { + if (allowFilter) { + this.metricForm.get('filter').enable({emitEvent: false}); + } else { + this.metricForm.get('filter').disable({emitEvent: false}); + } + this.filterExpanded = allowFilter; + } + + private validateInputTypeFilter(value: AggInputType): void { + const inputForm = this.metricForm.get('input'); + if (value === AggInputType.key) { + inputForm.get('key').enable({emitEvent: false}); + inputForm.get('function').disable({emitEvent: false}); + } else { + inputForm.get('key').disable({emitEvent: false}); + inputForm.get('function').enable({emitEvent: false}); + } + } + + saveZone(): void { + const value = this.metricForm.value as CalculatedFieldAggMetricValuePanel; + if (!value.allowFilter) { + delete value.filter; + } + delete value.allowFilter; + this.metricDataApplied.emit(value); + } + + cancel(): void { + this.popover.hide(); + } + + private uniqNameRequired(): ValidatorFn { + return (control: FormControl) => { + const newName = control.value.trim().toLowerCase(); + const isDuplicate = this.usedNames?.some(name => name.toLowerCase() === newName); + + return isDuplicate ? { duplicateName: true } : null; + }; + } + + private forbiddenNameValidator(): ValidatorFn { + return (control: FormControl) => { + const trimmedValue = control.value.trim().toLowerCase(); + const forbiddenNames = ['ctx', 'e', 'pi']; + return forbiddenNames.includes(trimmedValue) ? { forbiddenName: true } : null; + }; + } + + private observeUpdatePosition(): void { + merge( + this.metricForm.get('allowFilter').valueChanges, + this.metricForm.get('input.type').valueChanges + ) + .pipe(delay(50), takeUntilDestroyed()) + .subscribe(() => this.popover.updatePosition()); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.html new file mode 100644 index 0000000000..481ad9ee13 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.html @@ -0,0 +1,111 @@ + +
+
+ + + +
{{ 'calculated-fields.metrics.metric-name' | translate }}
+ + +
+
{{ metric.name }}
+ +
+ + + + + {{ 'calculated-fields.metrics.aggregation' | translate }} + + +
{{ AggFunctionTranslations.get(metric.function) | translate }}
+ + + + + {{ 'calculated-fields.metrics.filtered' | translate }} + + + {{ metric.filter ? 'check_box' : 'check_box_outline_blank' }} + + + + + {{ 'calculated-fields.metrics.value-source' | translate }} + + +
{{ AggInputTypeTranslations.get(metric.input.type) | translate }}
+ + + + + + +
+ + +
+ + + + +
+
+ {{ 'calculated-fields.metrics.no-metrics-configured' | translate }} +
+ @if (errorText) { + + } +
+
+ + @if (maxArgumentsPerCF && metricsFormArray.length >= maxArgumentsPerCF) { +
+ warning + {{ 'calculated-fields.metrics.max-metrics' | translate }} +
+ } +
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.scss new file mode 100644 index 0000000000..430958d0f4 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.scss @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .arguments-table { + min-height: 108px; + + &-with-error { + min-height: 150px; + } + + .mat-mdc-table { + table-layout: fixed; + } + + .key-text { + font-size: 13px; + } + + .copy-argument-name { + visibility: hidden; + transition: visibility 0.1s; + } + + .argument-name-cell:hover { + .copy-argument-name { + visibility: visible; + } + } + } + + .max-args-warning { + .mat-icon { + color: #FAA405; + } + } + + .tb-form-table-row-cell-buttons { + --mat-badge-legacy-small-size-container-size: 8px; + --mat-badge-small-size-container-overlap-offset: -5px; + --mat-badge-small-size-text-size: 0; + } +} + +:host ::ng-deep { + .arguments-table:not(.arguments-table-with-error) { + .mdc-data-table__row:last-child .mat-mdc-cell { + border-bottom: none; + } + } + + .arguments-table { + .mat-mdc-header-row.mat-row-select .mat-mdc-header-cell.entity-type-header { + padding: 0 28px 0 0; + } + } + + .copy-argument-name { + .mat-icon { + font-size: 16px; + padding: 4px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.ts new file mode 100644 index 0000000000..2e4ac30165 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.ts @@ -0,0 +1,242 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + AfterViewInit, + ChangeDetectorRef, + Component, + DestroyRef, + forwardRef, + Input, + Renderer2, + ViewChild, + ViewContainerRef, +} from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, +} from '@angular/forms'; +import { + AggFunctionTranslations, + AggInputTypeTranslations, + CalculatedFieldAggMetric, + CalculatedFieldAggMetricValue, +} from '@shared/models/calculated-field.models'; +import { MatButton } from '@angular/material/button'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { isDefinedAndNotNull, isEqual } from '@core/utils'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { TbTableDatasource } from '@shared/components/table/table-datasource.abstract'; +import { MatSort, SortDirection } from '@angular/material/sort'; +import { getCurrentAuthState } from '@core/auth/auth.selectors'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { + CalculatedFieldMetricsPanelComponent +} from '@home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component'; +import { TbEditorCompleter } from '@shared/models/ace/completion.models'; +import { AceHighlightRules } from '@shared/models/ace/ace.models'; + +@Component({ + selector: 'tb-calculated-field-metrics-table', + templateUrl: './calculated-field-metrics-table.component.html', + styleUrls: [`calculated-field-metrics-table.component.scss`], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => CalculatedFieldMetricsTableComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => CalculatedFieldMetricsTableComponent), + multi: true + } + ], +}) +export class CalculatedFieldMetricsTableComponent implements ControlValueAccessor, Validator, AfterViewInit { + + @Input() arguments: Array; + @Input() editorCompleter: TbEditorCompleter; + @Input() highlightRules: AceHighlightRules; + + @ViewChild(MatSort, { static: true }) sort: MatSort; + + errorText = ''; + metricsFormArray = this.fb.array([]); + sortOrder = { direction: 'asc' as SortDirection, property: '' }; + dataSource = new CalculatedFieldMetricsDatasource(); + + displayColumns = ['name', 'function', 'filter', 'valueSource', 'actions'] + + readonly AggFunctionTranslations = AggFunctionTranslations; + readonly AggInputTypeTranslations = AggInputTypeTranslations; + readonly maxArgumentsPerCF = getCurrentAuthState(this.store).maxArgumentsPerCF - 2; + + private popoverComponent: TbPopoverComponent; + private propagateChange: (zonesObj: Record) => void = () => {}; + + constructor( + private fb: FormBuilder, + private popoverService: TbPopoverService, + private viewContainerRef: ViewContainerRef, + private cd: ChangeDetectorRef, + private renderer: Renderer2, + private destroyRef: DestroyRef, + private store: Store + ) { + this.metricsFormArray.valueChanges.pipe(takeUntilDestroyed()).subscribe(value => { + this.updateDataSource(value); + this.propagateChange(this.getMetricsObject(value)); + }); + } + + ngAfterViewInit(): void { + this.sort.sortChange.asObservable().pipe( + takeUntilDestroyed(this.destroyRef) + ).subscribe(() => { + this.sortOrder.property = this.sort.active; + this.sortOrder.direction = this.sort.direction; + this.updateDataSource(this.metricsFormArray.value); + }); + } + + registerOnChange(fn: (zonesObj: Record) => void): void { + this.propagateChange = fn; + } + + registerOnTouched(_fn: any): void {} + + validate(): ValidationErrors | null { + this.updateErrorText(); + return this.errorText ? { metricsFormArray: false } : null; + } + + onDelete($event: Event, metric: CalculatedFieldAggMetricValue): void { + $event.stopPropagation(); + const index = this.metricsFormArray.controls.findIndex(control => isEqual(control.value, metric)); + this.metricsFormArray.removeAt(index); + this.metricsFormArray.markAsDirty(); + } + + manageMetrics($event: Event, matButton: MatButton, metric = {} as CalculatedFieldAggMetricValue): void { + $event?.stopPropagation(); + if (this.popoverComponent && !this.popoverComponent.tbHidden) { + this.popoverComponent.hide(); + } + const trigger = matButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const index = this.metricsFormArray.controls.findIndex(control => isEqual(control.value, metric)); + const isExists = index !== -1; + const ctx = { + index, + metric, + buttonTitle: isExists ? 'action.apply' : 'action.add', + usedNames: this.metricsFormArray.value.map(({ name }) => name).filter(name => name !== metric.name), + arguments: this.arguments, + editorCompleter: this.editorCompleter, + highlightRules: this.highlightRules + }; + this.popoverComponent = this.popoverService.displayPopover({ + trigger, + renderer: this.renderer, + componentType: CalculatedFieldMetricsPanelComponent, + hostView: this.viewContainerRef, + preferredPlacement: isExists ? ['leftOnly', 'leftTopOnly', 'leftBottomOnly'] : ['rightOnly', 'rightTopOnly', 'rightBottomOnly'], + context: ctx, + isModal: true + }); + this.popoverComponent.tbComponentRef.instance.metricDataApplied.subscribe((value) => { + this.popoverComponent.hide(); + if (isExists) { + this.metricsFormArray.at(index).setValue(value); + } else { + this.metricsFormArray.push(this.fb.control(value)); + } + this.cd.markForCheck(); + }); + } + } + + private updateDataSource(value: CalculatedFieldAggMetricValue[]): void { + const sortedValue = this.sortData(value); + this.dataSource.loadData(sortedValue); + } + + private updateErrorText(): void { + if (!this.metricsFormArray.controls.length) { + this.errorText = 'calculated-fields.metrics.metrics-empty'; + } else { + this.errorText = ''; + } + } + + private getMetricsObject(value: CalculatedFieldAggMetricValue[]): Record { + return value.reduce((acc, metricValue) => { + const { name, ...metric } = metricValue; + acc[name] = metric; + return acc; + }, {} as Record); + } + + writeValue(metrics: Record): void { + this.metricsFormArray.clear(); + this.populateZonesFormArray(metrics); + } + + private populateZonesFormArray(metrics: Record): void { + Object.keys(metrics).forEach(key => { + const value: CalculatedFieldAggMetricValue = { + ...metrics[key], + name: key + }; + this.metricsFormArray.push(this.fb.control(value), { emitEvent: false }); + }); + this.metricsFormArray.updateValueAndValidity(); + } + + private getSortValue(metric: CalculatedFieldAggMetricValue, column: string): string { + switch (column) { + case 'function': + return metric.function; + case 'filter': + return isDefinedAndNotNull(metric.filter).toString(); + default: + return metric.name; + } + } + + private sortData(data: CalculatedFieldAggMetricValue[]): CalculatedFieldAggMetricValue[] { + return data.sort((a, b) => { + const valA = this.getSortValue(a, this.sortOrder.property) ?? ''; + const valB = this.getSortValue(b, this.sortOrder.property) ?? ''; + return (this.sortOrder.direction === 'asc' ? 1 : -1) * valA.localeCompare(valB); + }); + } +} + +class CalculatedFieldMetricsDatasource extends TbTableDatasource { + constructor() { + super(); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html index 1988141c2a..ce2d22fbbc 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component.html @@ -51,8 +51,13 @@
- {{ 'calculated-fields.metrics' | translate }} + {{ 'calculated-fields.metrics.metrics' | translate }}
+ ({ scope: AttributeScope.SERVER_SCOPE, @@ -93,8 +94,8 @@ export class RelatedEntitiesAggregationComponentComponent implements ControlValu readonly minAllowedDeduplicationIntervalInSecForCF = getCurrentAuthState(this.store).minAllowedDeduplicationIntervalInSecForCF; - functionArgs$ = this.relatedAggregationConfiguration.get('arguments').valueChanges.pipe( - map(argumentsObj => ['ctx', ...Object.keys(argumentsObj)]) + arguments$ = this.relatedAggregationConfiguration.get('arguments').valueChanges.pipe( + map(argumentsObj => Object.keys(argumentsObj)) ); argumentsEditorCompleter$ = this.relatedAggregationConfiguration.get('arguments').valueChanges.pipe( diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts index a2c48c1896..b272f1e965 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.module.ts @@ -26,6 +26,12 @@ import { import { RelatedEntitiesAggregationComponentComponent } from '@home/components/calculated-fields/components/related-entities-aggregation-configuration/related-entities-aggregation-component.component'; +import { + CalculatedFieldMetricsTableComponent +} from '@home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component'; +import { + CalculatedFieldMetricsPanelComponent +} from '@home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component'; @NgModule({ imports: [ @@ -36,6 +42,8 @@ import { ], declarations: [ RelatedEntitiesAggregationComponentComponent, + CalculatedFieldMetricsTableComponent, + CalculatedFieldMetricsPanelComponent ], exports: [ RelatedEntitiesAggregationComponentComponent, diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 8ea6659bca..377b443fb6 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -113,6 +113,7 @@ export interface CalculatedFieldRelatedAggregationConfiguration { type: CalculatedFieldType.RELATED_ENTITIES_AGGREGATION; relation: RelationPathLevel; arguments: Record; + metrics: Record; deduplicationIntervalInSec: number; useLatestTs: boolean; output: Omit; @@ -251,6 +252,54 @@ export interface CalculatedFieldArgument { timeWindow?: number; } +export enum AggFunction { + AVG='AVG', + MIN='MIN', + MAX='MAX', + SUM='SUM', + COUNT='COUNT', + COUNT_UNIQUE='COUNT_UNIQUE' +} + +export const AggFunctionTranslations = new Map([ + [AggFunction.AVG, 'calculated-fields.metrics.aggregation-type.avg'], + [AggFunction.MIN, 'calculated-fields.metrics.aggregation-type.min'], + [AggFunction.MAX, 'calculated-fields.metrics.aggregation-type.max'], + [AggFunction.SUM, 'calculated-fields.metrics.aggregation-type.sum'], + [AggFunction.COUNT, 'calculated-fields.metrics.aggregation-type.count'], + [AggFunction.COUNT_UNIQUE, 'calculated-fields.metrics.aggregation-type.count-unique'], +]) + +export interface CalculatedFieldAggMetric { + function: AggFunction; + filter?: string; + input: AggKeyInput | AggFunctionInput; +} + +export interface CalculatedFieldAggMetricValue extends CalculatedFieldAggMetric { + name: string; +} + +export enum AggInputType { + key = 'key', + function = 'function' +} + +export const AggInputTypeTranslations = new Map([ + [AggInputType.key, 'calculated-fields.metrics.value-source-type.key'], + [AggInputType.function, 'calculated-fields.metrics.value-source-type.function'], +]) + +export interface AggKeyInput { + type: AggInputType.key; + key: string; +} + +export interface AggFunctionInput { + type: AggInputType.function; + function: string; +} + export interface CalculatedFieldGeofencing { perimeterKeyName: string; reportStrategy: GeofencingReportStrategy; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 0333c5ed2e..15207999f6 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1159,10 +1159,36 @@ "output-key": "Output key", "copy-output-key": "Copy output key", "aggregation-path-related-entities": "Aggregation path to related entities", - "metrics": "Metrics", "deduplication-interval": "Deduplication interval", "deduplication-interval-min": "Deduplication interval should be at least {{ sec }} second.", "deduplication-interval-required": "Deduplication interval is required.", + "metrics": { + "metrics": "Metrics", + "metrics-empty": "At least one metric must be configured.", + "metric-name": "Metric name", + "copy-metric-name": "Copy metric name", + "aggregation": "Aggregation", + "aggregation-type": { + "avg": "Average", + "min": "Minimum", + "max": "Maximum", + "sum": "Sum", + "count": "Count", + "count-unique": "Count unique" + }, + "filtered": "Filtered", + "value-source": "Value source", + "value-source-type": { + "key": "Key", + "function": "Function" + }, + "no-metrics-configured": "No metrics configured", + "add-metric": "Add metric", + "max-metrics": "Maximum number of metrics reached.", + "metric-settings": "Metric settings", + "filter": "Filter", + "filter-hint": "Enables filtering of entities during aggregation. The filter function must return a boolean value and can use all configured arguments." + }, "hint": { "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with time series rolling type.", "arguments-propagate-arguments-with-rolling": "'Time series rolling' type is incompatible with 'Arguments only' propagation.", @@ -1182,7 +1208,7 @@ "output-key-max-length": "Output key should be less than 256 characters.", "output-key-forbidden": "Output key is reserved and cannot be used.", "entity-type-required": "Entity type is required", - "name-required": "Mame is required.", + "name-required": "Name is required.", "name-pattern": "Name is invalid.", "name-duplicate": "Name with such name already exists.", "name-max-length": "Name should be less than 256 characters.", From 522369383b94d2b6b46cd3c377c6e28bca83022b Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 24 Oct 2025 19:31:06 +0300 Subject: [PATCH 76/88] UI: Improvement popover in cf; Simplify style --- .../calculated-fields-table-config.ts | 4 +- ...ulated-field-argument-panel.component.html | 8 +- ...ulated-field-argument-panel.component.scss | 19 +- ...lculated-field-argument-panel.component.ts | 19 +- .../common/calculated-field-panel.scss | 58 +++ ...eofencing-zone-groups-panel.component.html | 421 +++++++++--------- ...eofencing-zone-groups-panel.component.scss | 20 +- ...-geofencing-zone-groups-panel.component.ts | 33 +- ...eofencing-zone-groups-table.component.html | 2 +- ...eofencing-zone-groups-table.component.scss | 76 ---- ...-geofencing-zone-groups-table.component.ts | 6 +- .../calculated-field-output.component.html | 2 +- ...culated-field-metrics-panel.component.html | 220 ++++----- ...alculated-field-metrics-panel.component.ts | 16 +- ...culated-field-metrics-table.component.html | 20 +- ...culated-field-metrics-table.component.scss | 76 ---- ...alculated-field-metrics-table.component.ts | 4 +- ...ities-aggregation-component.component.html | 4 +- ...ties-aggregation-component.component.scss} | 21 +- ...ntities-aggregation-component.component.ts | 1 + 20 files changed, 431 insertions(+), 599 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/common/calculated-field-panel.scss delete mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/geofencing-configuration/calculated-field-geofencing-zone-groups-table.component.scss delete mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-table.component.scss rename ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/{calculated-field-metrics-panel.component.scss => related-entities-aggregation-component.component.scss} (73%) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 4104b97221..1b48771247 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -111,7 +111,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig('expression', 'calculated-fields.expression', '300px'); + const expressionColumn = new EntityTableColumn('expression', 'calculated-fields.expression', '250px'); expressionColumn.sortable = false; expressionColumn.cellContentFunction = entity => { const expressionLabel = this.getExpressionLabel(entity); @@ -124,7 +124,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig('createdTime', 'common.created-time', this.datePipe, '150px')); this.columns.push(new EntityTableColumn('name', 'common.name', '33%')); - this.columns.push(new EntityTableColumn('type', 'common.type', '80px', entity => this.translate.instant(CalculatedFieldTypeTranslations.get(entity.type)))); + this.columns.push(new EntityTableColumn('type', 'common.type', '170px', entity => this.translate.instant(CalculatedFieldTypeTranslations.get(entity.type)), () => ({whiteSpace: 'nowrap' }))); this.columns.push(expressionColumn); this.cellActionDescriptors.push( diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html index 607b107094..1ffa7ccf77 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/calculated-field-arguments/calculated-field-argument-panel.component.html @@ -15,9 +15,9 @@ limitations under the License. --> -
-
-
{{ 'calculated-fields.argument-settings' | translate }}
+
+
{{ 'calculated-fields.metrics.metric-settings' | translate }}
+
@if (hint) {
{{ hint | translate }} @@ -190,7 +190,7 @@ }
-
+
-
-
-
{{ $index+1 }}
- - - @for (direction of GeofencingDirectionList; track direction) { - {{ GeofencingDirectionLevelTranslations.get(direction) | translate }} - } - - - - -
-
- -
-
- } -
- } @else { - {{ 'calculated-fields.no-level' | translate }} - } - @if (levelsFormArray().errors) { - - } -
-
- @if (maxRelationLevelPerCfArgument && levelsFormArray().length >= maxRelationLevelPerCfArgument) { -
- warning - {{ 'calculated-fields.max-allowed-levels-error' | translate }} -
- } @else { - - } -
- +
{{ ArgumentEntityTypeParamsMap.get(entityType).title | translate }}
+
- - - @if (entityFilter.singleEntity?.id) { -
-
- {{ 'calculated-fields.perimeter-attribute-key' | translate }} + } + + +
+ + {{ 'calculated-fields.entity-zone-relationship' | translate }} +
+
+
+
calculated-fields.level
+
calculated-fields.direction-level
+
calculated-fields.relation-type
+
- @if (entityType === ArgumentEntityType.RelationQuery) { - - - @if (geofencingFormGroup.get('perimeterKeyName').touched && geofencingFormGroup.get('perimeterKeyName').hasError('required')) { - - warning - - } @else if (geofencingFormGroup.get('perimeterKeyName').touched && geofencingFormGroup.get('perimeterKeyName').hasError('pattern')) { - - warning - + @if (levelsFormArray()?.controls?.length) { +
+ @for (keyControl of levelsFormArray().controls; track trackByKey; ) { +
+
+ +
+
+
{{ $index + 1 }}
+ + + @for (direction of GeofencingDirectionList; track direction) { + {{ GeofencingDirectionLevelTranslations.get(direction) | translate }} + } + + + + +
+
+ +
+
} - +
} @else { - + {{ 'calculated-fields.no-level' | translate }} + } + @if (levelsFormArray().errors) { + }
- } +
+ @if (maxRelationLevelPerCfArgument && levelsFormArray().length >= maxRelationLevelPerCfArgument) { +
+ warning + {{ 'calculated-fields.max-allowed-levels-error' | translate }} +
+ } @else { + + } +
+ +
+ + + @if (entityFilter.singleEntity?.id) {
-
{{ 'calculated-fields.report-strategy' | translate }}
- - - @for (strategy of GeofencingReportStrategyList; track strategy) { - {{ GeofencingReportStrategyTranslations.get(strategy) | translate }} - } - - -
- -
- -
- {{ 'calculated-fields.create-relation-with-matched-zones' | translate }} +
+ {{ 'calculated-fields.perimeter-attribute-key' | translate }}
- -
-
{{ 'calculated-fields.direction' | translate }}
- - - @for (direction of GeofencingDirectionList; track direction) { - {{ GeofencingDirectionTranslations.get(direction) | translate }} + @if (entityType === ArgumentEntityType.RelationQuery) { + + + @if (geofencingFormGroup.get('perimeterKeyName').touched && geofencingFormGroup.get('perimeterKeyName').hasError('required')) { + + warning + + } @else if (geofencingFormGroup.get('perimeterKeyName').touched && geofencingFormGroup.get('perimeterKeyName').hasError('pattern')) { + + warning + } - - + + } @else { + + }
-
-
{{ 'calculated-fields.relation-type' | translate }}
- - + } +
+
{{ 'calculated-fields.report-strategy' | translate }}
+ + + @for (strategy of GeofencingReportStrategyList; track strategy) { + {{ GeofencingReportStrategyTranslations.get(strategy) | translate }} + } + + +
+ +
+ +
+ {{ 'calculated-fields.create-relation-with-matched-zones' | translate }}
+ +
+
{{ 'calculated-fields.direction' | translate }}
+ + + @for (direction of GeofencingDirectionList; track direction) { + {{ GeofencingDirectionTranslations.get(direction) | translate }} + } + + +
+
+
{{ 'calculated-fields.relation-type' | translate }}
+ +
-
+
@if (simpleMode) { @if (hiddenName) { -
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html index 5c2985321a..b7c58f7633 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/related-entities-aggregation-configuration/calculated-field-metrics-panel.component.html @@ -15,23 +15,23 @@ limitations under the License. --> -
-
-
{{ 'calculated-fields.metrics.metric-settings' | translate }}
-
-
-
{{ 'calculated-fields.metrics.metric-name' | translate }}
- - - @if (metricForm.get('name').touched && metricForm.get('name').hasError('required')) { - - warning - - } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('duplicateName')) { +
+
{{ 'calculated-fields.metrics.metric-settings' | translate }}
+
+
+
{{ 'calculated-fields.metrics.metric-name' | translate }}
+ + + @if (metricForm.get('name').touched && metricForm.get('name').hasError('required')) { + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('duplicateName')) { } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('pattern')) { - - warning - - } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('maxlength')) { - - warning - - } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('forbiddenName')) { - - warning - + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('maxlength')) { + + warning + + } @else if (metricForm.get('name').touched && metricForm.get('name').hasError('forbiddenName')) { + + warning + + } + +
+
+
{{ 'calculated-fields.metrics.aggregation' | translate }}
+ + + @for (aggFunction of AggFunctions; track aggFunction) { + {{ AggFunctionTranslations.get(aggFunction) | translate }} } - -
-
-
{{ 'calculated-fields.metrics.aggregation' | translate }}
- - - @for (aggFunction of AggFunctions; track aggFunction) { - {{ AggFunctionTranslations.get(aggFunction) | translate }} - } - - -
+ + +
-
- - - - -
- {{ 'calculated-fields.metrics.filter' | translate }} -
- - - - - -
{{ 'api-usage.tbel' | translate }} +
+ + + + +
+ {{ 'calculated-fields.metrics.filter' | translate }}
- - - -
- -
-
{{ 'calculated-fields.metrics.value-source' | translate }}
- - - @for (inputType of AggInputTypes; track inputType) { - {{ AggInputTypeTranslations.get(inputType) | translate }} - } - - -
- @if (this.metricForm.get('input.type').value === AggInputType.key) { -
-
{{ 'calculated-fields.argument-name' | translate }}
- - -
- } @else { + + + + {{ 'api-usage.tbel' | translate }}
- } - + +
+ +
+
{{ 'calculated-fields.metrics.value-source' | translate }}
+ + + @for (inputType of AggInputTypes; track inputType) { + {{ AggInputTypeTranslations.get(inputType) | translate }} + } + + +
+ @if (this.metricForm.get('input.type').value === AggInputType.key) { +
+
{{ 'calculated-fields.argument-name' | translate }}
+ + +
+ } @else { + +
{{ 'api-usage.tbel' | translate }} +
+ + } +
-
+