From 8c94ec5c5277610c1dd494c75c1c677898885210 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Mon, 16 Oct 2023 18:05:12 +0300 Subject: [PATCH 01/29] Edge: do not sync User login actions. Edge: do not save particular edge events in case edge is not activated or disconnected --- .../edge/EdgeEventSourcingListener.java | 42 +++++++++++--- .../edge/rpc/processor/BaseEdgeProcessor.java | 58 +++++++++++++++++++ .../server/edge/AbstractEdgeTest.java | 49 +++++++--------- .../server/edge/DeviceEdgeTest.java | 2 - .../thingsboard/server/edge/UserEdgeTest.java | 8 --- .../dao/eventsourcing/SaveEntityEvent.java | 1 + .../server/dao/user/UserServiceImpl.java | 7 ++- 7 files changed, 119 insertions(+), 48 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java index 43b05094a4..707dbec964 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.service.edge; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @@ -36,6 +38,7 @@ import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.RelationActionEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; +import org.thingsboard.server.dao.user.UserServiceImpl; import javax.annotation.PostConstruct; @@ -75,7 +78,7 @@ public class EdgeEventSourcingListener { return; } try { - if (!isValidEdgeEventEntity(event.getEntity())) { + if (!isValidEdgeEventEntity(event.getEntity(), event.getOldEntity())) { return; } log.trace("[{}] SaveEntityEvent called: {}", event.getTenantId(), event); @@ -83,7 +86,7 @@ public class EdgeEventSourcingListener { tbClusterService.sendNotificationMsgToEdge(event.getTenantId(), null, event.getEntityId(), null, null, action); } catch (Exception e) { - log.error("[{}] failed to process SaveEntityEvent: {}", event.getTenantId(), event); + log.error("[{}] failed to process SaveEntityEvent: {}", event.getTenantId(), event, e); } } @@ -97,7 +100,7 @@ public class EdgeEventSourcingListener { tbClusterService.sendNotificationMsgToEdge(event.getTenantId(), event.getEdgeId(), event.getEntityId(), JacksonUtil.toString(event.getEntity()), null, EdgeEventActionType.DELETED); } catch (Exception e) { - log.error("[{}] failed to process DeleteEntityEvent: {}", event.getTenantId(), event); + log.error("[{}] failed to process DeleteEntityEvent: {}", event.getTenantId(), event, e); } } @@ -111,7 +114,7 @@ public class EdgeEventSourcingListener { tbClusterService.sendNotificationMsgToEdge(event.getTenantId(), event.getEdgeId(), event.getEntityId(), event.getBody(), null, edgeTypeByActionType(event.getActionType())); } catch (Exception e) { - log.error("[{}] failed to process ActionEntityEvent: {}", event.getTenantId(), event); + log.error("[{}] failed to process ActionEntityEvent: {}", event.getTenantId(), event, e); } } @@ -134,11 +137,11 @@ public class EdgeEventSourcingListener { tbClusterService.sendNotificationMsgToEdge(event.getTenantId(), null, null, JacksonUtil.toString(relation), EdgeEventType.RELATION, edgeTypeByActionType(event.getActionType())); } catch (Exception e) { - log.error("[{}] failed to process RelationActionEvent: {}", event.getTenantId(), event); + log.error("[{}] failed to process RelationActionEvent: {}", event.getTenantId(), event, e); } } - private boolean isValidEdgeEventEntity(Object entity) { + private boolean isValidEdgeEventEntity(Object entity, Object oldEntity) { if (entity instanceof OtaPackageInfo) { OtaPackageInfo otaPackageInfo = (OtaPackageInfo) entity; return otaPackageInfo.hasUrl() || otaPackageInfo.isHasData(); @@ -147,7 +150,15 @@ public class EdgeEventSourcingListener { return RuleChainType.EDGE.equals(ruleChain.getType()); } else if (entity instanceof User) { User user = (User) entity; - return !Authority.SYS_ADMIN.equals(user.getAuthority()); + if (Authority.SYS_ADMIN.equals(user.getAuthority())) { + return false; + } + if (oldEntity != null) { + User oldUser = (User) oldEntity; + cleanUpUserAdditionalInfo(oldUser); + cleanUpUserAdditionalInfo(user); + return !user.equals(oldUser); + } } else if (entity instanceof AlarmApiCallResult) { AlarmApiCallResult alarmApiCallResult = (AlarmApiCallResult) entity; return alarmApiCallResult.isModified(); @@ -155,4 +166,21 @@ public class EdgeEventSourcingListener { // Default: If the entity doesn't match any of the conditions, consider it as valid. return true; } + + private void cleanUpUserAdditionalInfo(User user) { + // reset FAILED_LOGIN_ATTEMPTS and LAST_LOGIN_TS - edge is not interested in this information + if (user.getAdditionalInfo() instanceof NullNode) { + user.setAdditionalInfo(null); + } + if (user.getAdditionalInfo() instanceof ObjectNode) { + ObjectNode additionalInfo = ((ObjectNode) user.getAdditionalInfo()); + additionalInfo.remove(UserServiceImpl.FAILED_LOGIN_ATTEMPTS); + additionalInfo.remove(UserServiceImpl.LAST_LOGIN_TS); + if (additionalInfo.isEmpty()) { + user.setAdditionalInfo(null); + } else { + user.setAdditionalInfo(additionalInfo); + } + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index 328a64efe2..2d86c1e184 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -24,6 +24,7 @@ import org.springframework.context.annotation.Lazy; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EdgeUtils; @@ -46,6 +47,7 @@ import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -110,11 +112,13 @@ import org.thingsboard.server.service.entitiy.TbNotificationEntityService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import org.thingsboard.server.service.state.DefaultDeviceStateService; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -305,6 +309,60 @@ public abstract class BaseEdgeProcessor { EdgeEventActionType action, EntityId entityId, JsonNode body) { + ListenableFuture> future = attributesService.find(tenantId, edgeId, DataConstants.SERVER_SCOPE, DefaultDeviceStateService.ACTIVITY_STATE); + return Futures.transformAsync(future, activeOpt -> { + if (activeOpt.isEmpty()) { + log.trace("Edge is not activated. Skipping event. tenantId [{}], edgeId [{}], type[{}], " + + "action [{}], entityId [{}], body [{}]", + tenantId, edgeId, type, action, entityId, body); + return Futures.immediateFuture(null); + } + if (activeOpt.get().getBooleanValue().isPresent() && activeOpt.get().getBooleanValue().get()) { + return doSaveEdgeEvent(tenantId, edgeId, type, action, entityId, body); + } else { + if (doSaveIfEdgeIsOffline(type, action)) { + return doSaveEdgeEvent(tenantId, edgeId, type, action, entityId, body); + } else { + log.trace("Edge is not active at the moment. Skipping event. tenantId [{}], edgeId [{}], type[{}], " + + "action [{}], entityId [{}], body [{}]", + tenantId, edgeId, type, action, entityId, body); + return Futures.immediateFuture(null); + } + } + }, dbCallbackExecutorService); + } + + private boolean doSaveIfEdgeIsOffline(EdgeEventType type, + EdgeEventActionType action) { + switch (action) { + case TIMESERIES_UPDATED: + case ALARM_ACK: + case ALARM_CLEAR: + case ALARM_ASSIGNED: + case ALARM_UNASSIGNED: + case CREDENTIALS_REQUEST: + return true; + } + switch (type) { + case ALARM: + case RULE_CHAIN: + case RULE_CHAIN_METADATA: + case USER: + case CUSTOMER: + case TENANT: + case TENANT_PROFILE: + case WIDGETS_BUNDLE: + case WIDGET_TYPE: + case ADMIN_SETTINGS: + case OTA_PACKAGE: + case QUEUE: + case RELATION: + return true; + } + return false; + } + + private ListenableFuture doSaveEdgeEvent(TenantId tenantId, EdgeId edgeId, EdgeEventType type, EdgeEventActionType action, EntityId entityId, JsonNode body) { log.debug("Pushing event to edge queue. tenantId [{}], edgeId [{}], type[{}], " + "action [{}], entityId [{}], body [{}]", tenantId, edgeId, type, action, entityId, body); diff --git a/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java index 689847416c..43544dd5cf 100644 --- a/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java @@ -130,7 +130,7 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { installation(); edgeImitator = new EdgeImitator("localhost", 7070, edge.getRoutingKey(), edge.getSecret()); - edgeImitator.expectMessageAmount(26); + edgeImitator.expectMessageAmount(21); edgeImitator.connect(); requestEdgeRuleChainMetadata(); @@ -163,9 +163,7 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { @After public void teardownEdgeTest() { try { - edgeImitator.expectMessageAmount(2); loginTenantAdmin(); - Assert.assertTrue(edgeImitator.waitForMessages()); doDelete("/api/edge/" + edge.getId().toString()) .andExpect(status().isOk()); @@ -228,33 +226,33 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { // 1 message from queue fetcher validateQueues(); - // 2 messages - 1 from rule chain fetcher and 1 from rule chain controller + // 1 from rule chain fetcher UUID ruleChainUUID = validateRuleChains(); // 1 from request message validateRuleChainMetadataUpdates(ruleChainUUID); - // 4 messages - 4 messages from fetcher - 2 from system level ('mail', 'mailTemplates') and 2 from admin level ('mail', 'mailTemplates') + // 4 messages + // - 2 from fetcher - system level ('mail', 'mailTemplates') + // - 2 from fetcher - admin level ('mail', 'mailTemplates') validateAdminSettings(); - // 5 messages + // 4 messages // - 1 from default profile fetcher // - 2 from device profile fetcher (default and thermostat) // - 1 from device fetcher - // - 1 from device controller (thermostat) validateDeviceProfiles(); - // 4 messages + // 3 messages // - 1 from default profile fetcher // - 1 message from asset profile fetcher // - 1 message from asset fetcher - // - 1 message from asset controller validateAssetProfiles(); - // 2 messages - 1 from device fetcher and 1 from device controller + // 1 from device fetcher validateDevices(); - // 2 messages - 1 from asset fetcher and 1 from asset controller + // 1 from asset fetcher validateAssets(); // 1 message from public customer fetcher @@ -308,8 +306,7 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { // default msg device profile from fetcher // thermostat msg from device profile fetcher // thermostat msg from device fetcher - // thermostat msg from creation of device - Assert.assertEquals(5, deviceProfileUpdateMsgList.size()); + Assert.assertEquals(4, deviceProfileUpdateMsgList.size()); Optional thermostatProfileUpdateMsgOpt = deviceProfileUpdateMsgList.stream().filter(dfum -> THERMOSTAT_DEVICE_PROFILE_NAME.equals(dfum.getName())).findAny(); Assert.assertTrue(thermostatProfileUpdateMsgOpt.isPresent()); @@ -326,10 +323,9 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { } private void validateDevices() throws Exception { - List deviceUpdateMsgs = edgeImitator.findAllMessagesByType(DeviceUpdateMsg.class); - Assert.assertEquals(2, deviceUpdateMsgs.size()); - validateDevice(deviceUpdateMsgs.get(0)); - validateDevice(deviceUpdateMsgs.get(1)); + Optional deviceUpdateMsgOpt = edgeImitator.findMessageByType(DeviceUpdateMsg.class); + Assert.assertTrue(deviceUpdateMsgOpt.isPresent()); + validateDevice(deviceUpdateMsgOpt.get()); } private void validateDevice(DeviceUpdateMsg deviceUpdateMsg) throws Exception { @@ -345,10 +341,9 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { } private void validateAssets() throws Exception { - List assetUpdateMsgs = edgeImitator.findAllMessagesByType(AssetUpdateMsg.class); - Assert.assertEquals(2, assetUpdateMsgs.size()); - validateAsset(assetUpdateMsgs.get(0)); - validateAsset(assetUpdateMsgs.get(1)); + Optional assetUpdateMsgOpt = edgeImitator.findMessageByType(AssetUpdateMsg.class); + Assert.assertTrue(assetUpdateMsgOpt.isPresent()); + validateAsset(assetUpdateMsgOpt.get()); } private void validateAsset(AssetUpdateMsg assetUpdateMsg) throws Exception { @@ -365,12 +360,10 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { } private UUID validateRuleChains() throws Exception { - List ruleChainUpdateMsgs = edgeImitator.findAllMessagesByType(RuleChainUpdateMsg.class); - Assert.assertEquals(2, ruleChainUpdateMsgs.size()); - RuleChainUpdateMsg ruleChainCreateMsg = ruleChainUpdateMsgs.get(0); - RuleChainUpdateMsg ruleChainUpdateMsg = ruleChainUpdateMsgs.get(1); - validateRuleChain(ruleChainCreateMsg, UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE); - validateRuleChain(ruleChainUpdateMsg, UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE); + Optional ruleChainUpdateMsgOpt = edgeImitator.findMessageByType(RuleChainUpdateMsg.class); + Assert.assertTrue(ruleChainUpdateMsgOpt.isPresent()); + RuleChainUpdateMsg ruleChainUpdateMsg = ruleChainUpdateMsgOpt.get(); + validateRuleChain(ruleChainUpdateMsg, UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE); return new UUID(ruleChainUpdateMsg.getIdMSB(), ruleChainUpdateMsg.getIdLSB()); } @@ -429,7 +422,7 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { private void validateAssetProfiles() throws Exception { List assetProfileUpdateMsgs = edgeImitator.findAllMessagesByType(AssetProfileUpdateMsg.class); - Assert.assertEquals(4, assetProfileUpdateMsgs.size()); + Assert.assertEquals(3, assetProfileUpdateMsgs.size()); AssetProfileUpdateMsg assetProfileUpdateMsg = assetProfileUpdateMsgs.get(0); Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, assetProfileUpdateMsg.getMsgType()); UUID assetProfileUUID = new UUID(assetProfileUpdateMsg.getIdMSB(), assetProfileUpdateMsg.getIdLSB()); diff --git a/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java index aed99b4cba..ca8dd5dbf0 100644 --- a/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java @@ -274,9 +274,7 @@ public class DeviceEdgeTest extends AbstractEdgeTest { tenantProfile.getProfileData().setConfiguration(profileConfiguration); doPost("/api/tenantProfile/", tenantProfile, TenantProfile.class); - edgeImitator.expectMessageAmount(2); loginTenantAdmin(); - Assert.assertTrue(edgeImitator.waitForMessages()); UUID uuid = Uuids.timeBased(); diff --git a/application/src/test/java/org/thingsboard/server/edge/UserEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/UserEdgeTest.java index 3fd39328d5..3dc398b9f3 100644 --- a/application/src/test/java/org/thingsboard/server/edge/UserEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/UserEdgeTest.java @@ -79,9 +79,7 @@ public class UserEdgeTest extends AbstractEdgeTest { Assert.assertEquals(savedTenantAdmin.getLastName(), userUpdateMsg.getLastName()); // update user credentials - edgeImitator.expectMessageAmount(2); login(savedTenantAdmin.getEmail(), "tenant"); - Assert.assertTrue(edgeImitator.waitForMessages()); edgeImitator.expectMessageAmount(1); ChangePasswordRequest changePasswordRequest = new ChangePasswordRequest(); @@ -96,9 +94,7 @@ public class UserEdgeTest extends AbstractEdgeTest { Assert.assertEquals(savedTenantAdmin.getUuidId().getLeastSignificantBits(), userCredentialsUpdateMsg.getUserIdLSB()); Assert.assertTrue(passwordEncoder.matches(changePasswordRequest.getNewPassword(), userCredentialsUpdateMsg.getPassword())); - edgeImitator.expectMessageAmount(2); loginTenantAdmin(); - Assert.assertTrue(edgeImitator.waitForMessages()); // delete user edgeImitator.expectMessageAmount(1); @@ -164,9 +160,7 @@ public class UserEdgeTest extends AbstractEdgeTest { Assert.assertEquals(savedCustomerUser.getLastName(), userUpdateMsg.getLastName()); // update user credentials - edgeImitator.expectMessageAmount(2); login(savedCustomerUser.getEmail(), "customer"); - Assert.assertTrue(edgeImitator.waitForMessages()); edgeImitator.expectMessageAmount(1); ChangePasswordRequest changePasswordRequest = new ChangePasswordRequest(); @@ -181,9 +175,7 @@ public class UserEdgeTest extends AbstractEdgeTest { Assert.assertEquals(savedCustomerUser.getUuidId().getLeastSignificantBits(), userCredentialsUpdateMsg.getUserIdLSB()); Assert.assertTrue(passwordEncoder.matches(changePasswordRequest.getNewPassword(), userCredentialsUpdateMsg.getPassword())); - edgeImitator.expectMessageAmount(2); loginTenantAdmin(); - Assert.assertTrue(edgeImitator.waitForMessages()); // delete user edgeImitator.expectMessageAmount(1); diff --git a/dao/src/main/java/org/thingsboard/server/dao/eventsourcing/SaveEntityEvent.java b/dao/src/main/java/org/thingsboard/server/dao/eventsourcing/SaveEntityEvent.java index 205f592d43..cc2e854f59 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/eventsourcing/SaveEntityEvent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/eventsourcing/SaveEntityEvent.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.id.TenantId; public class SaveEntityEvent { private final TenantId tenantId; private final T entity; + private final T oldEntity; private final EntityId entityId; private final Boolean added; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index 4db1eb668a..a584fef86c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -70,8 +70,8 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic public static final String USER_PASSWORD_HISTORY = "userPasswordHistory"; - private static final String LAST_LOGIN_TS = "lastLoginTs"; - private static final String FAILED_LOGIN_ATTEMPTS = "failedLoginAttempts"; + public static final String LAST_LOGIN_TS = "lastLoginTs"; + public static final String FAILED_LOGIN_ATTEMPTS = "failedLoginAttempts"; private static final int DEFAULT_TOKEN_LENGTH = 30; public static final String INCORRECT_USER_ID = "Incorrect userId "; @@ -126,7 +126,7 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic @Override public User saveUser(TenantId tenantId, User user) { log.trace("Executing saveUser [{}]", user); - userValidator.validate(user, User::getTenantId); + User oldUser = userValidator.validate(user, User::getTenantId); if (!userLoginCaseSensitive) { user.setEmail(user.getEmail().toLowerCase()); } @@ -143,6 +143,7 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic eventPublisher.publishEvent(SaveEntityEvent.builder() .tenantId(tenantId == null ? TenantId.SYS_TENANT_ID : tenantId) .entity(savedUser) + .oldEntity(oldUser) .entityId(savedUser.getId()) .added(user.getId() == null).build()); return savedUser; From 9d19a15413cc2ec629aaed4d8ab6fef81542595d Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 17 Oct 2023 15:12:48 +0200 Subject: [PATCH 02/29] used bouncycastle for parsing cert pem credentials --- .../credentials/CertPemCredentials.java | 302 +++++------------- .../credentials/CertPemCredentialsTest.java | 28 ++ .../src/test/resources/pem/ec_key.pem | 8 + .../test/resources/pem/rsa_encrypted_key.pem | 30 ++ .../pem/rsa_encrypted_traditional_key.pem | 30 ++ .../src/test/resources/pem/rsa_key.pem | 52 +++ 6 files changed, 226 insertions(+), 224 deletions(-) create mode 100644 rule-engine/rule-engine-components/src/test/resources/pem/ec_key.pem create mode 100644 rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_key.pem create mode 100644 rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_traditional_key.pem create mode 100644 rule-engine/rule-engine-components/src/test/resources/pem/rsa_key.pem diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java index 0137887aaf..c17f71bc33 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java @@ -20,68 +20,57 @@ import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.codec.binary.Base64; +import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.openssl.PEMDecryptorProvider; import org.bouncycastle.openssl.PEMEncryptedKeyPair; import org.bouncycastle.openssl.PEMKeyPair; +import org.bouncycastle.openssl.PEMParser; import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter; import org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder; -import org.bouncycastle.util.encoders.Hex; +import org.bouncycastle.operator.InputDecryptorProvider; +import org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo; +import org.bouncycastle.pkcs.PKCSException; +import org.bouncycastle.pkcs.jcajce.JcePKCSPBEInputDecryptorProviderBuilder; import org.thingsboard.server.common.data.StringUtils; -import javax.crypto.Cipher; -import javax.crypto.EncryptedPrivateKeyInfo; -import javax.crypto.SecretKey; -import javax.crypto.SecretKeyFactory; -import javax.crypto.spec.IvParameterSpec; -import javax.crypto.spec.PBEKeySpec; -import javax.crypto.spec.SecretKeySpec; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.TrustManagerFactory; -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.security.AlgorithmParameters; -import java.security.Key; -import java.security.KeyFactory; -import java.security.KeyPair; +import java.io.IOException; +import java.io.StringReader; +import java.security.GeneralSecurityException; import java.security.KeyStore; -import java.security.MessageDigest; import java.security.PrivateKey; import java.security.Security; +import java.security.cert.CertPath; import java.security.cert.Certificate; import java.security.cert.CertificateFactory; import java.security.cert.X509Certificate; -import java.security.spec.KeySpec; -import java.security.spec.PKCS8EncodedKeySpec; -import java.security.spec.RSAPrivateCrtKeySpec; import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import java.util.stream.Collectors; @Data @Slf4j @JsonIgnoreProperties(ignoreUnknown = true) public class CertPemCredentials implements ClientCredentials { - private static final String TLS_VERSION = "TLSv1.2"; + public static final String PRIVATE_KEY_ALIAS = "private-key"; + public static final String X_509 = "X.509"; + public static final String CERT_ALIAS_PREFIX = "cert-"; + public static final String CA_CERT_CERT_ALIAS_PREFIX = "caCert-cert-"; protected String caCert; private String cert; private String privateKey; private String password; - static final String OPENSSL_ENCRYPTED_RSA_PRIVATEKEY_REGEX = "\\s*" - + "-----BEGIN RSA PRIVATE KEY-----" + "\\s*" - + "Proc-Type: 4,ENCRYPTED" + "\\s*" - + "DEK-Info:" + "\\s*([^\\s]+)" + "\\s+" - + "([\\s\\S]*)" - + "-----END RSA PRIVATE KEY-----" + "\\s*"; - - static final Pattern OPENSSL_ENCRYPTED_RSA_PRIVATEKEY_PATTERN = Pattern.compile(OPENSSL_ENCRYPTED_RSA_PRIVATEKEY_REGEX); + public CertPemCredentials() { + if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { + Security.addProvider(new BouncyCastleProvider()); + } + } @Override public CredentialsType getType() { @@ -91,7 +80,6 @@ public class CertPemCredentials implements ClientCredentials { @Override public SslContext initSslContext() { try { - Security.addProvider(new BouncyCastleProvider()); SslContextBuilder builder = SslContextBuilder.forClient(); if (StringUtils.hasLength(caCert)) { builder.trustManager(createAndInitTrustManagerFactory()); @@ -106,51 +94,13 @@ public class CertPemCredentials implements ClientCredentials { } } - private KeyManagerFactory createAndInitKeyManagerFactory() throws Exception { - List certHolders = readCertFile(cert); - Object keyObject = readPrivateKeyFile(privateKey); - char[] passwordCharArray = "".toCharArray(); - if (!StringUtils.isEmpty(password)) { - passwordCharArray = password.toCharArray(); - } - - JcaPEMKeyConverter keyConverter = new JcaPEMKeyConverter().setProvider("BC"); - - PrivateKey privateKey; - if (keyObject instanceof PEMEncryptedKeyPair) { - PEMDecryptorProvider provider = new JcePEMDecryptorProviderBuilder().build(passwordCharArray); - KeyPair key = keyConverter.getKeyPair(((PEMEncryptedKeyPair) keyObject).decryptKeyPair(provider)); - privateKey = key.getPrivate(); - } else if (keyObject instanceof PEMKeyPair) { - KeyPair key = keyConverter.getKeyPair((PEMKeyPair) keyObject); - privateKey = key.getPrivate(); - } else if (keyObject instanceof PrivateKey) { - privateKey = (PrivateKey) keyObject; - } else { - throw new RuntimeException("Unable to get private key from object: " + keyObject.getClass()); - } - - KeyStore clientKeyStore = KeyStore.getInstance(KeyStore.getDefaultType()); - clientKeyStore.load(null, null); - for (X509Certificate certHolder : certHolders) { - clientKeyStore.setCertificateEntry("cert-" + certHolder.getSubjectDN().getName(), certHolder); - } - clientKeyStore.setKeyEntry("private-key", - privateKey, - passwordCharArray, - certHolders.toArray(new Certificate[]{})); - KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - keyManagerFactory.init(clientKeyStore, passwordCharArray); - return keyManagerFactory; - } - protected TrustManagerFactory createAndInitTrustManagerFactory() throws Exception { - List caCertHolders = readCertFile(caCert); + List caCerts = readCertFile(caCert); KeyStore caKeyStore = KeyStore.getInstance(KeyStore.getDefaultType()); caKeyStore.load(null, null); - for (X509Certificate caCertHolder : caCertHolders) { - caKeyStore.setCertificateEntry("caCert-cert-" + caCertHolder.getSubjectDN().getName(), caCertHolder); + for (X509Certificate caCert : caCerts) { + caKeyStore.setCertificateEntry(CA_CERT_CERT_ALIAS_PREFIX + caCert.getSubjectDN().getName(), caCert); } TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); @@ -158,170 +108,74 @@ public class CertPemCredentials implements ClientCredentials { return trustManagerFactory; } - List readCertFile(String fileContent) throws Exception { - if (fileContent == null || fileContent.trim().isEmpty()) { - return Collections.emptyList(); + protected KeyManagerFactory createAndInitKeyManagerFactory() throws Exception { + KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + kmf.init(loadKeyStore(), password.toCharArray()); + return kmf; + } + + private KeyStore loadKeyStore() throws Exception { + List certificates = readCertFile(this.cert); + PrivateKey privateKey = readPrivateKey(this.privateKey, this.password); + + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + keyStore.load(null); + List unique = certificates.stream().distinct().collect(Collectors.toList()); + for (X509Certificate cert : unique) { + keyStore.setCertificateEntry(CERT_ALIAS_PREFIX + cert.getSubjectDN().getName(), cert); } + if (privateKey != null) { + CertificateFactory factory = CertificateFactory.getInstance(X_509); + CertPath certPath = factory.generateCertPath(certificates); + List path = certPath.getCertificates(); + Certificate[] x509Certificates = path.toArray(new Certificate[0]); + keyStore.setKeyEntry(PRIVATE_KEY_ALIAS, privateKey, password.toCharArray(), x509Certificates); + } + return keyStore; + } + + protected List readCertFile(String fileContent) throws IOException, GeneralSecurityException { List certificates = new ArrayList<>(); - String[] pems = fileContent.trim().split("-----END CERTIFICATE-----"); - for (String pem : pems) { - if (pem.trim().isEmpty()) { - continue; - } - pem = pem.replace("-----BEGIN CERTIFICATE-----", "") - .replace("-----END CERTIFICATE-----", "") - .replaceAll("\\s", ""); - byte[] decoded = Base64.decodeBase64(pem); - CertificateFactory certFactory = CertificateFactory.getInstance("X.509"); - try (InputStream inStream = new ByteArrayInputStream(decoded)) { - certificates.add((X509Certificate) certFactory.generateCertificate(inStream)); + JcaX509CertificateConverter certConverter = new JcaX509CertificateConverter(); + try (PEMParser pemParser = new PEMParser(new StringReader(fileContent))) { + Object object; + while ((object = pemParser.readObject()) != null) { + if (object instanceof X509CertificateHolder) { + X509Certificate x509Cert = certConverter.getCertificate((X509CertificateHolder) object); + certificates.add(x509Cert); + } } } return certificates; } - private PrivateKey readPrivateKeyFile(String fileContent) throws Exception { + protected PrivateKey readPrivateKey(String fileContent, String password) throws IOException, PKCSException { PrivateKey privateKey = null; - if (fileContent != null && !fileContent.isEmpty()) { - KeyFactory keyFactory = KeyFactory.getInstance("RSA"); - KeySpec keySpec = getKeySpec(fileContent); - privateKey = keyFactory.generatePrivate(keySpec); - } - return privateKey; - } - - private KeySpec getKeySpec(String encodedKey) throws Exception { - KeySpec keySpec = null; - Matcher matcher = OPENSSL_ENCRYPTED_RSA_PRIVATEKEY_PATTERN.matcher(encodedKey); - if (matcher.matches()) { - String encryptionDetails = matcher.group(1).trim(); - String encryptedKey = matcher.group(2).replaceAll("\\s", ""); - byte[] encryptedBinaryKey = java.util.Base64.getDecoder().decode(encryptedKey); - String[] encryptionDetailsParts = encryptionDetails.split(","); - if (encryptionDetailsParts.length == 2) { - String encryptionAlgorithm = encryptionDetailsParts[0]; - String encryptedAlgorithmParams = encryptionDetailsParts[1]; - byte[] pw = password.getBytes(); - byte[] iv = Hex.decode(encryptedAlgorithmParams); - - MessageDigest digest = MessageDigest.getInstance("MD5"); - digest.update(pw); - digest.update(iv, 0, 8); - - byte[] round1Digest = digest.digest(); - digest.update(round1Digest); - digest.update(pw); - digest.update(iv, 0, 8); - - byte[] round2Digest = digest.digest(); - Cipher cipher = null; - SecretKey secretKey = null; - byte[] key = null; - - switch(encryptionAlgorithm) { - case "AES-256-CBC": - cipher = Cipher.getInstance("AES/CBC/PKCS5Padding"); - key = new byte[32]; - System.arraycopy(round1Digest, 0, key, 0, 16); - System.arraycopy(round2Digest, 0, key, 16, 16); - secretKey = new SecretKeySpec(key, "AES"); - break; - case "AES-192-CBC": - cipher = Cipher.getInstance("AES/CBC/PKCS5Padding"); - key = new byte[24]; - System.arraycopy(round1Digest, 0, key, 0, 16); - System.arraycopy(round2Digest, 0, key, 16, 8); - secretKey = new SecretKeySpec(key, "AES"); + JcaPEMKeyConverter keyConverter = new JcaPEMKeyConverter(); + + if (StringUtils.isNotEmpty(fileContent)) { + try (PEMParser pemParser = new PEMParser(new StringReader(fileContent))) { + Object object; + while ((object = pemParser.readObject()) != null) { + if (object instanceof PEMEncryptedKeyPair) { + PEMDecryptorProvider decProv = new JcePEMDecryptorProviderBuilder().build(password.toCharArray()); + privateKey = keyConverter.getKeyPair(((PEMEncryptedKeyPair) object).decryptKeyPair(decProv)).getPrivate(); break; - case "AES-128-CBC": - cipher = Cipher.getInstance("AES/CBC/PKCS5Padding"); - key = new byte[16]; - System.arraycopy(round1Digest, 0, key, 0, 16); - secretKey = new SecretKeySpec(key, "AES"); + } else if (object instanceof PKCS8EncryptedPrivateKeyInfo) { + InputDecryptorProvider decProv = + new JcePKCSPBEInputDecryptorProviderBuilder().setProvider(new BouncyCastleProvider()).build(password.toCharArray()); + privateKey = keyConverter.getPrivateKey(((PKCS8EncryptedPrivateKeyInfo) object).decryptPrivateKeyInfo(decProv)); break; - case "DES-EDE3-CBC": - cipher = Cipher.getInstance("DESede/CBC/PKCS5Padding"); - key = new byte[24]; - System.arraycopy(round1Digest, 0, key, 0, 16); - System.arraycopy(round2Digest, 0, key, 16, 8); - secretKey = new SecretKeySpec(key, "DESede"); - break; - case "DES-CBC": - cipher = Cipher.getInstance("DES/CBC/PKCS5Padding"); - key = new byte[8]; - System.arraycopy(round1Digest, 0, key, 0, 8); - secretKey = new SecretKeySpec(key, "DES"); + } else if (object instanceof PEMKeyPair) { + privateKey = keyConverter.getKeyPair((PEMKeyPair) object).getPrivate(); break; + } else if (object instanceof PrivateKeyInfo) { + privateKey = keyConverter.getPrivateKey((PrivateKeyInfo) object); } - if (cipher != null) { - cipher.init(Cipher.DECRYPT_MODE, secretKey, new IvParameterSpec(iv)); - byte[] pkcs1 = cipher.doFinal(encryptedBinaryKey); - keySpec = decodeRSAPrivatePKCS1(pkcs1); - } else { - throw new RuntimeException("Unknown Encryption algorithm!"); } - } else { - throw new RuntimeException("Wrong encryption details!"); } - } else { - encodedKey = encodedKey.replaceAll(".*BEGIN.*PRIVATE KEY.*", "") - .replaceAll(".*END.*PRIVATE KEY.*", "") - .replaceAll("\\s", ""); - byte[] decoded = Base64.decodeBase64(encodedKey); - if (password == null || password.isEmpty()) { - keySpec = new PKCS8EncodedKeySpec(decoded); - } else { - PBEKeySpec pbeKeySpec = new PBEKeySpec(password.toCharArray()); - - EncryptedPrivateKeyInfo privateKeyInfo = new EncryptedPrivateKeyInfo(decoded); - String algorithmName = privateKeyInfo.getAlgName(); - Cipher cipher = Cipher.getInstance(algorithmName); - SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance(algorithmName); - - Key pbeKey = secretKeyFactory.generateSecret(pbeKeySpec); - AlgorithmParameters algParams = privateKeyInfo.getAlgParameters(); - cipher.init(Cipher.DECRYPT_MODE, pbeKey, algParams); - keySpec = privateKeyInfo.getKeySpec(cipher); - } - } - return keySpec; - } - - private static BigInteger derint(ByteBuffer input) { - int len = der(input, 0x02); - byte[] value = new byte[len]; - input.get(value); - return new BigInteger(+1, value); - } - - private static int der(ByteBuffer input, int exp) { - int tag = input.get() & 0xFF; - if (tag != exp) throw new IllegalArgumentException("Unexpected tag"); - int n = input.get() & 0xFF; - if (n < 128) return n; - n &= 0x7F; - if ((n < 1) || (n > 2)) throw new IllegalArgumentException("Invalid length"); - int len = 0; - while (n-- > 0) { - len <<= 8; - len |= input.get() & 0xFF; } - return len; - } - - static RSAPrivateCrtKeySpec decodeRSAPrivatePKCS1(byte[] encoded) { - ByteBuffer input = ByteBuffer.wrap(encoded); - if (der(input, 0x30) != input.remaining()) throw new IllegalArgumentException("Excess data"); - if (!BigInteger.ZERO.equals(derint(input))) throw new IllegalArgumentException("Unsupported version"); - BigInteger n = derint(input); - BigInteger e = derint(input); - BigInteger d = derint(input); - BigInteger p = derint(input); - BigInteger q = derint(input); - BigInteger ep = derint(input); - BigInteger eq = derint(input); - BigInteger c = derint(input); - return new RSAPrivateCrtKeySpec(n, e, d, p, q, ep, eq, c); + return privateKey; } } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java index 904f470001..fd7f0d3378 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java @@ -18,16 +18,27 @@ package org.thingsboard.rule.engine.credentials; import org.apache.commons.io.FileUtils; import org.junit.Assert; import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.File; import java.io.IOException; +import java.security.PrivateKey; import java.security.cert.X509Certificate; import java.util.List; +import java.util.stream.Stream; public class CertPemCredentialsTest { private final CertPemCredentials credentials = new CertPemCredentials(); + private static final String PASS = "test"; + private static final String EMPTY_PASS = ""; + private static final String RSA = "RSA"; + private static final String ECDSA = "ECDSA"; + @Test public void testChainOfCertificates() throws Exception { String fileContent = fileContent("pem/tb-cloud-chain.pem"); @@ -65,6 +76,23 @@ public class CertPemCredentialsTest { Assert.assertEquals(0, x509Certificates.size()); } + private static Stream testReadPrivateKey() { + return Stream.of( + Arguments.of("pem/rsa_key.pem", EMPTY_PASS, RSA), + Arguments.of("pem/rsa_encrypted_key.pem", PASS, RSA), + Arguments.of("pem/rsa_encrypted_traditional_key.pem", PASS, RSA), + Arguments.of("pem/ec_key.pem", EMPTY_PASS, ECDSA) + ); + } + + @ParameterizedTest + @MethodSource + public void testReadPrivateKey(String keyPath, String password, String algorithm) throws Exception { + PrivateKey privateKey = credentials.readPrivateKey(fileContent(keyPath), password); + Assertions.assertNotNull(privateKey); + Assertions.assertEquals(algorithm, privateKey.getAlgorithm()); + } + private String fileContent(String fileName) throws IOException { ClassLoader classLoader = getClass().getClassLoader(); File file = new File(classLoader.getResource(fileName).getFile()); diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/ec_key.pem b/rule-engine/rule-engine-components/src/test/resources/pem/ec_key.pem new file mode 100644 index 0000000000..74e0716259 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/ec_key.pem @@ -0,0 +1,8 @@ +-----BEGIN EC PARAMETERS----- +BggqhkjOPQMBBw== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIIEd0mMh0EEy3fMbOpbUY6kW0oAYcaYoTvoVpZxDr5qZoAoGCCqGSM49 +AwEHoUQDQgAEz4MgawieJfVc5zUOPiw5WFxfHGJf7dOMsHvudDxdOs27PXPbJfi0 +9BVJ3+JjNxA2wQz9KUk877oWRYrN/e+MbA== +-----END EC PRIVATE KEY----- \ No newline at end of file diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_key.pem b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_key.pem new file mode 100644 index 0000000000..045a44ce28 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_key.pem @@ -0,0 +1,30 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIFLTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQILTHGLs8mGUkCAggA +MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBAUnb+mChJ9Wu9F7q6ingLYBIIE +0Mwe8Zl6fs5kiT1AL7gXrSSXmyOVvxDFt0V3TX1w399VIadcUUO0RQeEqXoMEUzl +5at99Xmoo7ByvZSPWCdV4d/j5Yw+Z15euxzclSZJnmBgvQx8cFPLCTTaqlgv5r/Y +lTzBrczbgruMFKtzkvHgvZYiagccOtFHDNC1fUBcUR8dkOOsgTiy4QCVo5kkXHt/ +rblE/uVWI8/E318WZBZaz68HcmGIG6ivdMEsKskSKrH6zA3eLjyGB+zSAIPRB/Mp +s7Rj+RK74zFSYyaq6fgdTG8lug2f3rHImSOtQThcfme5XL4P66rUgJsh/sml1vqH +e848VArGoVy3wfvkss6CyXIJevhFh2xVWRyVqG1nraw2QssEnVqIZvdAnaJJONX8 +r1trjHkZ1JD0nO9Mns1c/bw6hjK6W3UwGfgEMM9VQ7wNI2B6CFOXHKTHg6r3us4k +UqaQtfbpTv+d0YKF/rolDcK+mK/rkxP3rtJA7Ud8nQ4VjxyYX4jTs3/BzDkP7Tsj +5gKy9e1zuTF+MUWs3G5oKGQUKVcbgoYJ+iOqgVSd1JbecRo7Pl8XgDv3I9RWHzUr +EAMjVJjRU9tJuPvILFBkpl1/OPC9sGxJz9Hy9qLtEGhGLUhNz6XmIy/aWPCyA4ea +ZES/n7f+aYmXIxulcxS7MUejkwl1EtNqVyrKvLiRBXjBk2HPCb7Te8fRu/LpHZXN +D7wjymg1fGZPPFzdKh7wMdAKiK50KIMGXTxS6kHb6qW/755oSUjWRLPGcPCfdbjn +UiC5WC9FCog6jfRq1rMlz5b8yjyb+UbJ6N4qFSHeQf+7WLeS0Di8k3cLDSWl6T7M +z82ePof2V2TrADNpXvAcR78uiDfUpfa7DhkimvbBZpRVaaQVU7unxUPVc93WgCWV +a7kBuFJAGt+Wt1LPPD/5KOQ5pRINSoh4VhiZzmnY/m7RDPWEaL5gsMjF5bFoP2UZ +MuyAiTmvO299lJDrdQyQds7yafO9PrTE4msuqpuZSHW7ZZIdRO6EXlfZ1We3icWr ++jE47bUIEl04k1PvXyv9LeoqlHZJTagxZIerMOEwq976MaVR7RJbqUpRUV9FFNCL +gTouPCwUcVtLCaTYQjz/+12/YeVkiBHIWkI8Vv5Mn3Vkwy303ygGCQ+brht1e8x+ +BbgzSpiX8aHiEuDAKooewxnKrf3Dk9BcwbnftxajOZcZ3iphk07t5VLRy86zLKCq +ZOY+KymcDCGaOPnSHFrZK3lZOOT+BB9Vi6EYAkxZCZgoDsb/voMEdpPlxK7ultf6 +is5/JQeQbeP9wbNh4Ru2x3p5Ir4wffhh1KT3UsMobusosTo55ErhMHPvH5amppwq +IrxdM7heo7JMaNKmtol4y45IqSt58iluF5m2Ds4m85xjDteRgEOjtNBStFxPCMAB +KUEzRxEaplAcJfzYzpYtoHZuZ8W3Gi7yeXQ+BV8Q9DeaZc5DDDhIcIkOoHOAKhit +d7Gpr8hpwc60AgHRjua8OdbhM4ntT1xnyDEqZbP8mN+UBAohOHMrqo+f4DL1ibB9 +qSwfdLiVItsEBqlfANV3i9rEeKNH5tOFwFCmmH1yBCSDCtWPmPUJ5tZae6fIetK5 +uSstFXLaDpm6fcHgkeqrnyteWpnk5X5SQQ+fMHPjQ2vp +-----END ENCRYPTED PRIVATE KEY----- diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_traditional_key.pem b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_traditional_key.pem new file mode 100644 index 0000000000..0b0e128a25 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_traditional_key.pem @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,FB5DE36A7A8B25DA + +skR15rUvZmdLzGqU5/BF4Yc3E6dxtXTvlOhuGnqH/idItMKUMWIIlQ7ZfWYF/CrC +CkeAUqhF4y+y+2eR4ejUzZKs6bYjTtkXAXAqQvCsTrTBdQSCcLwbHWLWMro0UG24 +e23Rx8kD0YC7VHqyr08NlLh94wR7kanhEeRUbmKBonZT/I9AZ5ntiBxq9QVBtc8M +f7LKIsnQDcd39cVXSo3LOJ/x7YVB//Ln1R1dexwxE0sXOyLq2hhrxzfHGuGXXW41 +/3+CeTgmX9Rzpawrq9vbVabPUgFcJlrogNRSnUAm9kz4b2zadCxEaCejVmhBy4wx +z2AJGcmE+D4VkQK1AAj5+AQQrPOIIFQnyGjwHkJSTGVTcKmttRYZBvUjdfQfj1fK +NsKOSZLzZGknM8Pz5MHgHqk70C7f+nm0uVhVuAiykA4PY4JdCAuTWJxMM2LWM3/q +rYCEMwxCGa6U92dakfC+W+d9pAbeN+xYOWkDrqG7BdAYg1P70cuMRPdd5bsq3YPp +G4n5NVyQvLlocGhZgC7NVzUtc18+rGblV4D657+GZwJnBZJN4TYey4+r2D5fv9rO +kcRVwfR704BbUBkjIzVzD7nXtBbr3ni8HSqde3g4aVL4WyV9XNvjUsYyrZ0u9Mt1 +IccAsa1xBquUNMxwO1H1mFLtzPKmFzKtlzqiiDsRQoRylwUa1k03sHKUflZRa8Sf +g4MpTRzK+vo1opMemlonty5YbvWXKlH68ioo49L8N457Y0hIUJOQgywg80NxT33t +x8y66lawd1Iv+Q7pptVxJtA4JmcdPvGwBLJZY4DacMyp/JqchAQfSQfmQ0tC+RfJ +z2By/s5wOEuVDksgp8RF1gn+VmvLyOoLK7tq+zpMO1mhfYTCMgSiz2GkNdiW6i25 +gjNWN/F62YL+9VJo4+olrcsYDFiiJq+deQk3H1tQJzu6qECfDqKDyw7IunvTwFil +5/d43LvLbRj75Kf/++xwTjfHudeTgw02/yPyELnURkUazvkOFsn7n8tU46Qm5TWh +fPFXSYxRf3m9rkKZB98YOJo595RuZyiYg9dEQX8Gybl1/7H7l4Cvw6yp71kgLrrI +JRYEt9pmWbQX97UFC3WTVMdKWakFziYVGPvFKkIzrHgcutbQVNsZ7GbO+rdWMIxr +SfUe6jCEclzGQjI9Ep4PTLjZvbusUMkoUjGasAluXFXDC4RKtpuXd4RmbOLdVuyN +OnZ5KZHFjrv4ch5PakRTViWFWSddV5CJ4fMkCG9qUHKrUWGjOvzu5rbcUzq3xJZG +9loIvlA4ekEAhQPHwx69uBmUwnCgyB9CosQGmUlwmC3KALA/EiXklTA6w0fGiiPk +uLA9oBGrVcD8Peug9Owfmj4fWbxJGP7x1UR/nZWpynIfzME0AD8MK5uqoWmQTG8I +cLSjVAB1CO//AZe4LpYQulMPq4dipmE+YnKLi0WPXuZVARDciAGGV2BfH6Iv8j9k +o9IoklcpGg22zXLoGn4tu+7Y5GcoV/mx68Gun1E/QFuY59damAqoc82EEsxr+UX0 +4vGX+KZn283xSYiilE3qpCZOER0ZUFInphUwJzzYfW3mW/AWR78tFQIQiuVKV2/a +-----END RSA PRIVATE KEY----- diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/rsa_key.pem b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_key.pem new file mode 100644 index 0000000000..8a92a68908 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCwefbtwf6lgUjR +4lh6vzDDb2D/GUqCv1pqtiWQgt5ecGE88cEJcb4V6B4ykdrwVQ8A0ipgmLAzXdqW +07qEws9rQnv6dCHAXk0JvHMB2L+RTyh+EjAkf4qA6wPcEp1HtRrsowNAWcZMVFH1 +nGy5hn/QAe7M8fDFItS+8xuJZu7k/YVgysTLIui5VWKFk+R4GFycjpIlHIJ0x0zL +bzadZD7juABQjdwVrt1fFRxe9FttOgw09MgNTLhWHEliEvhPoQPQOzu1lGwbvUtw +x/DZFBSCJLUW5iH47s3yVQy6QzLb+6ZCW4zT6oEJhZranVQJtXOWtbWqApCbVDwT +3VdJH0ov/N07yb3+414jWPIm7qLVyJtZHuCeHokKEvGi5CWsHO4vNLPs8byrUw5a +L6gdglZK+axyRtAOHASx+EWthWCd0l3GeaWTO9LU4/tEZJhct8wX5nELZgQmJTrl +AAIKXYgeSfI2IvwmDlqFeBkw2yHJ+86/lEHmOfPi/NGkaFRgJzIvM5FP7mBj1lht +f/nVw6iZBXWvJNNCz7MDyZ8RUuNtsuq9h0jzgsLdyEx1w6bxn1TXF1vYVf9zvbhc +PI+dA3LU8uCDOq84VnMt57LScFVRbJWtwLJ1tLk8C9WgC5w7ySo/pvu4HtCcN4Zo +KIBUX71BpfGPJAR0sS7fPkkwzu4fhQIDAQABAoICABm8z+yA/Hh60Hn7vte4Bo6a +MdVChQFokvE5O2VGENRJI4VV5MdR1V0wiybo6rteTF/cRt3rptb2+yhAHNW767BC +8/3k7f82QZoH5+X/DIFOwCMS1/6as0J2BAwWkuWgXhrg81pxPWBoc8OUWq78FKvr +fD5bkrfNiqWGox946aJv7wHc0LKnlrVg5IuCtDFnrCoRCPNsowIRBvwsbhSqSBnB +/hnBdrWa2SJC2+5lSOg3LQyUHpEB/Whhm7o39gr2+q1l1iF3UgUBqHz8S/381bjd +TaPXUGETwulyyfZoUoSOwQKwg2tsqgEPgTQc+eKomgEC40m2MgzVTiW/hDlf3NvA +aEaUUU1izN/t8tuXS/UBWSsVwPeVm2oPTWTVovoj9PFMSLrJ4oM9iMHl27lKP/Xt +aShnCIu0qwVSLqwCM0HxZNZLEvJIFJe3OV7dvFlbnMiEOlDsz4k2sylFdICOOqxC +Nb61hX8n6iYmAID9hahExOAFcJfpV/MrGnF1IfNDdOim5az1k0PUZlA+50NLjOzK +umfAQpsa7ZUpjfNq6HkX5mhJXelvv+pWuvbBogG10nio13I4J/YwydC/0qaijrhp +XTuV3Or2HhGr5Fe9lpzrnWB90q7iqAgGzevds8AagE0EdUIFupx6vr28Gb3mmlvD +yObUj/9cB+eYIae1jmzJAoIBAQC9pN8I0ltyR5IuuPBcEZcqdrRV04iwZylxEcYy +TIj1YBYvU6LP8AOg671Q4vGXOCo1uq/UsCzZMPn5yFa5fMg2AHu2B51nM/NILvD0 +QCgpyvNV64q+Mci9VWoctWZs93QiQyUKe1c/vUMYlWsYXbz/8osChX+r5doxBIQx +w60aXr9FLKfVfYfBfn1nakdjOVVvAFDPyoV6Dmfn/cAfPH50NIeOGYRpVTPHwHYr +ZCcIRW9MIzmS00GogAH1BM8JjRr0F9F+rRESeJKdLSqJbgLy2LtJnvU8YjlRUVWO +FLzqaUyT2PJS9Vqbohrlk52Znq5Gl+SbGhSEx+oTH3JM/n0jAoIBAQDuOZ8bqtRY +p4BuBOPOaiHRIor+ng48m+nhXec4TuKUlwKLFJHXu+lsEZfs7BWijbRqMH4I5GZl +10EmE4mpkp843kqbFi71s7l9xWnM7jgXWSauH0O4Cleq8/9l2ZOissYzakLhNz3C +9IT0JcnHFmPOPlH4McjoKM3zWDniKI2fRn9q4DAEvRxDuB6PYxbz2NY2OAVI6xSA +bNevnyYA0bwvWeigr6dNCAs1z9QwX2oDGfIEUk3ixdGIqIkpL3WcVPXva2hRPAm4 +1gaI39+q86rEPiZJWfpasUEBY2Ho1eyENqaPTGHCTfBq5fMVntOVhs79TbQ+s70c +1Wyfo6sjHh83AoIBAQCuqdLhhRzEPDbe4WY+5dScP4gIJDOYhOseQIiSevsJQ94q +6JTjfuNYqsZKYTqxVAFMSwz2juw/fWQ+Mc3uOIcNdZR7KrhF/QrsSI+T5iMXmtxT +HgVC9wczmh+JIWmcoqxLghvzc3YANog9dCCW6H7SHMj7IYldAO3ch5RZYSdlSi5P +v7k0X9FQ3PcS8Eefk4akHV5Qgu48ZFg+yu7P1h+BV4Ah2E6j1N1D9Hbhr/RjIdBI +B4lXOUsXrg4fZLZqzZMtjWJdkXhP0sz2BktPGAuPLx4PyF+FpdG0m3x4x5DXNPRa +l01YKrGw9bRgDXzxp7xLOEpMr9CGGrnzstrLHviRAoIBAFoyuwmQvuHqWfhOJasM +CE3VFGeflKhiKEXKdjedtrCoFLBwU2ApqBHg/3MXWIG5wavLPI1FXXgF7obqMt9f +wqWXlQvvdExXhk4Wpx6Ou/IrMTgQYmWWlOcHh5YasYmSwvTIsRXxApOEXarLfADD +e3qlogelYfp1KLWQnCoDTMwXtzrSM5w3tjH1zqxfylr9qO3SfD3FtHeDvo6iZZM9 +1lDfa/MbTu8dspDnZeIC3nLaKgZ020SXveROW9CaRZ+xk4TZWCAZ6VxwvPyqN1fU +9r1jAsAXL3GTV5ec939fMDRHNP1g4Erfk74F3uo6vsYIyuqhtzNefqYiMQSoxa2A +RDUCggEAFlN3ih1gpyvErW4Vy/wUd1ckSH/lojlNjbbyXocKE2eiUnJUwTPerVwX +dI8vqvlPohfDIZqfuBVV+8hiJQGeMiAts6roTQ0pu/w1+euQ4DsOpzUErqadVSOj +h8SpvfxDxrftZSMaN6F7g0Pxlix6qt79XH3Kpfzf9BGOfCG7lslXRAjfuk+HUptK +PijoVHwMwFuZVlN8GBh3uzg+wvME92c4Vr1tHpwqjTqDwZN4RmdnrfGdDb1HJUJW +kv+fD65qKnJz1fZ0RTAcWv4bVFi5GJhZarXD3Vr3C5SH8zNZxDeR29OrSuG7+23g +wOqb/axEbvcj5sV6/4p2zz6AzFPEmQ== +-----END PRIVATE KEY----- From 71dd96b08447b707015ff3cab5ae445a8b973be6 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 17 Oct 2023 18:00:04 +0300 Subject: [PATCH 03/29] Fix the widgets_bundle_widget table after upgrade --- application/src/main/data/upgrade/3.5.1/schema_update.sql | 2 +- application/src/main/data/upgrade/3.6.0/schema_update.sql | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/application/src/main/data/upgrade/3.5.1/schema_update.sql b/application/src/main/data/upgrade/3.5.1/schema_update.sql index f41e370ab5..fab91a4e67 100644 --- a/application/src/main/data/upgrade/3.5.1/schema_update.sql +++ b/application/src/main/data/upgrade/3.5.1/schema_update.sql @@ -189,7 +189,7 @@ DO $$ BEGIN IF EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = 'widget_type' and column_name='bundle_alias') THEN - INSERT INTO widgets_bundle_widget SELECT wb.id as widgets_bundle_id, wt.id as widget_type_id from widget_type wt left join widgets_bundle wb ON wt.bundle_alias = wb.alias ON CONFLICT (widgets_bundle_id, widget_type_id) DO NOTHING; + INSERT INTO widgets_bundle_widget SELECT wb.id as widgets_bundle_id, wt.id as widget_type_id from widget_type wt left join widgets_bundle wb ON wt.bundle_alias = wb.alias AND wt.tenant_id = wb.tenant_id ON CONFLICT (widgets_bundle_id, widget_type_id) DO NOTHING; ALTER TABLE widget_type DROP COLUMN IF EXISTS bundle_alias; END IF; END; diff --git a/application/src/main/data/upgrade/3.6.0/schema_update.sql b/application/src/main/data/upgrade/3.6.0/schema_update.sql index 0ad2d97b7a..ee1b311de2 100644 --- a/application/src/main/data/upgrade/3.6.0/schema_update.sql +++ b/application/src/main/data/upgrade/3.6.0/schema_update.sql @@ -24,3 +24,10 @@ ALTER TABLE notification DROP CONSTRAINT IF EXISTS fk_notification_request_id; ALTER TABLE notification DROP CONSTRAINT IF EXISTS fk_notification_recipient_id; CREATE INDEX IF NOT EXISTS idx_notification_notification_request_id ON notification(request_id); CREATE INDEX IF NOT EXISTS idx_notification_request_tenant_id ON notification_request(tenant_id); + +-- DELETE invalid records from M:N widgets_bundle_widget table caused by the bug in previous upgrade script; +DELETE +FROM widgets_bundle_widget wbw +WHERE (SELECT tenant_id FROM widgets_bundle wb WHERE wb.id = wbw.widgets_bundle_id) != + (SELECT tenant_id FROM widget_type WHERE id = wbw.widget_type_id); + From 3946b058cdda60fbdfb03d92d0c60fb3d1399605 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 17 Oct 2023 18:09:38 +0300 Subject: [PATCH 04/29] Do not push ALARM update to edge automatically - only using push to edge rule node --- .../service/edge/EdgeEventSourcingListener.java | 10 +++++----- .../thingsboard/server/edge/AlarmEdgeTest.java | 15 --------------- ui-ngx/src/app/core/http/entity.service.ts | 10 +++++++--- 3 files changed, 12 insertions(+), 23 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java index 707dbec964..006af577c9 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java @@ -25,6 +25,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmApiCallResult; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; @@ -78,7 +79,7 @@ public class EdgeEventSourcingListener { return; } try { - if (!isValidEdgeEventEntity(event.getEntity(), event.getOldEntity())) { + if (!isValidSaveEntityEventForEdgeProcessing(event.getEntity(), event.getOldEntity())) { return; } log.trace("[{}] SaveEntityEvent called: {}", event.getTenantId(), event); @@ -141,7 +142,7 @@ public class EdgeEventSourcingListener { } } - private boolean isValidEdgeEventEntity(Object entity, Object oldEntity) { + private boolean isValidSaveEntityEventForEdgeProcessing(Object entity, Object oldEntity) { if (entity instanceof OtaPackageInfo) { OtaPackageInfo otaPackageInfo = (OtaPackageInfo) entity; return otaPackageInfo.hasUrl() || otaPackageInfo.isHasData(); @@ -159,9 +160,8 @@ public class EdgeEventSourcingListener { cleanUpUserAdditionalInfo(user); return !user.equals(oldUser); } - } else if (entity instanceof AlarmApiCallResult) { - AlarmApiCallResult alarmApiCallResult = (AlarmApiCallResult) entity; - return alarmApiCallResult.isModified(); + } else if (entity instanceof AlarmApiCallResult || entity instanceof Alarm) { + return false; } // Default: If the entity doesn't match any of the conditions, consider it as valid. return true; diff --git a/application/src/test/java/org/thingsboard/server/edge/AlarmEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/AlarmEdgeTest.java index afccc634ed..259572851c 100644 --- a/application/src/test/java/org/thingsboard/server/edge/AlarmEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/AlarmEdgeTest.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.google.protobuf.AbstractMessage; import org.junit.Assert; import org.junit.Test; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; @@ -102,20 +101,6 @@ public class AlarmEdgeTest extends AbstractEdgeTest { Assert.assertEquals(savedAlarm.getStatus().name(), alarmUpdateMsg.getStatus()); Assert.assertEquals(savedAlarm.getSeverity().name(), alarmUpdateMsg.getSeverity()); - // update alarm - String updatedDetails = "{\"testKey\":\"testValue\"}"; - savedAlarm.setDetails(JacksonUtil.OBJECT_MAPPER.readTree(updatedDetails)); - edgeImitator.expectMessageAmount(1); - savedAlarm = doPost("/api/alarm", savedAlarm, Alarm.class); - Assert.assertTrue(edgeImitator.waitForMessages()); - latestMessage = edgeImitator.getLatestMessage(); - Assert.assertTrue(latestMessage instanceof AlarmUpdateMsg); - alarmUpdateMsg = (AlarmUpdateMsg) latestMessage; - Assert.assertEquals(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE, alarmUpdateMsg.getMsgType()); - Assert.assertEquals(savedAlarm.getUuidId().getMostSignificantBits(), alarmUpdateMsg.getIdMSB()); - Assert.assertEquals(savedAlarm.getUuidId().getLeastSignificantBits(), alarmUpdateMsg.getIdLSB()); - Assert.assertEquals(updatedDetails, alarmUpdateMsg.getDetails()); - // ack alarm edgeImitator.expectMessageAmount(1); doPost("/api/alarm/" + savedAlarm.getUuidId() + "/ack"); diff --git a/ui-ngx/src/app/core/http/entity.service.ts b/ui-ngx/src/app/core/http/entity.service.ts index 12f6300b93..cc1dd13345 100644 --- a/ui-ngx/src/app/core/http/entity.service.ts +++ b/ui-ngx/src/app/core/http/entity.service.ts @@ -92,6 +92,7 @@ import { NotificationService } from '@core/http/notification.service'; import { TenantProfileService } from '@core/http/tenant-profile.service'; import { NotificationType } from '@shared/models/notification.models'; import { UserId } from '@shared/models/id/user-id'; +import { AlarmService } from '@core/http/alarm.service'; @Injectable({ providedIn: 'root' @@ -119,7 +120,8 @@ export class EntityService { private assetProfileService: AssetProfileService, private utils: UtilsService, private queueService: QueueService, - private notificationService: NotificationService + private notificationService: NotificationService, + private alarmService: AlarmService ) { } private getEntityObservable(entityType: EntityType, entityId: string, @@ -155,7 +157,7 @@ export class EntityService { observable = this.ruleChainService.getRuleChain(entityId, config); break; case EntityType.ALARM: - console.error('Get Alarm Entity is not implemented!'); + observable = this.alarmService.getAlarm(entityId, config); break; case EntityType.OTA_PACKAGE: observable = this.otaPackageService.getOtaPackageInfo(entityId, config); @@ -238,7 +240,9 @@ export class EntityService { entityIds); break; case EntityType.ALARM: - console.error('Get Alarm Entity is not implemented!'); + observable = this.getEntitiesByIdsObservable( + (id) => this.alarmService.getAlarm(id, config), + entityIds); break; case EntityType.DEVICE_PROFILE: observable = this.getEntitiesByIdsObservable( From 7ccad683abae4fb6cbe2454e397369b5573006ed Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 17 Oct 2023 18:41:22 +0300 Subject: [PATCH 05/29] prettify script --- application/src/main/data/upgrade/3.6.0/schema_update.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/data/upgrade/3.6.0/schema_update.sql b/application/src/main/data/upgrade/3.6.0/schema_update.sql index ee1b311de2..0361a5db65 100644 --- a/application/src/main/data/upgrade/3.6.0/schema_update.sql +++ b/application/src/main/data/upgrade/3.6.0/schema_update.sql @@ -29,5 +29,5 @@ CREATE INDEX IF NOT EXISTS idx_notification_request_tenant_id ON notification_re DELETE FROM widgets_bundle_widget wbw WHERE (SELECT tenant_id FROM widgets_bundle wb WHERE wb.id = wbw.widgets_bundle_id) != - (SELECT tenant_id FROM widget_type WHERE id = wbw.widget_type_id); + (SELECT tenant_id FROM widget_type wt WHERE wt.id = wbw.widget_type_id); From 04cfa2021dcc989d6130a43334a435dfaeb2f0d9 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 17 Oct 2023 18:50:42 +0300 Subject: [PATCH 06/29] Fixed edge controller tests --- .../edge/rpc/processor/BaseEdgeProcessor.java | 3 ++- .../server/controller/EdgeControllerTest.java | 11 +++++++++-- .../server/controller/EdgeEventControllerTest.java | 13 ++++++++++--- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index 2d86c1e184..a888ddc27b 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -309,7 +309,8 @@ public abstract class BaseEdgeProcessor { EdgeEventActionType action, EntityId entityId, JsonNode body) { - ListenableFuture> future = attributesService.find(tenantId, edgeId, DataConstants.SERVER_SCOPE, DefaultDeviceStateService.ACTIVITY_STATE); + ListenableFuture> future = + attributesService.find(tenantId, edgeId, DataConstants.SERVER_SCOPE, DefaultDeviceStateService.ACTIVITY_STATE); return Futures.transformAsync(future, activeOpt -> { if (activeOpt.isEmpty()) { log.trace("Edge is not activated. Skipping event. tenantId [{}], edgeId [{}], type[{}], " + diff --git a/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java index 1afeaff0cb..b9e0b8cd6c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.controller; import com.datastax.oss.driver.api.core.uuid.Uuids; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -34,8 +35,10 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Primary; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.StringUtils; @@ -852,6 +855,11 @@ public class EdgeControllerTest extends AbstractControllerTest { Edge edge = doPost("/api/edge", constructEdge("Test Sync Edge", "test"), Edge.class); + // simulate edge activation + ObjectNode attributes = JacksonUtil.newObjectNode(); + attributes.put("active", true); + doPost("/api/plugins/telemetry/EDGE/" + edge.getId() + "/attributes/" + DataConstants.SERVER_SCOPE, attributes); + doPost("/api/edge/" + edge.getId().getId().toString() + "/device/" + savedDevice.getId().getId().toString(), Device.class); doPost("/api/edge/" + edge.getId().getId().toString() @@ -860,13 +868,12 @@ public class EdgeControllerTest extends AbstractControllerTest { EdgeImitator edgeImitator = new EdgeImitator(EDGE_HOST, EDGE_PORT, edge.getRoutingKey(), edge.getSecret()); edgeImitator.ignoreType(UserCredentialsUpdateMsg.class); - edgeImitator.expectMessageAmount(25); + edgeImitator.expectMessageAmount(24); edgeImitator.connect(); assertThat(edgeImitator.waitForMessages()).as("await for messages on first connect").isTrue(); verifyFetchersMsgs(edgeImitator); // verify queue msgs - Assert.assertTrue(popRuleChainMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE, "Edge Root Rule Chain")); Assert.assertTrue(popDeviceProfileMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "default")); Assert.assertTrue(popDeviceMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "Test Sync Edge Device 1")); Assert.assertTrue(popAssetProfileMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "test")); diff --git a/application/src/test/java/org/thingsboard/server/controller/EdgeEventControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdgeEventControllerTest.java index 04842df6a0..3303c64c27 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdgeEventControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdgeEventControllerTest.java @@ -16,6 +16,7 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import org.awaitility.Awaitility; import org.junit.After; @@ -26,6 +27,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.TestPropertySource; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.edge.Edge; @@ -86,6 +89,11 @@ public class EdgeEventControllerTest extends AbstractControllerTest { Edge edge = constructEdge("TestEdge", "default"); edge = doPost("/api/edge", edge, Edge.class); + // simulate edge activation + ObjectNode attributes = JacksonUtil.newObjectNode(); + attributes.put("active", true); + doPost("/api/plugins/telemetry/EDGE/" + edge.getId() + "/attributes/" + DataConstants.SERVER_SCOPE, attributes); + Device device = constructDevice("TestDevice", "default"); Device savedDevice = doPost("/api/device", device, Device.class); @@ -99,14 +107,13 @@ public class EdgeEventControllerTest extends AbstractControllerTest { EntityRelation relation = new EntityRelation(savedAsset.getId(), savedDevice.getId(), EntityRelation.CONTAINS_TYPE); - awaitForNumberOfEdgeEvents(edgeId, 3); + awaitForNumberOfEdgeEvents(edgeId, 2); doPost("/api/relation", relation); - awaitForNumberOfEdgeEvents(edgeId, 4); + awaitForNumberOfEdgeEvents(edgeId, 3); List edgeEvents = findEdgeEvents(edgeId); - Assert.assertTrue(popEdgeEvent(edgeEvents, EdgeEventType.RULE_CHAIN)); // root rule chain Assert.assertTrue(popEdgeEvent(edgeEvents, EdgeEventType.DEVICE)); // TestDevice Assert.assertTrue(popEdgeEvent(edgeEvents, EdgeEventType.ASSET)); // TestAsset Assert.assertTrue(popEdgeEvent(edgeEvents, EdgeEventType.RELATION)); From cf32544e71f7dd3bd43059fd2b4328c1a3747dc2 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 17 Oct 2023 23:20:00 +0200 Subject: [PATCH 07/29] added SslUtil and tests --- common/util/pom.xml | 8 ++ .../org/thingsboard/common/util/SslUtil.java | 105 ++++++++++++++++++ .../credentials/CertPemCredentials.java | 82 ++------------ .../credentials/CertPemCredentialsTest.java | 51 ++++++--- .../src/test/resources/pem/ec_cert.pem | 13 +++ .../src/test/resources/pem/rsa_cert.pem | 32 ++++++ .../test/resources/pem/rsa_encrypted_cert.pem | 22 ++++ .../pem/rsa_encrypted_traditional_cert.pem | 22 ++++ 8 files changed, 245 insertions(+), 90 deletions(-) create mode 100644 common/util/src/main/java/org/thingsboard/common/util/SslUtil.java create mode 100644 rule-engine/rule-engine-components/src/test/resources/pem/ec_cert.pem create mode 100644 rule-engine/rule-engine-components/src/test/resources/pem/rsa_cert.pem create mode 100644 rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_cert.pem create mode 100644 rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_traditional_cert.pem diff --git a/common/util/pom.xml b/common/util/pom.xml index 14ae1748b2..bce6a6fc80 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -36,6 +36,14 @@ + + org.bouncycastle + bcprov-jdk15on + + + org.bouncycastle + bcpkix-jdk15on + org.springframework spring-core diff --git a/common/util/src/main/java/org/thingsboard/common/util/SslUtil.java b/common/util/src/main/java/org/thingsboard/common/util/SslUtil.java new file mode 100644 index 0000000000..889436671a --- /dev/null +++ b/common/util/src/main/java/org/thingsboard/common/util/SslUtil.java @@ -0,0 +1,105 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.common.util; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.openssl.PEMDecryptorProvider; +import org.bouncycastle.openssl.PEMEncryptedKeyPair; +import org.bouncycastle.openssl.PEMKeyPair; +import org.bouncycastle.openssl.PEMParser; +import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter; +import org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder; +import org.bouncycastle.operator.InputDecryptorProvider; +import org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo; +import org.bouncycastle.pkcs.jcajce.JcePKCSPBEInputDecryptorProviderBuilder; +import org.thingsboard.server.common.data.StringUtils; + +import java.io.StringReader; +import java.security.PrivateKey; +import java.security.Security; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.List; + +@Slf4j +public class SslUtil { + + public static final char[] EMPTY_PASS = {}; + + public static final BouncyCastleProvider DEFAULT_PROVIDER = new BouncyCastleProvider(); + + static { + if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { + Security.addProvider(DEFAULT_PROVIDER); + } + } + + private SslUtil() { + } + + @SneakyThrows + public static List readCertFile(String fileContent) { + List certificates = new ArrayList<>(); + JcaX509CertificateConverter certConverter = new JcaX509CertificateConverter(); + try (PEMParser pemParser = new PEMParser(new StringReader(fileContent))) { + Object object; + while ((object = pemParser.readObject()) != null) { + if (object instanceof X509CertificateHolder) { + X509Certificate x509Cert = certConverter.getCertificate((X509CertificateHolder) object); + certificates.add(x509Cert); + } + } + } + return certificates; + } + + @SneakyThrows + public static PrivateKey readPrivateKey(String fileContent, String passStr) { + char[] password = StringUtils.isEmpty(passStr) ? EMPTY_PASS : passStr.toCharArray(); + + PrivateKey privateKey = null; + JcaPEMKeyConverter keyConverter = new JcaPEMKeyConverter(); + if (StringUtils.isNotEmpty(fileContent)) { + try (PEMParser pemParser = new PEMParser(new StringReader(fileContent))) { + Object object; + while ((object = pemParser.readObject()) != null) { + if (object instanceof PEMEncryptedKeyPair) { + PEMDecryptorProvider decProv = new JcePEMDecryptorProviderBuilder().build(password); + privateKey = keyConverter.getKeyPair(((PEMEncryptedKeyPair) object).decryptKeyPair(decProv)).getPrivate(); + break; + } else if (object instanceof PKCS8EncryptedPrivateKeyInfo) { + InputDecryptorProvider decProv = + new JcePKCSPBEInputDecryptorProviderBuilder().setProvider(DEFAULT_PROVIDER).build(password); + privateKey = keyConverter.getPrivateKey(((PKCS8EncryptedPrivateKeyInfo) object).decryptPrivateKeyInfo(decProv)); + break; + } else if (object instanceof PEMKeyPair) { + privateKey = keyConverter.getKeyPair((PEMKeyPair) object).getPrivate(); + break; + } else if (object instanceof PrivateKeyInfo) { + privateKey = keyConverter.getPrivateKey((PrivateKeyInfo) object); + } + } + } + } + return privateKey; + } + +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java index c17f71bc33..3824e71459 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java @@ -20,35 +20,17 @@ import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; -import org.bouncycastle.cert.X509CertificateHolder; -import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.bouncycastle.openssl.PEMDecryptorProvider; -import org.bouncycastle.openssl.PEMEncryptedKeyPair; -import org.bouncycastle.openssl.PEMKeyPair; -import org.bouncycastle.openssl.PEMParser; -import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter; -import org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder; -import org.bouncycastle.operator.InputDecryptorProvider; -import org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo; -import org.bouncycastle.pkcs.PKCSException; -import org.bouncycastle.pkcs.jcajce.JcePKCSPBEInputDecryptorProviderBuilder; +import org.thingsboard.common.util.SslUtil; import org.thingsboard.server.common.data.StringUtils; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.TrustManagerFactory; -import java.io.IOException; -import java.io.StringReader; -import java.security.GeneralSecurityException; import java.security.KeyStore; import java.security.PrivateKey; -import java.security.Security; import java.security.cert.CertPath; import java.security.cert.Certificate; import java.security.cert.CertificateFactory; import java.security.cert.X509Certificate; -import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -61,16 +43,11 @@ public class CertPemCredentials implements ClientCredentials { public static final String X_509 = "X.509"; public static final String CERT_ALIAS_PREFIX = "cert-"; public static final String CA_CERT_CERT_ALIAS_PREFIX = "caCert-cert-"; + protected String caCert; private String cert; private String privateKey; - private String password; - - public CertPemCredentials() { - if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { - Security.addProvider(new BouncyCastleProvider()); - } - } + private String password = ""; @Override public CredentialsType getType() { @@ -95,7 +72,7 @@ public class CertPemCredentials implements ClientCredentials { } protected TrustManagerFactory createAndInitTrustManagerFactory() throws Exception { - List caCerts = readCertFile(caCert); + List caCerts = SslUtil.readCertFile(caCert); KeyStore caKeyStore = KeyStore.getInstance(KeyStore.getDefaultType()); caKeyStore.load(null, null); @@ -108,15 +85,15 @@ public class CertPemCredentials implements ClientCredentials { return trustManagerFactory; } - protected KeyManagerFactory createAndInitKeyManagerFactory() throws Exception { + private KeyManagerFactory createAndInitKeyManagerFactory() throws Exception { KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); kmf.init(loadKeyStore(), password.toCharArray()); return kmf; } - private KeyStore loadKeyStore() throws Exception { - List certificates = readCertFile(this.cert); - PrivateKey privateKey = readPrivateKey(this.privateKey, this.password); + protected KeyStore loadKeyStore() throws Exception { + List certificates = SslUtil.readCertFile(this.cert); + PrivateKey privateKey = SslUtil.readPrivateKey(this.privateKey, password); KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); keyStore.load(null); @@ -135,47 +112,4 @@ public class CertPemCredentials implements ClientCredentials { return keyStore; } - protected List readCertFile(String fileContent) throws IOException, GeneralSecurityException { - List certificates = new ArrayList<>(); - JcaX509CertificateConverter certConverter = new JcaX509CertificateConverter(); - try (PEMParser pemParser = new PEMParser(new StringReader(fileContent))) { - Object object; - while ((object = pemParser.readObject()) != null) { - if (object instanceof X509CertificateHolder) { - X509Certificate x509Cert = certConverter.getCertificate((X509CertificateHolder) object); - certificates.add(x509Cert); - } - } - } - return certificates; - } - - protected PrivateKey readPrivateKey(String fileContent, String password) throws IOException, PKCSException { - PrivateKey privateKey = null; - JcaPEMKeyConverter keyConverter = new JcaPEMKeyConverter(); - - if (StringUtils.isNotEmpty(fileContent)) { - try (PEMParser pemParser = new PEMParser(new StringReader(fileContent))) { - Object object; - while ((object = pemParser.readObject()) != null) { - if (object instanceof PEMEncryptedKeyPair) { - PEMDecryptorProvider decProv = new JcePEMDecryptorProviderBuilder().build(password.toCharArray()); - privateKey = keyConverter.getKeyPair(((PEMEncryptedKeyPair) object).decryptKeyPair(decProv)).getPrivate(); - break; - } else if (object instanceof PKCS8EncryptedPrivateKeyInfo) { - InputDecryptorProvider decProv = - new JcePKCSPBEInputDecryptorProviderBuilder().setProvider(new BouncyCastleProvider()).build(password.toCharArray()); - privateKey = keyConverter.getPrivateKey(((PKCS8EncryptedPrivateKeyInfo) object).decryptPrivateKeyInfo(decProv)); - break; - } else if (object instanceof PEMKeyPair) { - privateKey = keyConverter.getKeyPair((PEMKeyPair) object).getPrivate(); - break; - } else if (object instanceof PrivateKeyInfo) { - privateKey = keyConverter.getPrivateKey((PrivateKeyInfo) object); - } - } - } - } - return privateKey; - } } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java index fd7f0d3378..2cecd1490d 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java @@ -22,28 +22,32 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.thingsboard.common.util.SslUtil; import java.io.File; import java.io.IOException; -import java.security.PrivateKey; +import java.security.Key; +import java.security.KeyStore; +import java.security.cert.Certificate; import java.security.cert.X509Certificate; import java.util.List; import java.util.stream.Stream; -public class CertPemCredentialsTest { +import static org.thingsboard.rule.engine.credentials.CertPemCredentials.CERT_ALIAS_PREFIX; +import static org.thingsboard.rule.engine.credentials.CertPemCredentials.PRIVATE_KEY_ALIAS; - private final CertPemCredentials credentials = new CertPemCredentials(); +public class CertPemCredentialsTest { private static final String PASS = "test"; private static final String EMPTY_PASS = ""; private static final String RSA = "RSA"; - private static final String ECDSA = "ECDSA"; + private static final String EC = "EC"; @Test public void testChainOfCertificates() throws Exception { String fileContent = fileContent("pem/tb-cloud-chain.pem"); - List x509Certificates = credentials.readCertFile(fileContent); + List x509Certificates = SslUtil.readCertFile(fileContent); Assert.assertEquals(4, x509Certificates.size()); Assert.assertEquals("CN=*.thingsboard.cloud, O=\"ThingsBoard, Inc.\", ST=New York, C=US", @@ -60,7 +64,7 @@ public class CertPemCredentialsTest { public void testSingleCertificate() throws Exception { String fileContent = fileContent("pem/tb-cloud.pem"); - List x509Certificates = credentials.readCertFile(fileContent); + List x509Certificates = SslUtil.readCertFile(fileContent); Assert.assertEquals(1, x509Certificates.size()); Assert.assertEquals("CN=*.thingsboard.cloud, O=\"ThingsBoard, Inc.\", ST=New York, C=US", @@ -71,26 +75,41 @@ public class CertPemCredentialsTest { public void testEmptyFileContent() throws Exception { String fileContent = fileContent("pem/empty.pem"); - List x509Certificates = credentials.readCertFile(fileContent); + List x509Certificates = SslUtil.readCertFile(fileContent); Assert.assertEquals(0, x509Certificates.size()); } - private static Stream testReadPrivateKey() { + private static Stream testLoadKeyStore() { return Stream.of( - Arguments.of("pem/rsa_key.pem", EMPTY_PASS, RSA), - Arguments.of("pem/rsa_encrypted_key.pem", PASS, RSA), - Arguments.of("pem/rsa_encrypted_traditional_key.pem", PASS, RSA), - Arguments.of("pem/ec_key.pem", EMPTY_PASS, ECDSA) + Arguments.of("pem/rsa_cert.pem", "pem/rsa_key.pem", EMPTY_PASS, RSA), + Arguments.of("pem/rsa_encrypted_cert.pem", "pem/rsa_encrypted_key.pem", PASS, RSA), + Arguments.of("pem/rsa_encrypted_traditional_cert.pem", "pem/rsa_encrypted_traditional_key.pem", PASS, RSA), + Arguments.of("pem/ec_cert.pem", "pem/ec_key.pem", EMPTY_PASS, EC) ); } @ParameterizedTest @MethodSource - public void testReadPrivateKey(String keyPath, String password, String algorithm) throws Exception { - PrivateKey privateKey = credentials.readPrivateKey(fileContent(keyPath), password); - Assertions.assertNotNull(privateKey); - Assertions.assertEquals(algorithm, privateKey.getAlgorithm()); + public void testLoadKeyStore(String certPath, String keyPath, String password, String algorithm) throws Exception { + CertPemCredentials certPemCredentials = new CertPemCredentials(); + String certContent = fileContent(certPath); + certPemCredentials.setCert(certContent); + certPemCredentials.setPrivateKey(fileContent(keyPath)); + certPemCredentials.setPassword(password); + KeyStore keyStore = certPemCredentials.loadKeyStore(); + Assertions.assertNotNull(keyStore); + Key key = keyStore.getKey(PRIVATE_KEY_ALIAS, password.toCharArray()); + Assertions.assertNotNull(key); + Assertions.assertEquals(algorithm, key.getAlgorithm()); + + List certs = SslUtil.readCertFile(certContent); + for (X509Certificate cert : certs) { + String alias = CERT_ALIAS_PREFIX + cert.getIssuerDN().getName(); + Certificate certificate = keyStore.getCertificate(alias); + Assertions.assertNotNull(certificate); + Assertions.assertEquals(new String(cert.getEncoded()), new String(certificate.getEncoded())); + } } private String fileContent(String fileName) throws IOException { diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/ec_cert.pem b/rule-engine/rule-engine-components/src/test/resources/pem/ec_cert.pem new file mode 100644 index 0000000000..f22f61d3a1 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/ec_cert.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIICCDCCAa2gAwIBAgIUGx/SZqIWza/i/gaKFUVIyTEu2oMwCgYIKoZIzj0EAwIw +WTELMAkGA1UEBhMCVUExDTALBgNVBAgMBEtZSVYxDTALBgNVBAcMBEtZSVYxCzAJ +BgNVBAoMAlRCMQswCQYDVQQLDAJUQjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIz +MTAxNjEyMjMyMVoXDTI0MTAxNTEyMjMyMVowWTELMAkGA1UEBhMCVUExDTALBgNV +BAgMBEtZSVYxDTALBgNVBAcMBEtZSVYxCzAJBgNVBAoMAlRCMQswCQYDVQQLDAJU +QjESMBAGA1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE +z4MgawieJfVc5zUOPiw5WFxfHGJf7dOMsHvudDxdOs27PXPbJfi09BVJ3+JjNxA2 +wQz9KUk877oWRYrN/e+MbKNTMFEwHQYDVR0OBBYEFDTV8VD3m+8IBQOBJ+V/bcbl +4preMB8GA1UdIwQYMBaAFDTV8VD3m+8IBQOBJ+V/bcbl4preMA8GA1UdEwEB/wQF +MAMBAf8wCgYIKoZIzj0EAwIDSQAwRgIhAOgIkl8j8m51W7pWlNUAuUnHnOVhVjGr +h8Rc6cbwTapKAiEA2CLrduTweXEF5fBRtWyOsG8c9af6+MWHKmwHL1IDw9Q= +-----END CERTIFICATE----- diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/rsa_cert.pem b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_cert.pem new file mode 100644 index 0000000000..f89fce7444 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFkzCCA3ugAwIBAgIUUQa3cWUVoF58dzg8ycb/y7SdCj8wDQYJKoZIhvcNAQEL +BQAwWTELMAkGA1UEBhMCVUExDTALBgNVBAgMBEtZSVYxDTALBgNVBAcMBEtZSVYx +CzAJBgNVBAoMAlRCMQswCQYDVQQLDAJUQjESMBAGA1UEAwwJbG9jYWxob3N0MB4X +DTIzMTAxMzEyMzcwMVoXDTI0MTAxMjEyMzcwMVowWTELMAkGA1UEBhMCVUExDTAL +BgNVBAgMBEtZSVYxDTALBgNVBAcMBEtZSVYxCzAJBgNVBAoMAlRCMQswCQYDVQQL +DAJUQjESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8A +MIICCgKCAgEAsHn27cH+pYFI0eJYer8ww29g/xlKgr9aarYlkILeXnBhPPHBCXG+ +FegeMpHa8FUPANIqYJiwM13altO6hMLPa0J7+nQhwF5NCbxzAdi/kU8ofhIwJH+K +gOsD3BKdR7Ua7KMDQFnGTFRR9ZxsuYZ/0AHuzPHwxSLUvvMbiWbu5P2FYMrEyyLo +uVVihZPkeBhcnI6SJRyCdMdMy282nWQ+47gAUI3cFa7dXxUcXvRbbToMNPTIDUy4 +VhxJYhL4T6ED0Ds7tZRsG71LcMfw2RQUgiS1FuYh+O7N8lUMukMy2/umQluM0+qB +CYWa2p1UCbVzlrW1qgKQm1Q8E91XSR9KL/zdO8m9/uNeI1jyJu6i1cibWR7gnh6J +ChLxouQlrBzuLzSz7PG8q1MOWi+oHYJWSvmsckbQDhwEsfhFrYVgndJdxnmlkzvS +1OP7RGSYXLfMF+ZxC2YEJiU65QACCl2IHknyNiL8Jg5ahXgZMNshyfvOv5RB5jnz +4vzRpGhUYCcyLzORT+5gY9ZYbX/51cOomQV1ryTTQs+zA8mfEVLjbbLqvYdI84LC +3chMdcOm8Z9U1xdb2FX/c724XDyPnQNy1PLggzqvOFZzLeey0nBVUWyVrcCydbS5 +PAvVoAucO8kqP6b7uB7QnDeGaCiAVF+9QaXxjyQEdLEu3z5JMM7uH4UCAwEAAaNT +MFEwHQYDVR0OBBYEFHXrT3L+O3kJ2xNZ4Lh1ThGG6M1vMB8GA1UdIwQYMBaAFHXr +T3L+O3kJ2xNZ4Lh1ThGG6M1vMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL +BQADggIBABFDqkTdxsJyu5L2x3WSpw4jw4vgJYlgUvTSeU8i54DaSzncLZdpWsqb +37LFHkvlquIfvOi9f9EBT2KuZwaajPQBNE4m7kLchoAv8Mc8a2EXhN2caXamnN3F +vWAb4QW/VHKKz2vWprfARwqQO58TEPgzU4FcW1lPpX2ULBeoS5kZDDEgyfaFZETF +FnsSb9E3/YuH6sJCu880kbW8BIyQmbUytrbn+16J/iaZBwc+iD49t2VBLDOsr1x4 +5qzxknG3h9wiz9ob9v6hWFfMpdiK12S0P5FVsUkCpxoae8jc8rPS7W3HaYowFjVR +OHOjtWy5/SV2rypKShjg9manf6iwGdTGkD0qoqsRs9JQFabjNR23IQv+1OUbrEVC +DbS65IjwLJlIZBX8JuJaU3I8zqj/9q7TtRDp1NCiG5W0NgipERRCciWaLJ+Fz6Lu +QzhI2ZOJrl49hmr6e0bsyNUv9l89WcbKm3/IC+V7o80uADYCOaz2jDGfKbvcPHzN +mTma8qVsjpcedttsvNMyZOsM/Rpk+dbChgReRVvcmzQV0izEvJJBWFr4HrfcM6Ev +sZrnUiT8ENUZqiK40d+T3Q6JheHwm+ENI1aUDkYCpoWZ/PzKe+Bj8lR8dPvmeVrc +eiwS37nMFO/5X7aIkszTouScNO99cN0UqPldfJo+8ZTbai5VFxGD +-----END CERTIFICATE----- diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_cert.pem b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_cert.pem new file mode 100644 index 0000000000..2f4e55a612 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_cert.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDkzCCAnugAwIBAgIUKAylzm/K5OfbXSjm1zY9bX1a8HQwDQYJKoZIhvcNAQEL +BQAwWTELMAkGA1UEBhMCVUExDTALBgNVBAgMBEtZSVYxDTALBgNVBAcMBEtZSVYx +CzAJBgNVBAoMAlRCMQswCQYDVQQLDAJUQjESMBAGA1UEAwwJbG9jYWxob3N0MB4X +DTIzMTAxNjIxMDQwNVoXDTI0MTAxNTIxMDQwNVowWTELMAkGA1UEBhMCVUExDTAL +BgNVBAgMBEtZSVYxDTALBgNVBAcMBEtZSVYxCzAJBgNVBAoMAlRCMQswCQYDVQQL +DAJUQjESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEA87nLliszEWml8QvyAC+H80NZCxf4TcG826NBOp0AUPJ8xQBHCzc1 +t1ohVm2/fn2VJZAYXG2xSVcHyXjjjv3iGLE2AIDbXh06/yFg4TVjlbrWrAHFehyN +FwrK8ez36oGLa3ZVq+mx1fLfBQw5mStbh09NXmKTzqP6m9ggKtt63cUwoWdUTemT +qrjryJd69LiJi+MVqtbKO2j30/lgAZmaHtbojl9EcvWfeXLb20TnXRIctaIS1VGo +SluzjbNQErdN/VRW4RAOP6UFsK0xID2EuLODBmAWnI49fXO/OS+u3Kd3suABE0o9 +slfDXqNTp0r5N0OoSAFcc4EsV3+9Gf+mqwIDAQABo1MwUTAdBgNVHQ4EFgQUhS5K +XQDxGvaBCpKY1de+JZl8zjYwHwYDVR0jBBgwFoAUhS5KXQDxGvaBCpKY1de+JZl8 +zjYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAxez4vLtBCBNM +l6AQghViNAR9iiwYMxUwKwlU+uZftRGnT+6dXgfTR3PV6LCfMMmtuNs0JTGy0ff8 +erbzfZxExvHfIFXCwepwTWawQhvRRn9GHOJXIzESDRRhsXoJDzd0JVOx0wWxp1cz +EUts+ZbKLoC+kIhsOGY+0a+sopeV2rMO5bUMpA8P0mKZlGynEGMLzKxz65E/IA9h +EQKpJjpvYfN+7eUkF6ZRXNV2LI/8BCoG6mOVoOMEXnloPwwBtOevoCB43U3sT9Er +WQWgZdbeI4gEyEqgMTibNogZZF0KW+5as3iv7avDd8pCgONvD0iwKSlvi9RNjiw8 +p6bwNmBcuA== +-----END CERTIFICATE----- diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_traditional_cert.pem b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_traditional_cert.pem new file mode 100644 index 0000000000..38ab2e42fe --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_traditional_cert.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDkzCCAnugAwIBAgIUIo+5l07ZrQR/LxEEmUbnn4yxCwIwDQYJKoZIhvcNAQEL +BQAwWTELMAkGA1UEBhMCVUExDTALBgNVBAgMBEtZSVYxDTALBgNVBAcMBEtZSVYx +CzAJBgNVBAoMAlRCMQswCQYDVQQLDAJUQjESMBAGA1UEAwwJbG9jYWxob3N0MB4X +DTIzMTAxNjIxMDMxNVoXDTI0MTAxNTIxMDMxNVowWTELMAkGA1UEBhMCVUExDTAL +BgNVBAgMBEtZSVYxDTALBgNVBAcMBEtZSVYxCzAJBgNVBAoMAlRCMQswCQYDVQQL +DAJUQjESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEA87nLliszEWml8QvyAC+H80NZCxf4TcG826NBOp0AUPJ8xQBHCzc1 +t1ohVm2/fn2VJZAYXG2xSVcHyXjjjv3iGLE2AIDbXh06/yFg4TVjlbrWrAHFehyN +FwrK8ez36oGLa3ZVq+mx1fLfBQw5mStbh09NXmKTzqP6m9ggKtt63cUwoWdUTemT +qrjryJd69LiJi+MVqtbKO2j30/lgAZmaHtbojl9EcvWfeXLb20TnXRIctaIS1VGo +SluzjbNQErdN/VRW4RAOP6UFsK0xID2EuLODBmAWnI49fXO/OS+u3Kd3suABE0o9 +slfDXqNTp0r5N0OoSAFcc4EsV3+9Gf+mqwIDAQABo1MwUTAdBgNVHQ4EFgQUhS5K +XQDxGvaBCpKY1de+JZl8zjYwHwYDVR0jBBgwFoAUhS5KXQDxGvaBCpKY1de+JZl8 +zjYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAiUTgjnsVIg90 +Dm+XSlscIPbZEj/mJanoFFfAfbVJz1DadygG9viVUMf3jVQBcsJGeBDckR2b3OHY +82cQVpdu3Heqld+gnfsyi8QBi7EdK4i0q8NVqFgpw83KxNm9xt7xrgHtxhE0kWfW +dpTgeIu0hFf0qLUObw/g8+0awBuxNY2crLtLXQM/dRgtv5Zt/DilW3jMLAE5wke+ +/HM4/emOJO6DSI9BC8iUsmNpIpq45267jcjpczNBo3ap7Bad+jM/paRDng9Uavvr +VCsaJFaL5HG6TtNXN60npBouOWnivPzUeuTI4PnjGRgdp3lgb0IuXbuwxIW6FVG/ +73RHc0gGOA== +-----END CERTIFICATE----- From c5c037b874a32519c79490e7d1f75ad8e832dd52 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 18 Oct 2023 08:47:03 +0300 Subject: [PATCH 08/29] Update AlarmControllerTest with latest business logic - no edge notification during alarm udpate --- .../thingsboard/server/controller/AlarmControllerTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java index 0e6eb16713..2f859055bd 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java @@ -136,7 +136,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(AlarmSeverity.MAJOR, updatedAlarm.getSeverity()); AlarmInfo foundAlarm = doGet("/api/alarm/info/" + updatedAlarm.getId(), AlarmInfo.class); - testNotifyEntityAllOneTime(foundAlarm, updatedAlarm.getId(), updatedAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, updatedAlarm.getId(), updatedAlarm.getOriginator(), tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.UPDATED); } @@ -153,7 +153,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(AlarmSeverity.MAJOR, updatedAlarm.getSeverity()); AlarmInfo foundAlarm = doGet("/api/alarm/info/" + updatedAlarm.getId(), AlarmInfo.class); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.UPDATED); alarm = updatedAlarm; From 9b6b161d37e79506f9895c53f4abd58d5dfdeda5 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 18 Oct 2023 10:15:51 +0300 Subject: [PATCH 09/29] Revert changes for getting alarms by ids --- ui-ngx/src/app/core/http/entity.service.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ui-ngx/src/app/core/http/entity.service.ts b/ui-ngx/src/app/core/http/entity.service.ts index cc1dd13345..b4eab83b08 100644 --- a/ui-ngx/src/app/core/http/entity.service.ts +++ b/ui-ngx/src/app/core/http/entity.service.ts @@ -240,9 +240,7 @@ export class EntityService { entityIds); break; case EntityType.ALARM: - observable = this.getEntitiesByIdsObservable( - (id) => this.alarmService.getAlarm(id, config), - entityIds); + console.error('Get Alarm Entity is not implemented!'); break; case EntityType.DEVICE_PROFILE: observable = this.getEntitiesByIdsObservable( From ef292c5cbb026328cfe7ed503f6a9805503cb216 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 18 Oct 2023 12:56:01 +0300 Subject: [PATCH 10/29] Fix delete queries in repositories --- .../server/dao/sql/alarm/EntityAlarmRepository.java | 4 +++- .../dao/sql/notification/NotificationRepository.java | 12 +++++++++--- .../notification/NotificationRequestRepository.java | 8 ++++++-- .../sql/notification/NotificationRuleRepository.java | 5 ++++- .../notification/NotificationTargetRepository.java | 5 ++++- .../notification/NotificationTemplateRepository.java | 5 ++++- 6 files changed, 30 insertions(+), 9 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java index f6250e98f0..2ed2364621 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java @@ -36,6 +36,8 @@ public interface EntityAlarmRepository extends JpaRepository findByRequestId(UUID requestId, Pageable pageable); @Transactional - int deleteByIdAndRecipientId(UUID id, UUID recipientId); + @Modifying + @Query("DELETE FROM NotificationEntity n WHERE n.id = :id AND n.recipientId = :recipientId") + int deleteByIdAndRecipientId(@Param("id") UUID id, @Param("recipientId") UUID recipientId); @Transactional - void deleteByRequestId(UUID requestId); + @Modifying + @Query("DELETE FROM NotificationEntity n WHERE n.requestId = :requestId") + void deleteByRequestId(@Param("requestId") UUID requestId); @Transactional - void deleteByRecipientId(UUID recipientId); + @Modifying + @Query("DELETE FROM NotificationEntity n WHERE n.recipientId = :recipientId") + void deleteByRecipientId(@Param("recipientId") UUID recipientId); @Modifying @Transactional diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationRequestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationRequestRepository.java index 126295616f..e7f3ac529b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationRequestRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationRequestRepository.java @@ -70,9 +70,13 @@ public interface NotificationRequestRepository extends JpaRepository findByTenantIdAndIdIn(UUID tenantId, List ids); @Transactional - void deleteByTenantId(UUID tenantId); + @Modifying + @Query("DELETE FROM NotificationTargetEntity t WHERE t.tenantId = :tenantId") + void deleteByTenantId(@Param("tenantId") UUID tenantId); long countByTenantId(UUID tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationTemplateRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationTemplateRepository.java index 73a25b98c3..3e68819b26 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationTemplateRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationTemplateRepository.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.notification; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; @@ -42,7 +43,9 @@ public interface NotificationTemplateRepository extends JpaRepository Date: Tue, 17 Oct 2023 12:03:24 +0200 Subject: [PATCH 11/29] ComponentLifecycleMsgProto implemented as replacement of raw byte encoding --- .../queue/DefaultTbClusterService.java | 6 +- .../queue/DefaultTbCoreConsumerService.java | 4 + .../DefaultTbRuleEngineConsumerService.java | 6 +- .../server/service/queue/ProtoUtils.java | 47 ++++++++++ .../service/queue/TbCoreConsumerStats.java | 2 + .../processing/AbstractConsumerService.java | 86 ++++++++++--------- .../server/service/queue/ProtoUtilsTest.java | 71 +++++++++++++++ common/cluster-api/src/main/proto/queue.proto | 58 ++++++++++++- .../server/common/data/EntityType.java | 1 + .../common/data/id/EntityIdFactory.java | 4 + .../data/plugin/ComponentLifecycleEvent.java | 1 + .../msg/plugin/ComponentLifecycleMsg.java | 14 +-- .../common/msg/plugin/RuleNodeUpdatedMsg.java | 6 +- 13 files changed, 245 insertions(+), 61 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java create mode 100644 application/src/test/java/org/thingsboard/server/service/queue/ProtoUtilsTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 53ef38d96a..937ee12fab 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -395,7 +395,7 @@ public class DefaultTbClusterService implements TbClusterService { } private void broadcast(ComponentLifecycleMsg msg) { - byte[] msgBytes = encodingService.encode(msg); + TransportProtos.ComponentLifecycleMsgProto componentLifecycleMsgProto = ProtoUtils.toProto(msg); TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); Set tbRuleEngineServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); EntityType entityType = msg.getEntityId().getEntityType(); @@ -413,7 +413,7 @@ public class DefaultTbClusterService implements TbClusterService { Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); for (String serviceId : tbCoreServices) { TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); - ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); + ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setComponentLifecycle(componentLifecycleMsgProto).build(); toCoreNfProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCoreMsg), null); toCoreNfs.incrementAndGet(); } @@ -422,7 +422,7 @@ public class DefaultTbClusterService implements TbClusterService { } for (String serviceId : tbRuleEngineServices) { TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); - ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); + ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycle(componentLifecycleMsgProto).build(); toRuleEngineProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toRuleEngineMsg), null); toRuleEngineNfs.incrementAndGet(); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index fd5a252cc5..bae1ea35a6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -346,7 +346,11 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService msg, TbCallback callback) throws Exception { ToRuleEngineNotificationMsg nfMsg = msg.getValue(); - if (nfMsg.getComponentLifecycleMsg() != null && !nfMsg.getComponentLifecycleMsg().isEmpty()) { + if (nfMsg.hasComponentLifecycle()) { + handleComponentLifecycleMsg(id, ProtoUtils.fromProto(nfMsg.getComponentLifecycle())); + callback.onSuccess(); + } else if (nfMsg.getComponentLifecycleMsg() != null && !nfMsg.getComponentLifecycleMsg().isEmpty()) { + //will be removed in 3.6.1 in favour of hasComponentLifecycle() handleComponentLifecycleMsg(id, nfMsg.getComponentLifecycleMsg()); callback.onSuccess(); } else if (nfMsg.hasFromDeviceRpcResponse()) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java b/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java new file mode 100644 index 0000000000..3406265747 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.UUID; + +public class ProtoUtils { + + public static TransportProtos.ComponentLifecycleMsgProto toProto(ComponentLifecycleMsg msg) { + return TransportProtos.ComponentLifecycleMsgProto.newBuilder() + .setTenantIdMSB(msg.getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(msg.getTenantId().getId().getLeastSignificantBits()) + .setEntityType(TransportProtos.EntityType.forNumber(msg.getEntityId().getEntityType().ordinal())) + .setEntityIdMSB(msg.getEntityId().getId().getMostSignificantBits()) + .setEntityIdLSB(msg.getEntityId().getId().getLeastSignificantBits()) + .setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(msg.getEvent().ordinal())) + .build(); + } + + public static ComponentLifecycleMsg fromProto(TransportProtos.ComponentLifecycleMsgProto proto) { + return new ComponentLifecycleMsg( + TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), + EntityIdFactory.getByTypeAndUuid(proto.getEntityTypeValue(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())), + ComponentLifecycleEvent.values()[proto.getEventValue()] + ); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java index 4d195a0dec..9107159615 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java @@ -168,6 +168,8 @@ public class TbCoreConsumerStats { toCoreNfSubscriptionServiceCounter.increment(); } else if (msg.hasFromDeviceRpcResponse()) { toCoreNfDeviceRpcResponseCounter.increment(); + } else if (msg.hasComponentLifecycle()) { + toCoreNfComponentLifecycleCounter.increment(); } else if (!msg.getComponentLifecycleMsg().isEmpty()) { toCoreNfComponentLifecycleCounter.increment(); } else if (!msg.getEdgeEventUpdateMsg().isEmpty()) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index b59086a350..591bc058c1 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -166,55 +166,57 @@ public abstract class AbstractConsumerService actorMsgOpt = encodingService.decode(nfMsg.toByteArray()); - if (actorMsgOpt.isPresent()) { - TbActorMsg actorMsg = actorMsgOpt.get(); - if (actorMsg instanceof ComponentLifecycleMsg) { - ComponentLifecycleMsg componentLifecycleMsg = (ComponentLifecycleMsg) actorMsg; - log.debug("[{}][{}][{}] Received Lifecycle event: {}", componentLifecycleMsg.getTenantId(), componentLifecycleMsg.getEntityId().getEntityType(), - componentLifecycleMsg.getEntityId(), componentLifecycleMsg.getEvent()); - if (EntityType.TENANT_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { - TenantProfileId tenantProfileId = new TenantProfileId(componentLifecycleMsg.getEntityId().getId()); - tenantProfileCache.evict(tenantProfileId); + actorMsgOpt.ifPresent(tbActorMsg -> handleComponentLifecycleMsg(id, tbActorMsg)); + } + + protected void handleComponentLifecycleMsg(UUID id, TbActorMsg actorMsg) { + if (actorMsg instanceof ComponentLifecycleMsg) { + ComponentLifecycleMsg componentLifecycleMsg = (ComponentLifecycleMsg) actorMsg; + log.debug("[{}][{}][{}] Received Lifecycle event: {}", componentLifecycleMsg.getTenantId(), componentLifecycleMsg.getEntityId().getEntityType(), + componentLifecycleMsg.getEntityId(), componentLifecycleMsg.getEvent()); + if (EntityType.TENANT_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { + TenantProfileId tenantProfileId = new TenantProfileId(componentLifecycleMsg.getEntityId().getId()); + tenantProfileCache.evict(tenantProfileId); + if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.UPDATED)) { + apiUsageStateService.onTenantProfileUpdate(tenantProfileId); + } + } else if (EntityType.TENANT.equals(componentLifecycleMsg.getEntityId().getEntityType())) { + if (TenantId.SYS_TENANT_ID.equals(componentLifecycleMsg.getTenantId())) { + jwtSettingsService.ifPresent(JwtSettingsService::reloadJwtSettings); + return; + } else { + tenantProfileCache.evict(componentLifecycleMsg.getTenantId()); + partitionService.removeTenant(componentLifecycleMsg.getTenantId()); if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.UPDATED)) { - apiUsageStateService.onTenantProfileUpdate(tenantProfileId); - } - } else if (EntityType.TENANT.equals(componentLifecycleMsg.getEntityId().getEntityType())) { - if (TenantId.SYS_TENANT_ID.equals(componentLifecycleMsg.getTenantId())) { - jwtSettingsService.ifPresent(JwtSettingsService::reloadJwtSettings); - return; - } else { - tenantProfileCache.evict(componentLifecycleMsg.getTenantId()); - partitionService.removeTenant(componentLifecycleMsg.getTenantId()); - if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.UPDATED)) { - apiUsageStateService.onTenantUpdate(componentLifecycleMsg.getTenantId()); - } else if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.DELETED)) { - apiUsageStateService.onTenantDelete((TenantId) componentLifecycleMsg.getEntityId()); - } - } - } else if (EntityType.DEVICE_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { - deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceProfileId(componentLifecycleMsg.getEntityId().getId())); - } else if (EntityType.DEVICE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { - deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceId(componentLifecycleMsg.getEntityId().getId())); - } else if (EntityType.ASSET_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { - assetProfileCache.evict(componentLifecycleMsg.getTenantId(), new AssetProfileId(componentLifecycleMsg.getEntityId().getId())); - } else if (EntityType.ASSET.equals(componentLifecycleMsg.getEntityId().getEntityType())) { - assetProfileCache.evict(componentLifecycleMsg.getTenantId(), new AssetId(componentLifecycleMsg.getEntityId().getId())); - } else if (EntityType.ENTITY_VIEW.equals(componentLifecycleMsg.getEntityId().getEntityType())) { - actorContext.getTbEntityViewService().onComponentLifecycleMsg(componentLifecycleMsg); - } else if (EntityType.API_USAGE_STATE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { - apiUsageStateService.onApiUsageStateUpdate(componentLifecycleMsg.getTenantId()); - } else if (EntityType.CUSTOMER.equals(componentLifecycleMsg.getEntityId().getEntityType())) { - if (componentLifecycleMsg.getEvent() == ComponentLifecycleEvent.DELETED) { - apiUsageStateService.onCustomerDelete((CustomerId) componentLifecycleMsg.getEntityId()); + apiUsageStateService.onTenantUpdate(componentLifecycleMsg.getTenantId()); + } else if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.DELETED)) { + apiUsageStateService.onTenantDelete((TenantId) componentLifecycleMsg.getEntityId()); } } - eventPublisher.publishEvent(componentLifecycleMsg); + } else if (EntityType.DEVICE_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { + deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceProfileId(componentLifecycleMsg.getEntityId().getId())); + } else if (EntityType.DEVICE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { + deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceId(componentLifecycleMsg.getEntityId().getId())); + } else if (EntityType.ASSET_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { + assetProfileCache.evict(componentLifecycleMsg.getTenantId(), new AssetProfileId(componentLifecycleMsg.getEntityId().getId())); + } else if (EntityType.ASSET.equals(componentLifecycleMsg.getEntityId().getEntityType())) { + assetProfileCache.evict(componentLifecycleMsg.getTenantId(), new AssetId(componentLifecycleMsg.getEntityId().getId())); + } else if (EntityType.ENTITY_VIEW.equals(componentLifecycleMsg.getEntityId().getEntityType())) { + actorContext.getTbEntityViewService().onComponentLifecycleMsg(componentLifecycleMsg); + } else if (EntityType.API_USAGE_STATE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { + apiUsageStateService.onApiUsageStateUpdate(componentLifecycleMsg.getTenantId()); + } else if (EntityType.CUSTOMER.equals(componentLifecycleMsg.getEntityId().getEntityType())) { + if (componentLifecycleMsg.getEvent() == ComponentLifecycleEvent.DELETED) { + apiUsageStateService.onCustomerDelete((CustomerId) componentLifecycleMsg.getEntityId()); + } } - log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg); - actorContext.tellWithHighPriority(actorMsg); + eventPublisher.publishEvent(componentLifecycleMsg); } + log.trace("[{}] Forwarding component lifecycle message to App Actor {}", id, actorMsg); + actorContext.tellWithHighPriority(actorMsg); } protected abstract void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) throws Exception; diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ProtoUtilsTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ProtoUtilsTest.java new file mode 100644 index 0000000000..c63055a6f1 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/queue/ProtoUtilsTest.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.gen.transport.TransportProtos; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +class ProtoUtilsTest { + + TenantId tenantId = TenantId.fromUUID(UUID.fromString("35e10f77-16e7-424d-ae46-ee780f87ac4f")); + EntityId entityId = new RuleChainId(UUID.fromString("c640b635-4f0f-41e6-b10b-25a86003094e")); + @Test + void toProtoComponentLifecycleMsg() { + ComponentLifecycleMsg msg = new ComponentLifecycleMsg(tenantId, entityId, ComponentLifecycleEvent.UPDATED); + + TransportProtos.ComponentLifecycleMsgProto proto = ProtoUtils.toProto(msg); + + assertThat(proto).as("to proto").isEqualTo(TransportProtos.ComponentLifecycleMsgProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setEntityType(TransportProtos.EntityType.forNumber(entityId.getEntityType().ordinal())) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) + .setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(ComponentLifecycleEvent.UPDATED.ordinal())) + .build() + ); + + assertThat(ProtoUtils.fromProto(proto)).as("from proto").isEqualTo(msg); + } + + @Test + void fromProtoComponentLifecycleMsg() { + TransportProtos.ComponentLifecycleMsgProto proto = TransportProtos.ComponentLifecycleMsgProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setEntityType(TransportProtos.EntityType.forNumber(entityId.getEntityType().ordinal())) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) + .setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(ComponentLifecycleEvent.STARTED.ordinal())) + .build(); + + ComponentLifecycleMsg msg = ProtoUtils.fromProto(proto); + + assertThat(msg).as("from proto").isEqualTo( + new ComponentLifecycleMsg(tenantId, entityId, ComponentLifecycleEvent.STARTED)); + + assertThat(ProtoUtils.toProto(msg)).as("to proto").isEqualTo(proto); + } + +} \ No newline at end of file diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 60dfcae1de..d8439d0cd0 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -20,6 +20,38 @@ package transport; option java_package = "org.thingsboard.server.gen.transport"; option java_outer_classname = "TransportProtos"; +/** + * Common data structures + */ +enum EntityType { + TENANT = 0; + CUSTOMER = 1; + USER = 2; + DASHBOARD = 3; + ASSET = 4; + DEVICE = 5; + ALARM = 6; + RULE_CHAIN = 7; + RULE_NODE = 8; + ENTITY_VIEW = 9; + WIDGETS_BUNDLE = 10; + WIDGET_TYPE = 11; + TENANT_PROFILE = 12; + DEVICE_PROFILE = 13; + ASSET_PROFILE = 14; + API_USAGE_STATE = 15; + TB_RESOURCE = 16; + OTA_PACKAGE = 17; + EDGE = 18; + RPC = 19; + QUEUE = 20; + NOTIFICATION_TARGET = 21; + NOTIFICATION_TEMPLATE = 22; + NOTIFICATION_REQUEST = 23; + NOTIFICATION = 24; + NOTIFICATION_RULE = 25; +} + /** * Service Discovery Data Structures; */ @@ -731,6 +763,25 @@ message FromDeviceRPCResponseProto { int32 error = 4; } +enum ComponentLifecycleEvent { + CREATED = 0; + STARTED = 1; + ACTIVATED = 2; + SUSPENDED = 3; + UPDATED = 4; + STOPPED = 5; + DELETED = 6; +} + +message ComponentLifecycleMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + EntityType entityType = 3; + int64 entityIdMSB = 4; + int64 entityIdLSB = 5; + ComponentLifecycleEvent event = 6; +} + message EdgeNotificationMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; @@ -980,10 +1031,11 @@ message ToCoreMsg { } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ +/* Please, adjust the TbCoreConsumerStats when modifying the ToCoreNotificationMsg */ message ToCoreNotificationMsg { LocalSubscriptionServiceMsgProto toLocalSubscriptionServiceMsg = 1; FromDeviceRPCResponseProto fromDeviceRpcResponse = 2; - bytes componentLifecycleMsg = 3; + bytes componentLifecycleMsg = 3; //will be removed in 3.6.1 in favour of ComponentLifecycleMsgProto bytes edgeEventUpdateMsg = 4; QueueUpdateMsg queueUpdateMsg = 5; QueueDeleteMsg queueDeleteMsg = 6; @@ -992,6 +1044,7 @@ message ToCoreNotificationMsg { bytes fromEdgeSyncResponseMsg = 9; SubscriptionMgrMsgProto toSubscriptionMgrMsg = 10; NotificationRuleProcessorMsg notificationRuleProcessorMsg = 11; + ComponentLifecycleMsgProto componentLifecycle = 12; } /* Messages that are handled by ThingsBoard RuleEngine Service */ @@ -1004,10 +1057,11 @@ message ToRuleEngineMsg { } message ToRuleEngineNotificationMsg { - bytes componentLifecycleMsg = 1; + bytes componentLifecycleMsg = 1; // will be removed in 3.6.1 in favour of ComponentLifecycleMsgProto FromDeviceRPCResponseProto fromDeviceRpcResponse = 2; QueueUpdateMsg queueUpdateMsg = 3; QueueDeleteMsg queueDeleteMsg = 4; + ComponentLifecycleMsgProto componentLifecycle = 5; } /* Messages that are handled by ThingsBoard Transport Service */ diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index 8ca6585718..c4344e8789 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -26,6 +26,7 @@ import java.util.stream.Collectors; * @author Andrew Shvayka */ public enum EntityType { + // In sync with EntityType proto TENANT, CUSTOMER, USER, diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index 0cdf3ad1eb..da92eb65d6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -29,6 +29,10 @@ public class EntityIdFactory { return getByTypeAndUuid(EntityType.values()[type], UUID.fromString(uuid)); } + public static EntityId getByTypeAndUuid(int type, UUID uuid) { + return getByTypeAndUuid(EntityType.values()[type], uuid); + } + public static EntityId getByTypeAndUuid(String type, String uuid) { return getByTypeAndUuid(EntityType.valueOf(type), UUID.fromString(uuid)); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java index a7ae9599ea..969baee088 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java @@ -21,5 +21,6 @@ import java.io.Serializable; * @author Andrew Shvayka */ public enum ComponentLifecycleEvent implements Serializable { + // In sync with ComponentLifecycleEvent proto CREATED, STARTED, ACTIVATED, SUSPENDED, UPDATED, STOPPED, DELETED } \ No newline at end of file diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index b25a8cd3ea..f110a6f209 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -15,8 +15,7 @@ */ package org.thingsboard.server.common.msg.plugin; -import lombok.Getter; -import lombok.ToString; +import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; @@ -31,21 +30,12 @@ import java.util.Optional; /** * @author Andrew Shvayka */ -@ToString +@Data public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { - @Getter private final TenantId tenantId; - @Getter private final EntityId entityId; - @Getter private final ComponentLifecycleEvent event; - public ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event) { - this.tenantId = tenantId; - this.entityId = entityId; - this.event = event; - } - public Optional getRuleChainId() { return entityId.getEntityType() == EntityType.RULE_CHAIN ? Optional.of((RuleChainId) entityId) : Optional.empty(); } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/RuleNodeUpdatedMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/RuleNodeUpdatedMsg.java index 0a5dfa484f..d912aaa29c 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/RuleNodeUpdatedMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/RuleNodeUpdatedMsg.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.msg.plugin; +import lombok.EqualsAndHashCode; import lombok.ToString; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -23,8 +24,11 @@ import org.thingsboard.server.common.msg.MsgType; /** * @author Andrew Shvayka + * This class used only to tell local rule-node actor like 'existing.getSelfActor().tellWithHighPriority(new RuleNodeUpdatedMs( ...' + * Never serialized to/from proto, otherwise you need to change proto mappers in ProtoUtils class */ -@ToString +@ToString(callSuper = true) +@EqualsAndHashCode(callSuper = true) public class RuleNodeUpdatedMsg extends ComponentLifecycleMsg { public RuleNodeUpdatedMsg(TenantId tenantId, EntityId entityId) { From 45fd970898930b91a839fcf4fd7f847f00163454 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 18 Sep 2023 12:49:19 +0300 Subject: [PATCH 12/29] Refactoring in progress --- .../DefaultTbRuleEngineConsumerService.java | 201 ++++++++--------- .../queue/TbQueueConsumerLauncher.java | 37 ++++ .../queue/TbQueueConsumerManagerTask.java | 17 ++ .../queue/TbRuleEngineConsumerStats.java | 5 +- .../TbRuleEngineQueueConsumerManager.java | 205 ++++++++++++++++++ .../provider/TbRuleEngineQueueFactory.java | 1 - 6 files changed, 356 insertions(+), 110 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerLauncher.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerManagerTask.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineQueueConsumerManager.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index c715b7b03c..ae04de456b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -68,6 +68,7 @@ import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrateg import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; import org.thingsboard.server.service.stats.RuleEngineStatisticsService; +import org.threadly.concurrent.wrapper.KeyDistributedExecutor; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -116,12 +117,12 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< private final QueueService queueService; private final TbQueueProducerProvider producerProvider; private final TbQueueAdmin queueAdmin; - private final ConcurrentMap>> consumers = new ConcurrentHashMap<>(); - private final ConcurrentMap consumerConfigurations = new ConcurrentHashMap<>(); - private final ConcurrentMap consumerStats = new ConcurrentHashMap<>(); - private final ConcurrentMap topicsConsumerPerPartition = new ConcurrentHashMap<>(); - final ExecutorService submitExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-submit")); - final ScheduledExecutorService repartitionExecutor = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-repartition")); + + private final ConcurrentMap consumerMap = new ConcurrentHashMap<>(); + private final ExecutorService consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer")); + private final ExecutorService submitExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-submit")); + private final ScheduledExecutorService repartitionExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-repartition")); + public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory, TbRuleEngineSubmitStrategyFactory submitStrategyFactory, @@ -153,7 +154,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @PostConstruct public void init() { - super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer"); + super.init("tb-rule-engine-notifications-consumer"); // TODO: restore init of the main consumer? List queues = queueService.findAllQueues(); for (Queue configuration : queues) { if (partitionService.isManagedByCurrentService(configuration.getTenantId())) { @@ -163,19 +164,14 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } private void initConsumer(Queue configuration) { - QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, configuration); - consumerConfigurations.putIfAbsent(queueKey, configuration); - consumerStats.putIfAbsent(queueKey, new TbRuleEngineConsumerStats(configuration, statsFactory)); - if (!configuration.isConsumerPerPartition()) { - consumers.computeIfAbsent(queueKey, queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration)); - } else { - topicsConsumerPerPartition.computeIfAbsent(queueKey, k -> new TbTopicWithConsumerPerPartition(k.getQueueName())); - } + consumerMap.computeIfAbsent(new QueueKey(ServiceType.TB_RULE_ENGINE, configuration), + key -> new TbRuleEngineQueueConsumerManager(repartitionExecutor, consumersExecutor, statsFactory, tbRuleEngineQueueFactory, key)).init(configuration); } @PreDestroy public void stop() { super.destroy(); + consumersExecutor.shutdownNow(); // TODO: shutdown or shutdownNow? submitExecutor.shutdownNow(); repartitionExecutor.shutdownNow(); } @@ -183,119 +179,107 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { if (event.getServiceType().equals(getServiceType())) { - event.getPartitionsMap().forEach((queueKey, partitions) -> { - String serviceQueue = queueKey.getQueueName(); - log.info("[{}] Subscribing to partitions: {}", serviceQueue, partitions); - Queue configuration = consumerConfigurations.get(queueKey); - if (configuration == null) { - return; - } - if (!configuration.isConsumerPerPartition()) { - consumers.get(queueKey).subscribe(partitions); - } else { - log.info("[{}] Subscribing consumer per partition: {}", serviceQueue, partitions); - subscribeConsumerPerPartition(queueKey, partitions); - } - }); - } - } - - void subscribeConsumerPerPartition(QueueKey queue, Set partitions) { - topicsConsumerPerPartition.get(queue).getSubscribeQueue().add(partitions); - scheduleTopicRepartition(queue); - } - - private void scheduleTopicRepartition(QueueKey queue) { - repartitionExecutor.schedule(() -> repartitionTopicWithConsumerPerPartition(queue), 1, TimeUnit.SECONDS); - } - - void repartitionTopicWithConsumerPerPartition(final QueueKey queueKey) { - if (stopped) { - return; - } - TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.get(queueKey); - java.util.Queue> subscribeQueue = tbTopicWithConsumerPerPartition.getSubscribeQueue(); - if (subscribeQueue.isEmpty()) { - return; - } - if (tbTopicWithConsumerPerPartition.getLock().tryLock()) { - try { - Set partitions = null; - while (!subscribeQueue.isEmpty()) { - partitions = subscribeQueue.poll(); - } - if (partitions == null) { - return; - } - - Set addedPartitions = new HashSet<>(partitions); - ConcurrentMap>> consumers = tbTopicWithConsumerPerPartition.getConsumers(); - addedPartitions.removeAll(consumers.keySet()); - log.info("calculated addedPartitions {}", addedPartitions); - - Set removedPartitions = new HashSet<>(consumers.keySet()); - removedPartitions.removeAll(partitions); - log.info("calculated removedPartitions {}", removedPartitions); - - removedPartitions.forEach((tpi) -> { - removeConsumerForTopicByTpi(queueKey.getQueueName(), consumers, tpi); - }); - - addedPartitions.forEach((tpi) -> { - log.info("[{}] Adding consumer for topic: {}", queueKey, tpi); - Queue configuration = consumerConfigurations.get(queueKey); - TbQueueConsumer> consumer = tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration); - consumers.put(tpi, consumer); - launchConsumer(consumer, consumerConfigurations.get(queueKey), consumerStats.get(queueKey), "" + queueKey + "-" + tpi.getPartition().orElse(-999999)); - consumer.subscribe(Collections.singleton(tpi)); - }); - } finally { - tbTopicWithConsumerPerPartition.getLock().unlock(); + var consumer = consumerMap.get(event.getQueueKey()); + if (consumer != null) { + consumer.subscribe(event); + } else { + log.warn("Received invalid partition change event for {} that is not managed by this service", event.getQueueKey()); } - } else { - scheduleTopicRepartition(queueKey); //reschedule later } - } +// void subscribeConsumerPerPartition(QueueKey queue, Set partitions) { +// topicsConsumerPerPartition.get(queue).getSubscribeQueue().add(partitions); +// scheduleTopicRepartition(queue); +// } +// +// private void scheduleTopicRepartition(QueueKey queue) { +// repartitionExecutor.schedule(() -> repartitionTopicWithConsumerPerPartition(queue), 1, TimeUnit.SECONDS); +// } +// +// void repartitionTopicWithConsumerPerPartition(final QueueKey queueKey) { +// if (stopped) { +// return; +// } +// TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.get(queueKey); +// java.util.Queue> subscribeQueue = tbTopicWithConsumerPerPartition.getSubscribeQueue(); +// if (subscribeQueue.isEmpty()) { +// return; +// } +// if (tbTopicWithConsumerPerPartition.getLock().tryLock()) { +// try { +// Set partitions = null; +// while (!subscribeQueue.isEmpty()) { +// partitions = subscribeQueue.poll(); +// } +// if (partitions == null) { +// return; +// } +// +// Set addedPartitions = new HashSet<>(partitions); +// ConcurrentMap>> consumers = tbTopicWithConsumerPerPartition.getConsumers(); +// addedPartitions.removeAll(consumers.keySet()); +// log.info("calculated addedPartitions {}", addedPartitions); +// +// Set removedPartitions = new HashSet<>(consumers.keySet()); +// removedPartitions.removeAll(partitions); +// log.info("calculated removedPartitions {}", removedPartitions); +// +// removedPartitions.forEach((tpi) -> { +// removeConsumerForTopicByTpi(queueKey.getQueueName(), consumers, tpi); +// }); +// +// addedPartitions.forEach((tpi) -> { +// log.info("[{}] Adding consumer for topic: {}", queueKey, tpi); +// Queue configuration = consumerConfigurations.get(queueKey); +// TbQueueConsumer> consumer = tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration); +// consumers.put(tpi, consumer); +// launchConsumer(consumer, queueKey, tpi.getFullTopicName(), queueKey + "-" + tpi.getPartition().orElse(-999999)); +// consumer.subscribe(Collections.singleton(tpi)); +// }); +// } finally { +// tbTopicWithConsumerPerPartition.getLock().unlock(); +// } +// } else { +// scheduleTopicRepartition(queueKey); //reschedule later +// } +// } + void removeConsumerForTopicByTpi(String queue, ConcurrentMap>> consumers, TopicPartitionInfo tpi) { log.info("[{}] Removing consumer for topic: {}", queue, tpi); - consumers.get(tpi).unsubscribe(); - consumers.remove(tpi); + consumers.remove(tpi).stop(); } @Override protected void launchMainConsumers() { - consumers.forEach((queue, consumer) -> launchConsumer(consumer, consumerConfigurations.get(queue), consumerStats.get(queue), queue.getQueueName())); + consumers.forEach((queue, consumer) -> launchConsumer(consumer, queue, queue, queue.getQueueName())); } @Override protected void stopMainConsumers() { - consumers.values().forEach(TbQueueConsumer::unsubscribe); - topicsConsumerPerPartition.values().forEach(tbTopicWithConsumerPerPartition -> tbTopicWithConsumerPerPartition.getConsumers().keySet() - .forEach((tpi) -> removeConsumerForTopicByTpi(tbTopicWithConsumerPerPartition.getTopic(), tbTopicWithConsumerPerPartition.getConsumers(), tpi))); + consumerMap.values().forEach(TbRuleEngineQueueConsumerManager::stop); } - void launchConsumer(TbQueueConsumer> consumer, Queue configuration, TbRuleEngineConsumerStats stats, String threadSuffix) { + void launchConsumer(TbQueueConsumer> consumer, QueueKey queueKey, Object consumerKey, String threadSuffix) { if (isReady) { - consumersExecutor.execute(() -> consumerLoop(consumer, configuration, stats, threadSuffix)); + log.info("[{}] Launching consumer", consumerKey); + consumersExecutor.execute(consumerKey, () -> consumerLoop(consumer, queueKey, threadSuffix)); } else { - scheduleLaunchConsumer(consumer, configuration, stats, threadSuffix); + scheduleLaunchConsumer(consumer, queueKey, consumerKey, threadSuffix); } } - private void scheduleLaunchConsumer(TbQueueConsumer> consumer, Queue configuration, TbRuleEngineConsumerStats stats, String threadSuffix) { + private void scheduleLaunchConsumer(TbQueueConsumer> consumer, QueueKey queueKey, Object consumerKey, String threadSuffix) { repartitionExecutor.schedule(() -> { - if (isReady) { - consumersExecutor.execute(() -> consumerLoop(consumer, configuration, stats, threadSuffix)); - } else { - scheduleLaunchConsumer(consumer, configuration, stats, threadSuffix); - } + launchConsumer(consumer, queueKey, consumerKey, threadSuffix); }, 10, TimeUnit.SECONDS); } - void consumerLoop(TbQueueConsumer> consumer, org.thingsboard.server.common.data.queue.Queue configuration, TbRuleEngineConsumerStats stats, String threadSuffix) { + void consumerLoop(TbQueueConsumer> consumer, QueueKey queueKey, String threadSuffix) { + Queue configuration = consumerConfigurations.get(queueKey); + TbRuleEngineConsumerStats stats = consumerStats.get(queueKey); updateCurrentThreadName(threadSuffix); + while (!stopped && !consumer.isStopped() && !consumer.isQueueDeleted()) { try { List> msgs = consumer.poll(configuration.getPollInterval()); @@ -347,7 +331,9 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } } - if (consumer.isQueueDeleted()) { + if (consumer.isStopped()) { + consumer.unsubscribe(); + } else if (consumer.isQueueDeleted()) { processQueueDeletion(configuration, consumer); } log.info("TB Rule Engine Consumer stopped."); @@ -460,20 +446,20 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< ReentrantLock lock = consumerPerPartition.getLock(); try { lock.lock(); - consumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::unsubscribe); + consumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::stop); } finally { lock.unlock(); } } else { TbQueueConsumer> consumer = consumers.remove(queueKey); - consumer.unsubscribe(); + consumer.stop(); } } initConsumer(queue); if (!queue.isConsumerPerPartition()) { - launchConsumer(consumers.get(queueKey), consumerConfigurations.get(queueKey), consumerStats.get(queueKey), queueName); + launchConsumer(consumers.get(queueKey), queueKey, queueKey, queueName); } } @@ -502,6 +488,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } } } + partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), new ArrayList<>(partitionService.getOtherServices(ServiceType.TB_RULE_ENGINE))); } private void forwardToRuleEngineActor(String queueName, TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerLauncher.java b/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerLauncher.java new file mode 100644 index 0000000000..f5688eb207 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerLauncher.java @@ -0,0 +1,37 @@ +package org.thingsboard.server.service.queue; + +import lombok.Data; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.QueueKey; + +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +@Data +public class TbQueueConsumerLauncher { + + private final TbQueueConsumer> consumer; + private volatile Future task; + + public void stop() { + this.consumer.stop(); + } + + public void awaitStopped() throws ExecutionException, InterruptedException, TimeoutException { + if (task != null) { + this.task.get(3, TimeUnit.MINUTES); + } + } + + public void subscribe(Set partitions) { + this.consumer.subscribe(partitions); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerManagerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerManagerTask.java new file mode 100644 index 0000000000..11316a9e05 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerManagerTask.java @@ -0,0 +1,17 @@ +package org.thingsboard.server.service.queue; + +import lombok.Data; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; + +import java.util.Set; + +@Data +public class TbQueueConsumerManagerTask { + + private final ComponentLifecycleEvent event; + private final Queue queue; + private final Set partitions; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index 2904c299ce..45ccbcf32a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsType; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; import java.util.ArrayList; @@ -66,8 +67,8 @@ public class TbRuleEngineConsumerStats { private final String queueName; private final TenantId tenantId; - public TbRuleEngineConsumerStats(Queue queue, StatsFactory statsFactory) { - this.queueName = queue.getName(); + public TbRuleEngineConsumerStats(QueueKey queue, StatsFactory statsFactory) { + this.queueName = queue.getQueueName(); this.tenantId = queue.getTenantId(); this.statsFactory = statsFactory; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineQueueConsumerManager.java new file mode 100644 index 0000000000..2c6d191c0a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineQueueConsumerManager.java @@ -0,0 +1,205 @@ +package org.thingsboard.server.service.queue; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +@Data +@Slf4j +public class TbRuleEngineQueueConsumerManager { + + private final ScheduledExecutorService scheduler; + private final ExecutorService consumerExecutor; + private final StatsFactory statsFactory; + private final TbRuleEngineQueueFactory queueFactory; + private final QueueKey key; + private final ReentrantLock lock = new ReentrantLock(); //NonfairSync + private final ConcurrentMap consumers = new ConcurrentHashMap<>(); + private final TbRuleEngineConsumerStats stats; + + private volatile Set partitions = Collections.emptySet(); + private volatile Queue queue; + private volatile TbQueueConsumerLauncher mainConsumer; + + private final java.util.Queue tasks = new ConcurrentLinkedQueue<>(); + + public TbRuleEngineQueueConsumerManager(ScheduledExecutorService scheduler, ExecutorService consumerExecutor, StatsFactory statsFactory, TbRuleEngineQueueFactory queueFactory, QueueKey key) { + this.scheduler = scheduler; + this.consumerExecutor = consumerExecutor; + this.statsFactory = statsFactory; + this.queueFactory = queueFactory; + this.key = key; + this.stats = new TbRuleEngineConsumerStats(key, statsFactory); + } + + public void init(Queue queue) { + processTask(new TbQueueConsumerManagerTask(ComponentLifecycleEvent.CREATED, queue, null)); + } + + private void processTask(TbQueueConsumerManagerTask todo) { + tasks.add(todo); + log.info("[{}] Adding task: {}", key, todo); + tryProcessTasks(); + } + + private void tryProcessTasks() { + consumerExecutor.submit(() -> { + if (lock.tryLock()) { + try { + TbQueueConsumerManagerTask lastUpdateTask = null; + while (!tasks.isEmpty()) { + TbQueueConsumerManagerTask task = tasks.poll(); + switch (task.getEvent()) { + case CREATED: + doInit(task.getQueue()); + break; + case UPDATED: + lastUpdateTask = task; + break; + case DELETED: + lastUpdateTask = null; + doDelete(); + break; + } + } + if (lastUpdateTask != null) { + doUpdate(lastUpdateTask.getQueue(), lastUpdateTask.getPartitions()); + } + } finally { + lock.unlock(); + } + } else { + log.debug("[{}] Failed to acquire lock.", key); + scheduler.schedule(this::tryProcessTasks, 1, TimeUnit.SECONDS); + } + }); + } + + public void doInit(Queue queue) { + log.info("[{}] Init consumer with queue: {}", key, queue); + this.queue = queue; + if (!queue.isConsumerPerPartition()) { + mainConsumer = new TbQueueConsumerLauncher(queueFactory.createToRuleEngineMsgConsumer(queue)); + } + } + + private void doUpdate(Queue newQueue, Set partitions) { + if (newQueue.isConsumerPerPartition()) { + + } else { + for (var oldConsumer : consumers.values()) { + oldConsumer.stop(); + } + for (var oldConsumer : consumers.entrySet()) { + try { + oldConsumer.getValue().awaitStopped(); + } catch (Exception e) { + log.info("[{}][{}] Failed to stop the consumer during update", key, oldConsumer.getKey().getPartition().orElse(-1), e); + } + } + if (mainConsumer == null) { + mainConsumer = new TbQueueConsumerLauncher(queueFactory.createToRuleEngineMsgConsumer(queue)); + //TODO: launch + } + mainConsumer.subscribe(partitions); + + } + } + + private void doDelete() { + } + + public void subscribe(PartitionChangeEvent event) { + log.info("[{}] Subscribing to partitions: {}", key, event.getPartitions()); + if (!queue.isConsumerPerPartition()) { + mainConsumer.subscribe(event.getPartitions()); + } else { + log.info("[{}] Subscribing consumer per partition: {}", key, event.getPartitions()); + subscribeConsumerPerPartition(event.getQueueKey(), event.getPartitions()); + } + } + + void subscribeConsumerPerPartition(QueueKey queue, Set partitions) { + topicsConsumerPerPartition.get(queue).getSubscribeQueue().add(partitions); + scheduleTopicRepartition(queue); + } + + private void scheduleTopicRepartition(QueueKey queue) { + repartitionExecutor.schedule(() -> repartitionTopicWithConsumerPerPartition(queue), 1, TimeUnit.SECONDS); + } + + void repartitionTopicWithConsumerPerPartition(final QueueKey queueKey) { + if (stopped) { + return; + } + TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.get(queueKey); + java.util.Queue> subscribeQueue = tbTopicWithConsumerPerPartition.getSubscribeQueue(); + if (subscribeQueue.isEmpty()) { + return; + } + if (tbTopicWithConsumerPerPartition.getLock().tryLock()) { + try { + Set partitions = null; + while (!subscribeQueue.isEmpty()) { + partitions = subscribeQueue.poll(); + } + if (partitions == null) { + return; + } + + Set addedPartitions = new HashSet<>(partitions); + ConcurrentMap>> consumers = tbTopicWithConsumerPerPartition.getConsumers(); + addedPartitions.removeAll(consumers.keySet()); + log.info("calculated addedPartitions {}", addedPartitions); + + Set removedPartitions = new HashSet<>(consumers.keySet()); + removedPartitions.removeAll(partitions); + log.info("calculated removedPartitions {}", removedPartitions); + + removedPartitions.forEach((tpi) -> { + removeConsumerForTopicByTpi(queueKey.getQueueName(), consumers, tpi); + }); + + addedPartitions.forEach((tpi) -> { + log.info("[{}] Adding consumer for topic: {}", queueKey, tpi); + Queue configuration = consumerConfigurations.get(queueKey); + TbQueueConsumer> consumer = tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration); + consumers.put(tpi, consumer); + launchConsumer(consumer, queueKey, tpi.getFullTopicName(), queueKey + "-" + tpi.getPartition().orElse(-999999)); + consumer.subscribe(Collections.singleton(tpi)); + }); + } finally { + tbTopicWithConsumerPerPartition.getLock().unlock(); + } + } else { + scheduleTopicRepartition(queueKey); //reschedule later + } + } + + public void stop() { +// consumers.values().forEach(TbQueueConsumer::stop); +// topicsConsumerPerPartition.values().forEach(tbTopicWithConsumerPerPartition -> tbTopicWithConsumerPerPartition.getConsumers().keySet() +// .forEach((tpi) -> removeConsumerForTopicByTpi(tbTopicWithConsumerPerPartition.getTopic(), tbTopicWithConsumerPerPartition.getConsumers(), tpi))); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 386447045d..bb1943a7d5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -82,7 +82,6 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory * @return * @param configuration */ - //TODO 2.5 ybondarenko: make sure you use queueName to distinct consumers where necessary TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration); /** From 3a0afd8a5ab9e5ed518da6bc1f63f909695875c7 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 2 Oct 2023 18:51:03 +0300 Subject: [PATCH 13/29] Refactoring of the consumer service --- .../DefaultTbRuleEngineConsumerService.java | 426 ++--------------- .../queue/TbQueueConsumerLauncher.java | 37 -- .../queue/TbQueueConsumerManagerTask.java | 17 - .../TbRuleEngineQueueConsumerManager.java | 205 --------- .../service/queue/ruleengine/QueueEvent.java | 9 + .../TbQueueConsumerManagerTask.java | 32 ++ .../queue/ruleengine/TbQueueConsumerTask.java | 70 +++ .../TbRuleEngineConsumerContext.java | 75 +++ .../TbRuleEngineQueueConsumerManager.java | 431 ++++++++++++++++++ .../common/util/ThingsBoardThreadFactory.java | 12 + 10 files changed, 659 insertions(+), 655 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerLauncher.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerManagerTask.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineQueueConsumerManager.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index ae04de456b..b37e5abff3 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -15,44 +15,25 @@ */ package org.thingsboard.server.service.queue; -import com.google.protobuf.ProtocolStringList; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.rpc.RpcError; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.gen.MsgProtos; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; -import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.common.msg.queue.RuleNodeInfo; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; -import org.thingsboard.server.common.stats.StatsFactory; -import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.QueueKey; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.TbRuleEngineComponent; @@ -60,102 +41,48 @@ import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; -import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; -import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; -import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; -import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; -import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; -import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; +import org.thingsboard.server.service.queue.ruleengine.TbRuleEngineConsumerContext; +import org.thingsboard.server.service.queue.ruleengine.TbRuleEngineQueueConsumerManager; import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; -import org.thingsboard.server.service.stats.RuleEngineStatisticsService; -import org.threadly.concurrent.wrapper.KeyDistributedExecutor; import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; @Service @TbRuleEngineComponent @Slf4j public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService implements TbRuleEngineConsumerService { - public static final String SUCCESSFUL_STATUS = "successful"; - public static final String FAILED_STATUS = "failed"; - public static final String THREAD_TOPIC_SEPARATOR = " | "; - @Value("${queue.rule-engine.poll-interval}") - private long pollDuration; - @Value("${queue.rule-engine.pack-processing-timeout}") - private long packProcessingTimeout; - @Value("${queue.rule-engine.stats.enabled:true}") - private boolean statsEnabled; - @Value("${queue.rule-engine.prometheus-stats.enabled:false}") - boolean prometheusStatsEnabled; - @Value("${queue.rule-engine.topic-deletion-delay:30}") - private int topicDeletionDelayInSec; - - private final StatsFactory statsFactory; - private final TbRuleEngineSubmitStrategyFactory submitStrategyFactory; - private final TbRuleEngineProcessingStrategyFactory processingStrategyFactory; - private final TbRuleEngineQueueFactory tbRuleEngineQueueFactory; - private final RuleEngineStatisticsService statisticsService; + private final TbRuleEngineConsumerContext ctx; private final TbRuleEngineDeviceRpcService tbDeviceRpcService; - private final TbServiceInfoProvider serviceInfoProvider; - private final QueueService queueService; - private final TbQueueProducerProvider producerProvider; - private final TbQueueAdmin queueAdmin; - - private final ConcurrentMap consumerMap = new ConcurrentHashMap<>(); - private final ExecutorService consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer")); - private final ExecutorService submitExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-submit")); - private final ScheduledExecutorService repartitionExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-repartition")); + private final ConcurrentMap consumers = new ConcurrentHashMap<>(); - public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory, - TbRuleEngineSubmitStrategyFactory submitStrategyFactory, + public DefaultTbRuleEngineConsumerService(TbRuleEngineConsumerContext ctx, TbRuleEngineQueueFactory tbRuleEngineQueueFactory, - RuleEngineStatisticsService statisticsService, ActorSystemContext actorContext, DataDecodingEncodingService encodingService, TbRuleEngineDeviceRpcService tbDeviceRpcService, - StatsFactory statsFactory, TbDeviceProfileCache deviceProfileCache, TbAssetProfileCache assetProfileCache, TbTenantProfileCache tenantProfileCache, TbApiUsageStateService apiUsageStateService, - PartitionService partitionService, ApplicationEventPublisher eventPublisher, - TbServiceInfoProvider serviceInfoProvider, QueueService queueService, - TbQueueProducerProvider producerProvider, TbQueueAdmin queueAdmin) { - super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, eventPublisher, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer(), Optional.empty()); - this.statisticsService = statisticsService; - this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; - this.submitStrategyFactory = submitStrategyFactory; - this.processingStrategyFactory = processingStrategyFactory; + PartitionService partitionService, ApplicationEventPublisher eventPublisher) { + super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, + eventPublisher, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer(), Optional.empty()); + this.ctx = ctx; this.tbDeviceRpcService = tbDeviceRpcService; - this.statsFactory = statsFactory; - this.serviceInfoProvider = serviceInfoProvider; - this.queueService = queueService; - this.producerProvider = producerProvider; - this.queueAdmin = queueAdmin; } @PostConstruct public void init() { super.init("tb-rule-engine-notifications-consumer"); // TODO: restore init of the main consumer? - List queues = queueService.findAllQueues(); + List queues = ctx.findAllQueues(); for (Queue configuration : queues) { if (partitionService.isManagedByCurrentService(configuration.getTenantId())) { initConsumer(configuration); @@ -164,22 +91,14 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } private void initConsumer(Queue configuration) { - consumerMap.computeIfAbsent(new QueueKey(ServiceType.TB_RULE_ENGINE, configuration), - key -> new TbRuleEngineQueueConsumerManager(repartitionExecutor, consumersExecutor, statsFactory, tbRuleEngineQueueFactory, key)).init(configuration); - } - - @PreDestroy - public void stop() { - super.destroy(); - consumersExecutor.shutdownNow(); // TODO: shutdown or shutdownNow? - submitExecutor.shutdownNow(); - repartitionExecutor.shutdownNow(); + consumers.computeIfAbsent(new QueueKey(ServiceType.TB_RULE_ENGINE, configuration), + key -> new TbRuleEngineQueueConsumerManager(ctx, key)).init(configuration); } @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { if (event.getServiceType().equals(getServiceType())) { - var consumer = consumerMap.get(event.getQueueKey()); + var consumer = consumers.get(event.getQueueKey()); if (consumer != null) { consumer.subscribe(event); } else { @@ -188,207 +107,14 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } } -// void subscribeConsumerPerPartition(QueueKey queue, Set partitions) { -// topicsConsumerPerPartition.get(queue).getSubscribeQueue().add(partitions); -// scheduleTopicRepartition(queue); -// } -// -// private void scheduleTopicRepartition(QueueKey queue) { -// repartitionExecutor.schedule(() -> repartitionTopicWithConsumerPerPartition(queue), 1, TimeUnit.SECONDS); -// } -// -// void repartitionTopicWithConsumerPerPartition(final QueueKey queueKey) { -// if (stopped) { -// return; -// } -// TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.get(queueKey); -// java.util.Queue> subscribeQueue = tbTopicWithConsumerPerPartition.getSubscribeQueue(); -// if (subscribeQueue.isEmpty()) { -// return; -// } -// if (tbTopicWithConsumerPerPartition.getLock().tryLock()) { -// try { -// Set partitions = null; -// while (!subscribeQueue.isEmpty()) { -// partitions = subscribeQueue.poll(); -// } -// if (partitions == null) { -// return; -// } -// -// Set addedPartitions = new HashSet<>(partitions); -// ConcurrentMap>> consumers = tbTopicWithConsumerPerPartition.getConsumers(); -// addedPartitions.removeAll(consumers.keySet()); -// log.info("calculated addedPartitions {}", addedPartitions); -// -// Set removedPartitions = new HashSet<>(consumers.keySet()); -// removedPartitions.removeAll(partitions); -// log.info("calculated removedPartitions {}", removedPartitions); -// -// removedPartitions.forEach((tpi) -> { -// removeConsumerForTopicByTpi(queueKey.getQueueName(), consumers, tpi); -// }); -// -// addedPartitions.forEach((tpi) -> { -// log.info("[{}] Adding consumer for topic: {}", queueKey, tpi); -// Queue configuration = consumerConfigurations.get(queueKey); -// TbQueueConsumer> consumer = tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration); -// consumers.put(tpi, consumer); -// launchConsumer(consumer, queueKey, tpi.getFullTopicName(), queueKey + "-" + tpi.getPartition().orElse(-999999)); -// consumer.subscribe(Collections.singleton(tpi)); -// }); -// } finally { -// tbTopicWithConsumerPerPartition.getLock().unlock(); -// } -// } else { -// scheduleTopicRepartition(queueKey); //reschedule later -// } -// } - - void removeConsumerForTopicByTpi(String queue, ConcurrentMap>> consumers, TopicPartitionInfo tpi) { - log.info("[{}] Removing consumer for topic: {}", queue, tpi); - consumers.remove(tpi).stop(); - } - @Override protected void launchMainConsumers() { - consumers.forEach((queue, consumer) -> launchConsumer(consumer, queue, queue, queue.getQueueName())); + consumers.values().forEach(TbRuleEngineQueueConsumerManager::launchMainConsumer); } @Override protected void stopMainConsumers() { - consumerMap.values().forEach(TbRuleEngineQueueConsumerManager::stop); - } - - void launchConsumer(TbQueueConsumer> consumer, QueueKey queueKey, Object consumerKey, String threadSuffix) { - if (isReady) { - log.info("[{}] Launching consumer", consumerKey); - consumersExecutor.execute(consumerKey, () -> consumerLoop(consumer, queueKey, threadSuffix)); - } else { - scheduleLaunchConsumer(consumer, queueKey, consumerKey, threadSuffix); - } - } - - private void scheduleLaunchConsumer(TbQueueConsumer> consumer, QueueKey queueKey, Object consumerKey, String threadSuffix) { - repartitionExecutor.schedule(() -> { - launchConsumer(consumer, queueKey, consumerKey, threadSuffix); - }, 10, TimeUnit.SECONDS); - } - - void consumerLoop(TbQueueConsumer> consumer, QueueKey queueKey, String threadSuffix) { - Queue configuration = consumerConfigurations.get(queueKey); - TbRuleEngineConsumerStats stats = consumerStats.get(queueKey); - updateCurrentThreadName(threadSuffix); - - while (!stopped && !consumer.isStopped() && !consumer.isQueueDeleted()) { - try { - List> msgs = consumer.poll(configuration.getPollInterval()); - if (msgs.isEmpty()) { - continue; - } - final TbRuleEngineSubmitStrategy submitStrategy = getSubmitStrategy(configuration); - final TbRuleEngineProcessingStrategy ackStrategy = getAckStrategy(configuration); - submitStrategy.init(msgs); - while (!stopped && !consumer.isStopped()) { - TbMsgPackProcessingContext ctx = new TbMsgPackProcessingContext(configuration.getName(), submitStrategy, ackStrategy.isSkipTimeoutMsgs()); - submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> submitMessage(configuration, stats, ctx, id, msg))); - - final boolean timeout = !ctx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS); - - TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(configuration.getName(), timeout, ctx); - if (timeout) { - printFirstOrAll(configuration, ctx, ctx.getPendingMap(), "Timeout"); - } - if (!ctx.getFailedMap().isEmpty()) { - printFirstOrAll(configuration, ctx, ctx.getFailedMap(), "Failed"); - } - ctx.printProfilerStats(); - - TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result); - if (statsEnabled) { - stats.log(result, decision.isCommit()); - } - - ctx.cleanup(); - - if (decision.isCommit()) { - submitStrategy.stop(); - break; - } else { - submitStrategy.update(decision.getReprocessMap()); - } - } - consumer.commit(); - } catch (Exception e) { - if (!stopped) { - log.warn("Failed to process messages from queue.", e); - try { - Thread.sleep(pollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - } - - if (consumer.isStopped()) { - consumer.unsubscribe(); - } else if (consumer.isQueueDeleted()) { - processQueueDeletion(configuration, consumer); - } - log.info("TB Rule Engine Consumer stopped."); - } - - void updateCurrentThreadName(String threadSuffix) { - String name = Thread.currentThread().getName(); - int spliteratorIndex = name.indexOf(THREAD_TOPIC_SEPARATOR); - if (spliteratorIndex > 0) { - name = name.substring(0, spliteratorIndex); - } - name = name + THREAD_TOPIC_SEPARATOR + threadSuffix; - Thread.currentThread().setName(name); - } - - TbRuleEngineProcessingStrategy getAckStrategy(Queue configuration) { - return processingStrategyFactory.newInstance(configuration.getName(), configuration.getProcessingStrategy()); - } - - TbRuleEngineSubmitStrategy getSubmitStrategy(Queue configuration) { - return submitStrategyFactory.newInstance(configuration.getName(), configuration.getSubmitStrategy()); - } - - void submitMessage(Queue configuration, TbRuleEngineConsumerStats stats, TbMsgPackProcessingContext ctx, UUID id, TbProtoQueueMsg msg) { - log.trace("[{}] Creating callback for topic {} message: {}", id, configuration.getName(), msg.getValue()); - ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); - TenantId tenantId = TenantId.fromUUID(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); - TbMsgCallback callback = prometheusStatsEnabled ? - new TbMsgPackCallback(id, tenantId, ctx, stats.getTimer(tenantId, SUCCESSFUL_STATUS), stats.getTimer(tenantId, FAILED_STATUS)) : - new TbMsgPackCallback(id, tenantId, ctx); - try { - if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { - forwardToRuleEngineActor(configuration.getName(), tenantId, toRuleEngineMsg, callback); - } else { - callback.onSuccess(); - } - } catch (Exception e) { - callback.onFailure(new RuleEngineException(e.getMessage(), e)); - } - } - - private void printFirstOrAll(Queue configuration, TbMsgPackProcessingContext ctx, Map> map, String prefix) { - boolean printAll = log.isTraceEnabled(); - log.info("{} to process [{}] messages", prefix, map.size()); - for (Map.Entry> pending : map.entrySet()) { - ToRuleEngineMsg tmp = pending.getValue().getValue(); - TbMsg tmpMsg = TbMsg.fromBytes(configuration.getName(), tmp.getTbMsg().toByteArray(), TbMsgCallback.EMPTY); - RuleNodeInfo ruleNodeInfo = ctx.getLastVisitedRuleNode(pending.getKey()); - if (printAll) { - log.trace("[{}] {} to process message: {}, Last Rule Node: {}", TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); - } else { - log.info("[{}] {} to process message: {}, Last Rule Node: {}", TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); - break; - } - } + consumers.values().forEach(TbRuleEngineQueueConsumerManager::stop); } @Override @@ -398,18 +124,18 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @Override protected long getNotificationPollDuration() { - return pollDuration; + return ctx.getPollDuration(); } @Override protected long getNotificationPackProcessingTimeout() { - return packProcessingTimeout; + return ctx.getPackProcessingTimeout(); } @Override protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) throws Exception { ToRuleEngineNotificationMsg nfMsg = msg.getValue(); - if (nfMsg.getComponentLifecycleMsg() != null && !nfMsg.getComponentLifecycleMsg().isEmpty()) { + if (!nfMsg.getComponentLifecycleMsg().isEmpty()) { handleComponentLifecycleMsg(id, nfMsg.getComponentLifecycleMsg()); callback.onSuccess(); } else if (nfMsg.hasFromDeviceRpcResponse()) { @@ -420,10 +146,10 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< tbDeviceRpcService.processRpcResponseFromDevice(response); callback.onSuccess(); } else if (nfMsg.hasQueueUpdateMsg()) { - repartitionExecutor.execute(() -> updateQueue(nfMsg.getQueueUpdateMsg())); + ctx.getScheduler().execute(() -> updateQueue(nfMsg.getQueueUpdateMsg())); callback.onSuccess(); } else if (nfMsg.hasQueueDeleteMsg()) { - repartitionExecutor.execute(() -> deleteQueue(nfMsg.getQueueDeleteMsg())); + ctx.getScheduler().execute(() -> deleteQueue(nfMsg.getQueueDeleteMsg())); callback.onSuccess(); } else { log.trace("Received notification with missing handler"); @@ -438,33 +164,13 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< QueueId queueId = new QueueId(new UUID(queueUpdateMsg.getQueueIdMSB(), queueUpdateMsg.getQueueIdLSB())); String queueName = queueUpdateMsg.getQueueName(); QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueName, tenantId); - Queue queue = queueService.findQueueById(tenantId, queueId); - Queue oldQueue = consumerConfigurations.remove(queueKey); - if (oldQueue != null) { - if (oldQueue.isConsumerPerPartition()) { - TbTopicWithConsumerPerPartition consumerPerPartition = topicsConsumerPerPartition.remove(queueKey); - ReentrantLock lock = consumerPerPartition.getLock(); - try { - lock.lock(); - consumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::stop); - } finally { - lock.unlock(); - } - } else { - TbQueueConsumer> consumer = consumers.remove(queueKey); - consumer.stop(); - } - } - - initConsumer(queue); - - if (!queue.isConsumerPerPartition()) { - launchConsumer(consumers.get(queueKey), queueKey, queueKey, queueName); - } + Queue queue = ctx.getQueueService().findQueueById(tenantId, queueId); + consumers.computeIfAbsent(queueKey, key -> new TbRuleEngineQueueConsumerManager(ctx, key)).update(queue); } partitionService.updateQueue(queueUpdateMsg); - partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), new ArrayList<>(partitionService.getOtherServices(ServiceType.TB_RULE_ENGINE))); + partitionService.recalculatePartitions(ctx.getServiceInfoProvider().getServiceInfo(), + new ArrayList<>(partitionService.getOtherServices(ServiceType.TB_RULE_ENGINE))); } private void deleteQueue(TransportProtos.QueueDeleteMsg queueDeleteMsg) { @@ -473,90 +179,18 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueDeleteMsg.getQueueName(), tenantId); partitionService.removeQueue(queueDeleteMsg); - Queue queue = consumerConfigurations.remove(queueKey); - if (queue != null) { - if (queue.isConsumerPerPartition()) { - TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.remove(queueKey); - if (tbTopicWithConsumerPerPartition != null) { - tbTopicWithConsumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::onQueueDelete); - tbTopicWithConsumerPerPartition.getConsumers().clear(); - } - } else { - TbQueueConsumer> consumer = consumers.remove(queueKey); - if (consumer != null) { - consumer.onQueueDelete(); - } - } - } - partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), new ArrayList<>(partitionService.getOtherServices(ServiceType.TB_RULE_ENGINE))); - } - - private void forwardToRuleEngineActor(String queueName, TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { - TbMsg tbMsg = TbMsg.fromBytes(queueName, toRuleEngineMsg.getTbMsg().toByteArray(), callback); - QueueToRuleEngineMsg msg; - ProtocolStringList relationTypesList = toRuleEngineMsg.getRelationTypesList(); - Set relationTypes = null; - if (relationTypesList != null) { - if (relationTypesList.size() == 1) { - relationTypes = Collections.singleton(relationTypesList.get(0)); - } else { - relationTypes = new HashSet<>(relationTypesList); - } - } - msg = new QueueToRuleEngineMsg(tenantId, tbMsg, relationTypes, toRuleEngineMsg.getFailureMessage()); - actorContext.tell(msg); - } - - private void processQueueDeletion(Queue queue, TbQueueConsumer> consumer) { - long finishTs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(topicDeletionDelayInSec); - try { - int n = 0; - while (System.currentTimeMillis() <= finishTs) { - List> msgs = consumer.poll(queue.getPollInterval()); - if (msgs.isEmpty()) { - continue; - } - for (TbProtoQueueMsg msg : msgs) { - try { - MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray()); - EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB())); - - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue.getName(), TenantId.SYS_TENANT_ID, originator); - producerProvider.getRuleEngineMsgProducer().send(tpi, msg, null); - n++; - } catch (Throwable e) { - log.debug("Failed to move message to system {}: {}", consumer.getTopic(), msg, e); - } - } - consumer.commit(); - } - if (n > 0) { - log.info("Moved {} messages from {} to system {}", n, consumer.getFullTopicNames(), consumer.getTopic()); - } - - consumer.unsubscribe(); - for (String topic : consumer.getFullTopicNames()) { - try { - queueAdmin.deleteTopic(topic); - log.info("Deleted topic {}", topic); - } catch (Exception e) { - log.error("Failed to delete topic {} after unsubscribing", topic, e); - } - } - } catch (Exception e) { - log.error("Failed to process deletion of {} ({})", consumer.getTopic(), queue.getTenantId(), e); + var manager = consumers.remove(queueKey); + if (manager != null) { + manager.delete(); } + partitionService.recalculatePartitions(ctx.getServiceInfoProvider().getServiceInfo(), new ArrayList<>(partitionService.getOtherServices(ServiceType.TB_RULE_ENGINE))); } @Scheduled(fixedDelayString = "${queue.rule-engine.stats.print-interval-ms}") public void printStats() { - if (statsEnabled) { + if (ctx.isStatsEnabled()) { long ts = System.currentTimeMillis(); - consumerStats.forEach((queue, stats) -> { - stats.printStats(); - statisticsService.reportQueueStats(ts, stats); - stats.reset(); - }); + consumers.values().forEach(manager -> manager.printStats(ts)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerLauncher.java b/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerLauncher.java deleted file mode 100644 index f5688eb207..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerLauncher.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.thingsboard.server.service.queue; - -import lombok.Data; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.QueueKey; - -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -@Data -public class TbQueueConsumerLauncher { - - private final TbQueueConsumer> consumer; - private volatile Future task; - - public void stop() { - this.consumer.stop(); - } - - public void awaitStopped() throws ExecutionException, InterruptedException, TimeoutException { - if (task != null) { - this.task.get(3, TimeUnit.MINUTES); - } - } - - public void subscribe(Set partitions) { - this.consumer.subscribe(partitions); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerManagerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerManagerTask.java deleted file mode 100644 index 11316a9e05..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerManagerTask.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.thingsboard.server.service.queue; - -import lombok.Data; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; - -import java.util.Set; - -@Data -public class TbQueueConsumerManagerTask { - - private final ComponentLifecycleEvent event; - private final Queue queue; - private final Set partitions; - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineQueueConsumerManager.java deleted file mode 100644 index 2c6d191c0a..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineQueueConsumerManager.java +++ /dev/null @@ -1,205 +0,0 @@ -package org.thingsboard.server.service.queue; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.common.stats.StatsFactory; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.QueueKey; -import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; -import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; - -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; - -@Data -@Slf4j -public class TbRuleEngineQueueConsumerManager { - - private final ScheduledExecutorService scheduler; - private final ExecutorService consumerExecutor; - private final StatsFactory statsFactory; - private final TbRuleEngineQueueFactory queueFactory; - private final QueueKey key; - private final ReentrantLock lock = new ReentrantLock(); //NonfairSync - private final ConcurrentMap consumers = new ConcurrentHashMap<>(); - private final TbRuleEngineConsumerStats stats; - - private volatile Set partitions = Collections.emptySet(); - private volatile Queue queue; - private volatile TbQueueConsumerLauncher mainConsumer; - - private final java.util.Queue tasks = new ConcurrentLinkedQueue<>(); - - public TbRuleEngineQueueConsumerManager(ScheduledExecutorService scheduler, ExecutorService consumerExecutor, StatsFactory statsFactory, TbRuleEngineQueueFactory queueFactory, QueueKey key) { - this.scheduler = scheduler; - this.consumerExecutor = consumerExecutor; - this.statsFactory = statsFactory; - this.queueFactory = queueFactory; - this.key = key; - this.stats = new TbRuleEngineConsumerStats(key, statsFactory); - } - - public void init(Queue queue) { - processTask(new TbQueueConsumerManagerTask(ComponentLifecycleEvent.CREATED, queue, null)); - } - - private void processTask(TbQueueConsumerManagerTask todo) { - tasks.add(todo); - log.info("[{}] Adding task: {}", key, todo); - tryProcessTasks(); - } - - private void tryProcessTasks() { - consumerExecutor.submit(() -> { - if (lock.tryLock()) { - try { - TbQueueConsumerManagerTask lastUpdateTask = null; - while (!tasks.isEmpty()) { - TbQueueConsumerManagerTask task = tasks.poll(); - switch (task.getEvent()) { - case CREATED: - doInit(task.getQueue()); - break; - case UPDATED: - lastUpdateTask = task; - break; - case DELETED: - lastUpdateTask = null; - doDelete(); - break; - } - } - if (lastUpdateTask != null) { - doUpdate(lastUpdateTask.getQueue(), lastUpdateTask.getPartitions()); - } - } finally { - lock.unlock(); - } - } else { - log.debug("[{}] Failed to acquire lock.", key); - scheduler.schedule(this::tryProcessTasks, 1, TimeUnit.SECONDS); - } - }); - } - - public void doInit(Queue queue) { - log.info("[{}] Init consumer with queue: {}", key, queue); - this.queue = queue; - if (!queue.isConsumerPerPartition()) { - mainConsumer = new TbQueueConsumerLauncher(queueFactory.createToRuleEngineMsgConsumer(queue)); - } - } - - private void doUpdate(Queue newQueue, Set partitions) { - if (newQueue.isConsumerPerPartition()) { - - } else { - for (var oldConsumer : consumers.values()) { - oldConsumer.stop(); - } - for (var oldConsumer : consumers.entrySet()) { - try { - oldConsumer.getValue().awaitStopped(); - } catch (Exception e) { - log.info("[{}][{}] Failed to stop the consumer during update", key, oldConsumer.getKey().getPartition().orElse(-1), e); - } - } - if (mainConsumer == null) { - mainConsumer = new TbQueueConsumerLauncher(queueFactory.createToRuleEngineMsgConsumer(queue)); - //TODO: launch - } - mainConsumer.subscribe(partitions); - - } - } - - private void doDelete() { - } - - public void subscribe(PartitionChangeEvent event) { - log.info("[{}] Subscribing to partitions: {}", key, event.getPartitions()); - if (!queue.isConsumerPerPartition()) { - mainConsumer.subscribe(event.getPartitions()); - } else { - log.info("[{}] Subscribing consumer per partition: {}", key, event.getPartitions()); - subscribeConsumerPerPartition(event.getQueueKey(), event.getPartitions()); - } - } - - void subscribeConsumerPerPartition(QueueKey queue, Set partitions) { - topicsConsumerPerPartition.get(queue).getSubscribeQueue().add(partitions); - scheduleTopicRepartition(queue); - } - - private void scheduleTopicRepartition(QueueKey queue) { - repartitionExecutor.schedule(() -> repartitionTopicWithConsumerPerPartition(queue), 1, TimeUnit.SECONDS); - } - - void repartitionTopicWithConsumerPerPartition(final QueueKey queueKey) { - if (stopped) { - return; - } - TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.get(queueKey); - java.util.Queue> subscribeQueue = tbTopicWithConsumerPerPartition.getSubscribeQueue(); - if (subscribeQueue.isEmpty()) { - return; - } - if (tbTopicWithConsumerPerPartition.getLock().tryLock()) { - try { - Set partitions = null; - while (!subscribeQueue.isEmpty()) { - partitions = subscribeQueue.poll(); - } - if (partitions == null) { - return; - } - - Set addedPartitions = new HashSet<>(partitions); - ConcurrentMap>> consumers = tbTopicWithConsumerPerPartition.getConsumers(); - addedPartitions.removeAll(consumers.keySet()); - log.info("calculated addedPartitions {}", addedPartitions); - - Set removedPartitions = new HashSet<>(consumers.keySet()); - removedPartitions.removeAll(partitions); - log.info("calculated removedPartitions {}", removedPartitions); - - removedPartitions.forEach((tpi) -> { - removeConsumerForTopicByTpi(queueKey.getQueueName(), consumers, tpi); - }); - - addedPartitions.forEach((tpi) -> { - log.info("[{}] Adding consumer for topic: {}", queueKey, tpi); - Queue configuration = consumerConfigurations.get(queueKey); - TbQueueConsumer> consumer = tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration); - consumers.put(tpi, consumer); - launchConsumer(consumer, queueKey, tpi.getFullTopicName(), queueKey + "-" + tpi.getPartition().orElse(-999999)); - consumer.subscribe(Collections.singleton(tpi)); - }); - } finally { - tbTopicWithConsumerPerPartition.getLock().unlock(); - } - } else { - scheduleTopicRepartition(queueKey); //reschedule later - } - } - - public void stop() { -// consumers.values().forEach(TbQueueConsumer::stop); -// topicsConsumerPerPartition.values().forEach(tbTopicWithConsumerPerPartition -> tbTopicWithConsumerPerPartition.getConsumers().keySet() -// .forEach((tpi) -> removeConsumerForTopicByTpi(tbTopicWithConsumerPerPartition.getTopic(), tbTopicWithConsumerPerPartition.getConsumers(), tpi))); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java new file mode 100644 index 0000000000..4d01d15d9a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java @@ -0,0 +1,9 @@ +package org.thingsboard.server.service.queue.ruleengine; + +import java.io.Serializable; + +public enum QueueEvent implements Serializable { + + CREATED, LAUNCHED, UPDATED, PARTITION_CHANGE, STOP, DELETED + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java new file mode 100644 index 0000000000..5b4f164a10 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.ruleengine; + +import lombok.Data; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; + +import java.util.Set; + +@Data +public class TbQueueConsumerManagerTask { + + private final QueueEvent event; + private final Queue queue; + private final Set partitions; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java new file mode 100644 index 0000000000..3f549b273c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java @@ -0,0 +1,70 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.ruleengine; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.QueueKey; + +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +@Data +@Slf4j +public class TbQueueConsumerTask { + + private final QueueKey key; + private final Object id; + private final TbQueueConsumer> consumer; + private volatile Future task; + + public void stop() { + this.consumer.stop(); + } + + public boolean stopAndAwait() { + this.consumer.stop(); + return await(); + } + + public boolean await() { + if (task != null) { + try { + this.task.get(3, TimeUnit.MINUTES); + } catch (ExecutionException | InterruptedException | TimeoutException e) { + log.warn("[{}][{}] Failed to await for consumer to stop", key, id, e); + return false; + } + } + return true; + } + + public void subscribe(Set partitions) { + this.consumer.subscribe(partitions); + } + + + public void unsubscribe() { + this.consumer.unsubscribe(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java new file mode 100644 index 0000000000..49f1ad1fab --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java @@ -0,0 +1,75 @@ +package org.thingsboard.server.service.queue.ruleengine; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.dao.queue.QueueService; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; +import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; +import org.thingsboard.server.service.stats.RuleEngineStatisticsService; + +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +@Component +@TbRuleEngineComponent +@Slf4j +@Data +public class TbRuleEngineConsumerContext { + + @Value("${queue.rule-engine.poll-interval}") + private long pollDuration; + @Value("${queue.rule-engine.pack-processing-timeout}") + private long packProcessingTimeout; + @Value("${queue.rule-engine.stats.enabled:true}") + private boolean statsEnabled; + @Value("${queue.rule-engine.prometheus-stats.enabled:false}") + boolean prometheusStatsEnabled; + @Value("${queue.rule-engine.topic-deletion-delay:30}") + private int topicDeletionDelayInSec; + + protected volatile boolean stopped = false; + protected volatile boolean isReady = false; + + private final ActorSystemContext actorContext; + private final StatsFactory statsFactory; + private final TbRuleEngineSubmitStrategyFactory submitStrategyFactory; + private final TbRuleEngineProcessingStrategyFactory processingStrategyFactory; + private final TbRuleEngineQueueFactory queueFactory; + private final RuleEngineStatisticsService statisticsService; + private final TbServiceInfoProvider serviceInfoProvider; + private final QueueService queueService; + private final TbQueueProducerProvider producerProvider; + private final TbQueueAdmin queueAdmin; + + //TODO: add reasonable limit for mgmt pool. + private final ExecutorService mgmtExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-mgmt")); + private final ExecutorService consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer")); + private final ExecutorService submitExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-submit")); + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-scheduler")); + + public List findAllQueues() { + return queueService.findAllQueues(); + } + + @PreDestroy + public void stop() { + consumersExecutor.shutdownNow(); // TODO: shutdown or shutdownNow? + submitExecutor.shutdownNow(); + scheduler.shutdownNow(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java new file mode 100644 index 0000000000..2599a214f0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -0,0 +1,431 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.ruleengine; + +import com.google.protobuf.ProtocolStringList; +import lombok.Data; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.gen.MsgProtos; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.RuleNodeInfo; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.service.queue.TbMsgPackCallback; +import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; +import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +@Data +@Slf4j +public class TbRuleEngineQueueConsumerManager { + + public static final String SUCCESSFUL_STATUS = "successful"; + public static final String FAILED_STATUS = "failed"; + + private final TbRuleEngineConsumerContext ctx; + private final QueueKey key; + + private final ReentrantLock lock = new ReentrantLock(); //NonfairSync + private final ConcurrentMap consumersPerPartition = new ConcurrentHashMap<>(); + private final TbRuleEngineConsumerStats stats; + + private volatile Set partitions = Collections.emptySet(); + private volatile Queue queue; + private volatile TbQueueConsumerTask mainConsumer; + + private final java.util.Queue tasks = new ConcurrentLinkedQueue<>(); + + public TbRuleEngineQueueConsumerManager(TbRuleEngineConsumerContext ctx, QueueKey key) { + this.ctx = ctx; + this.key = key; + this.stats = new TbRuleEngineConsumerStats(key, ctx.getStatsFactory()); + } + + public void init(Queue queue) { + processTask(new TbQueueConsumerManagerTask(QueueEvent.CREATED, queue, null)); + } + + public void update(Queue queue) { + processTask(new TbQueueConsumerManagerTask(QueueEvent.UPDATED, queue, null)); + } + + public void subscribe(PartitionChangeEvent event) { + processTask(new TbQueueConsumerManagerTask(QueueEvent.PARTITION_CHANGE, queue, event.getPartitions())); + } + + public void launchMainConsumer() { + processTask(new TbQueueConsumerManagerTask(QueueEvent.LAUNCHED, null, null)); + } + + public void stop() { + processTask(new TbQueueConsumerManagerTask(QueueEvent.STOP, null, null)); + } + + public void delete() { + processTask(new TbQueueConsumerManagerTask(QueueEvent.DELETED, null, null)); + } + + private void processTask(TbQueueConsumerManagerTask todo) { + tasks.add(todo); + log.info("[{}] Adding task: {}", key, todo); + tryProcessTasks(); + } + + private void tryProcessTasks() { + ctx.getMgmtExecutor().submit(() -> { + if (lock.tryLock()) { + try { + Queue newConfiguration = null; + Set newPartitions = null; + while (!tasks.isEmpty()) { + TbQueueConsumerManagerTask task = tasks.poll(); + switch (task.getEvent()) { + case CREATED: + doInit(task.getQueue()); + break; + case LAUNCHED: + if (!queue.isConsumerPerPartition()) { + doLaunchMainConsumer(); + } + break; + case UPDATED: + newConfiguration = task.getQueue(); + break; + case PARTITION_CHANGE: + newPartitions = task.getPartitions(); + break; + case STOP: + newConfiguration = null; + newPartitions = null; + doStop(); + break; + case DELETED: + newConfiguration = null; + newPartitions = null; + doDelete(); + break; + } + } + if (newConfiguration != null) { + doUpdate(newConfiguration); + } + if (newPartitions != null) { + doUpdate(newPartitions); + } + } finally { + lock.unlock(); + } + } else { + log.debug("[{}] Failed to acquire lock.", key); + ctx.getScheduler().schedule(this::tryProcessTasks, 1, TimeUnit.SECONDS); + } + }); + } + + public void doInit(Queue queue) { + log.info("[{}] Init consumer with queue: {}", key, queue); + this.queue = queue; + if (queue.isConsumerPerPartition()) { + log.debug("[{}] Ignore init event since isConsumerPerPartition is enabled.", key); + } else { + mainConsumer = new TbQueueConsumerTask(key, "main", ctx.getQueueFactory().createToRuleEngineMsgConsumer(queue)); + } + } + + private void doLaunchMainConsumer() { + if (mainConsumer != null) { + launchConsumer(mainConsumer, queue, mainConsumer.getId(), queue.getName()); + } else { + log.warn("[{}] Can't launch main consumer since it is empty!", key); + } + } + + private void doUpdate(Queue newQueue) { + log.info("[{}] Processing queue update: {}", key, newQueue); + var oldQueue = queue; + if (log.isTraceEnabled()) { + log.trace("[{}] Old queue configuration: {}", key, oldQueue); + log.trace("[{}] New queue configuration: {}", key, newQueue); + } + if (oldQueue != null) { + doStop(oldQueue); + } + doInit(newQueue); + if (!newQueue.isConsumerPerPartition()) { + doLaunchMainConsumer(); + } + } + + private void doUpdate(Set partitions) { + log.info("[{}] Subscribing to partitions: {}", key, partitions); + if (queue.isConsumerPerPartition()) { + log.debug("[{}] Subscribing consumers per partition separately: {}", key, partitions); + Set addedPartitions = new HashSet<>(partitions); + addedPartitions.removeAll(consumersPerPartition.keySet()); + log.info("calculated addedPartitions {}", addedPartitions); + + Set removedPartitions = new HashSet<>(consumersPerPartition.keySet()); + removedPartitions.removeAll(partitions); + log.info("calculated removedPartitions {}", removedPartitions); + + removedPartitions.forEach((tpi) -> { + log.info("[{}] Unsubscribing from topic: {}", queue, tpi); + consumersPerPartition.get(tpi).unsubscribe(); + }); + + removedPartitions.forEach((tpi) -> { + log.info("[{}] Removing consumer for topic: {}", queue, tpi); + consumersPerPartition.get(tpi).stopAndAwait(); + consumersPerPartition.remove(tpi); + }); + + addedPartitions.forEach((tpi) -> { + log.info("[{}] Adding consumer for topic: {}", key, tpi); + TbQueueConsumerTask consumerTask = new TbQueueConsumerTask(key, tpi, ctx.getQueueFactory().createToRuleEngineMsgConsumer(queue)); + consumersPerPartition.put(tpi, consumerTask); + //TODO: Is it ok to subscribe first? + consumerTask.subscribe(Collections.singleton(tpi)); + launchConsumer(consumerTask, queue, mainConsumer.getId(), key + "-" + tpi.getPartition().orElse(-999999)); + }); + } else { + mainConsumer.subscribe(partitions); + } + } + + private void doStop() { + doStop(queue); + } + + private void doStop(Queue queue) { + if (queue.isConsumerPerPartition()) { + consumersPerPartition.values().forEach(TbQueueConsumerTask::unsubscribe); + consumersPerPartition.values().forEach(TbQueueConsumerTask::stopAndAwait); + } else if (mainConsumer != null) { + mainConsumer.unsubscribe(); + mainConsumer.stopAndAwait(); + } + } + + private void doDelete() { + doStop(); + //TODO: repack messages + } + + @SneakyThrows + void launchConsumer(TbQueueConsumerTask consumerTask, Queue configuration, Object consumerKey, String threadSuffix) { + log.info("[{}] Launching consumer: [{}]", key, consumerKey); + while (!ctx.isReady) { + //TODO: Remember this task. Cancel previous task if needed. + log.debug("[{}][{}] Waiting for consumer to get ready..", key, consumerKey); + Thread.sleep(1000); + } + consumerTask.setTask(ctx.getConsumersExecutor().submit(() -> consumerLoop(consumerTask.getConsumer(), configuration, threadSuffix))); + } + + void consumerLoop(TbQueueConsumer> consumer, Queue configuration, String threadSuffix) { + ThingsBoardThreadFactory.updateCurrentThreadName(threadSuffix); + while (!ctx.stopped && !consumer.isStopped() + //TODO: remove this. + && !consumer.isQueueDeleted()) { + try { + List> msgs = consumer.poll(queue.getPollInterval()); + if (msgs.isEmpty()) { + continue; + } + final TbRuleEngineSubmitStrategy submitStrategy = getSubmitStrategy(queue); + final TbRuleEngineProcessingStrategy ackStrategy = getProcessingStrategy(queue); + submitStrategy.init(msgs); + while (!ctx.isStopped() && !consumer.isStopped()) { + TbMsgPackProcessingContext packCtx = new TbMsgPackProcessingContext(queue.getName(), submitStrategy, ackStrategy.isSkipTimeoutMsgs()); + submitStrategy.submitAttempt((id, msg) -> ctx.getSubmitExecutor().submit(() -> submitMessage(configuration, stats, packCtx, id, msg))); + + final boolean timeout = !packCtx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS); + + TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(configuration.getName(), timeout, packCtx); + if (timeout) { + printFirstOrAll(configuration, packCtx, packCtx.getPendingMap(), "Timeout"); + } + if (!packCtx.getFailedMap().isEmpty()) { + printFirstOrAll(configuration, packCtx, packCtx.getFailedMap(), "Failed"); + } + packCtx.printProfilerStats(); + + TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result); + if (ctx.isStatsEnabled()) { + stats.log(result, decision.isCommit()); + } + + packCtx.cleanup(); + + if (decision.isCommit()) { + submitStrategy.stop(); + break; + } else { + submitStrategy.update(decision.getReprocessMap()); + } + } + consumer.commit(); + } catch (Exception e) { + if (!ctx.stopped) { + log.warn("Failed to process messages from queue.", e); + try { + Thread.sleep(ctx.getPollDuration()); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + } + //TODO: refactor and move to the "doDelete" method. Use separate consumer if needed (it is still synchronous). + if (consumer.isQueueDeleted()) { + processQueueDeletion(configuration, consumer); + } + log.info("TB Rule Engine Consumer stopped."); + } + + private void processQueueDeletion(Queue queue, TbQueueConsumer> consumer) { +// long finishTs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(topicDeletionDelayInSec); +// try { +// int n = 0; +// while (System.currentTimeMillis() <= finishTs) { +// List> msgs = consumer.poll(queue.getPollInterval()); +// if (msgs.isEmpty()) { +// continue; +// } +// for (TbProtoQueueMsg msg : msgs) { +// try { +// MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray()); +// EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB())); +// +// TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue.getName(), TenantId.SYS_TENANT_ID, originator); +// producerProvider.getRuleEngineMsgProducer().send(tpi, msg, null); +// n++; +// } catch (Throwable e) { +// log.debug("Failed to move message to system {}: {}", consumer.getTopic(), msg, e); +// } +// } +// consumer.commit(); +// } +// if (n > 0) { +// log.info("Moved {} messages from {} to system {}", n, consumer.getFullTopicNames(), consumer.getTopic()); +// } +// +// consumer.unsubscribe(); +// for (String topic : consumer.getFullTopicNames()) { +// try { +// queueAdmin.deleteTopic(topic); +// log.info("Deleted topic {}", topic); +// } catch (Exception e) { +// log.error("Failed to delete topic {} after unsubscribing", topic, e); +// } +// } +// } catch (Exception e) { +// log.error("Failed to process deletion of {} ({})", consumer.getTopic(), queue.getTenantId(), e); +// } + } + + TbRuleEngineSubmitStrategy getSubmitStrategy(Queue configuration) { + return ctx.getSubmitStrategyFactory().newInstance(configuration.getName(), configuration.getSubmitStrategy()); + } + + TbRuleEngineProcessingStrategy getProcessingStrategy(Queue configuration) { + return ctx.getProcessingStrategyFactory().newInstance(configuration.getName(), configuration.getProcessingStrategy()); + } + + void submitMessage(Queue configuration, TbRuleEngineConsumerStats stats, TbMsgPackProcessingContext packCtx, UUID id, TbProtoQueueMsg msg) { + log.trace("[{}] Creating callback for topic {} message: {}", id, configuration.getName(), msg.getValue()); + TransportProtos.ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); + TenantId tenantId = TenantId.fromUUID(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); + TbMsgCallback callback = ctx.prometheusStatsEnabled ? + new TbMsgPackCallback(id, tenantId, packCtx, stats.getTimer(tenantId, SUCCESSFUL_STATUS), stats.getTimer(tenantId, FAILED_STATUS)) : + new TbMsgPackCallback(id, tenantId, packCtx); + try { + if (!toRuleEngineMsg.getTbMsg().isEmpty()) { + forwardToRuleEngineActor(configuration.getName(), tenantId, toRuleEngineMsg, callback); + } else { + callback.onSuccess(); + } + } catch (Exception e) { + callback.onFailure(new RuleEngineException(e.getMessage(), e)); + } + } + + private void forwardToRuleEngineActor(String queueName, TenantId tenantId, TransportProtos.ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { + TbMsg tbMsg = TbMsg.fromBytes(queueName, toRuleEngineMsg.getTbMsg().toByteArray(), callback); + QueueToRuleEngineMsg msg; + ProtocolStringList relationTypesList = toRuleEngineMsg.getRelationTypesList(); + Set relationTypes; + if (relationTypesList.size() == 1) { + relationTypes = Collections.singleton(relationTypesList.get(0)); + } else { + relationTypes = new HashSet<>(relationTypesList); + } + msg = new QueueToRuleEngineMsg(tenantId, tbMsg, relationTypes, toRuleEngineMsg.getFailureMessage()); + ctx.getActorContext().tell(msg); + } + + + private void printFirstOrAll(Queue configuration, TbMsgPackProcessingContext ctx, Map> map, String prefix) { + boolean printAll = log.isTraceEnabled(); + log.info("{} to process [{}] messages", prefix, map.size()); + for (Map.Entry> pending : map.entrySet()) { + TransportProtos.ToRuleEngineMsg tmp = pending.getValue().getValue(); + TbMsg tmpMsg = TbMsg.fromBytes(configuration.getName(), tmp.getTbMsg().toByteArray(), TbMsgCallback.EMPTY); + RuleNodeInfo ruleNodeInfo = ctx.getLastVisitedRuleNode(pending.getKey()); + if (printAll) { + log.trace("[{}] {} to process message: {}, Last Rule Node: {}", TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); + } else { + log.info("[{}] {} to process message: {}, Last Rule Node: {}", TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); + break; + } + } + } + + public void printStats(long ts) { + stats.printStats(); + ctx.getStatisticsService().reportQueueStats(ts, stats); + stats.reset(); + } +} diff --git a/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardThreadFactory.java b/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardThreadFactory.java index 1f41c48f43..9eaca3ca47 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardThreadFactory.java +++ b/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardThreadFactory.java @@ -22,6 +22,7 @@ import java.util.concurrent.atomic.AtomicInteger; * Copy of Executors.DefaultThreadFactory but with ability to set name of the pool */ public class ThingsBoardThreadFactory implements ThreadFactory { + public static final String THREAD_TOPIC_SEPARATOR = " | "; private static final AtomicInteger poolNumber = new AtomicInteger(1); private final ThreadGroup group; private final AtomicInteger threadNumber = new AtomicInteger(1); @@ -40,6 +41,17 @@ public class ThingsBoardThreadFactory implements ThreadFactory { "-thread-"; } + public static void updateCurrentThreadName(String threadSuffix) { + String name = Thread.currentThread().getName(); + int spliteratorIndex = name.indexOf(THREAD_TOPIC_SEPARATOR); + if (spliteratorIndex > 0) { + name = name.substring(0, spliteratorIndex); + } + name = name + THREAD_TOPIC_SEPARATOR + threadSuffix; + Thread.currentThread().setName(name); + } + + @Override public Thread newThread(Runnable r) { Thread t = new Thread(group, r, From c56132d6e89d00043b0d644850891663ef0b9122 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 3 Oct 2023 15:04:41 +0300 Subject: [PATCH 14/29] Minor todos --- .../server/service/queue/ruleengine/TbQueueConsumerTask.java | 1 + .../service/queue/ruleengine/TbRuleEngineConsumerContext.java | 3 +++ 2 files changed, 4 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java index 3f549b273c..ed1f36074f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java @@ -49,6 +49,7 @@ public class TbQueueConsumerTask { public boolean await() { if (task != null) { + //TODO: maybe task.cancel() to interrupt the consumer? try { this.task.get(3, TimeUnit.MINUTES); } catch (ExecutionException | InterruptedException | TimeoutException e) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java index 49f1ad1fab..07307a3f10 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java @@ -42,6 +42,7 @@ public class TbRuleEngineConsumerContext { @Value("${queue.rule-engine.topic-deletion-delay:30}") private int topicDeletionDelayInSec; + //TODO: check if they are set correctly. protected volatile boolean stopped = false; protected volatile boolean isReady = false; @@ -59,6 +60,7 @@ public class TbRuleEngineConsumerContext { //TODO: add reasonable limit for mgmt pool. private final ExecutorService mgmtExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-mgmt")); private final ExecutorService consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer")); + //TODO: do we actually need this? private final ExecutorService submitExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-submit")); private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-scheduler")); @@ -68,6 +70,7 @@ public class TbRuleEngineConsumerContext { @PreDestroy public void stop() { + mgmtExecutor.shutdownNow(); consumersExecutor.shutdownNow(); // TODO: shutdown or shutdownNow? submitExecutor.shutdownNow(); scheduler.shutdownNow(); From 92e0c65a3740f179a9ec651a392fdcfbbfa6f175 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Sat, 14 Oct 2023 21:50:40 +0300 Subject: [PATCH 15/29] Minor refactoring --- .../queue/DefaultTbCoreConsumerService.java | 10 +++- .../DefaultTbRuleEngineConsumerService.java | 59 +++++++++++++------ .../queue/TbRuleEngineConsumerStats.java | 6 +- .../processing/AbstractConsumerService.java | 11 +--- .../TbRuleEngineConsumerContext.java | 45 +++++++------- .../src/main/resources/thingsboard.yml | 4 +- .../controller/BaseQueueControllerTest.java | 6 +- 7 files changed, 85 insertions(+), 56 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index fd5a252cc5..b2d7700441 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -142,8 +142,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> usageStatsConsumer; private final TbQueueConsumer> firmwareStatesConsumer; + protected volatile ExecutorService consumersExecutor; protected volatile ExecutorService usageStatsExecutor; - private volatile ExecutorService firmwareStatesExecutor; public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, @@ -186,7 +186,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService implements TbRuleEngineConsumerService { private final TbRuleEngineConsumerContext ctx; + private final QueueService queueService; private final TbRuleEngineDeviceRpcService tbDeviceRpcService; private final ConcurrentMap consumers = new ConcurrentHashMap<>(); @@ -68,6 +72,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< ActorSystemContext actorContext, DataDecodingEncodingService encodingService, TbRuleEngineDeviceRpcService tbDeviceRpcService, + QueueService queueService, TbDeviceProfileCache deviceProfileCache, TbAssetProfileCache assetProfileCache, TbTenantProfileCache tenantProfileCache, @@ -77,12 +82,13 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< eventPublisher, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer(), Optional.empty()); this.ctx = ctx; this.tbDeviceRpcService = tbDeviceRpcService; + this.queueService = queueService; } @PostConstruct public void init() { - super.init("tb-rule-engine-notifications-consumer"); // TODO: restore init of the main consumer? - List queues = ctx.findAllQueues(); + super.init("tb-rule-engine-notifications-consumer"); + List queues = queueService.findAllQueues(); for (Queue configuration : queues) { if (partitionService.isManagedByCurrentService(configuration.getTenantId())) { initConsumer(configuration); @@ -91,30 +97,38 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } private void initConsumer(Queue configuration) { - consumers.computeIfAbsent(new QueueKey(ServiceType.TB_RULE_ENGINE, configuration), - key -> new TbRuleEngineQueueConsumerManager(ctx, key)).init(configuration); + getOrCreateConsumer(new QueueKey(ServiceType.TB_RULE_ENGINE, configuration)).init(configuration); } @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { if (event.getServiceType().equals(getServiceType())) { - var consumer = consumers.get(event.getQueueKey()); - if (consumer != null) { - consumer.subscribe(event); - } else { - log.warn("Received invalid partition change event for {} that is not managed by this service", event.getQueueKey()); - } + event.getPartitionsMap().forEach((queueKey, partitions) -> { + var consumer = consumers.get(queueKey); + if (consumer != null) { + consumer.update(partitions); + } else { + log.warn("Received invalid partition change event for {} that is not managed by this service", queueKey); + } + }); } } + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void onApplicationEvent(ApplicationReadyEvent event) { + super.onApplicationEvent(event); + ctx.setReady(true); + } + @Override protected void launchMainConsumers() { consumers.values().forEach(TbRuleEngineQueueConsumerManager::launchMainConsumer); } @Override - protected void stopMainConsumers() { + protected void stopConsumers() { consumers.values().forEach(TbRuleEngineQueueConsumerManager::stop); + ctx.stop(); } @Override @@ -164,8 +178,15 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< QueueId queueId = new QueueId(new UUID(queueUpdateMsg.getQueueIdMSB(), queueUpdateMsg.getQueueIdLSB())); String queueName = queueUpdateMsg.getQueueName(); QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueName, tenantId); - Queue queue = ctx.getQueueService().findQueueById(tenantId, queueId); - consumers.computeIfAbsent(queueKey, key -> new TbRuleEngineQueueConsumerManager(ctx, key)).update(queue); + Queue queue = queueService.findQueueById(tenantId, queueId); + + TbRuleEngineQueueConsumerManager consumerManager = getOrCreateConsumer(queueKey); + Queue oldQueue = consumerManager.getQueue(); + consumerManager.update(queue); + + if (oldQueue != null && queue.getPartitions() == oldQueue.getPartitions()) { + return; + } } partitionService.updateQueue(queueUpdateMsg); @@ -177,15 +198,19 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< log.info("Received queue delete msg: [{}]", queueDeleteMsg); TenantId tenantId = new TenantId(new UUID(queueDeleteMsg.getTenantIdMSB(), queueDeleteMsg.getTenantIdLSB())); QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueDeleteMsg.getQueueName(), tenantId); + var consumerManager = consumers.remove(queueKey); + if (consumerManager != null) { + consumerManager.delete(); + } partitionService.removeQueue(queueDeleteMsg); - var manager = consumers.remove(queueKey); - if (manager != null) { - manager.delete(); - } partitionService.recalculatePartitions(ctx.getServiceInfoProvider().getServiceInfo(), new ArrayList<>(partitionService.getOtherServices(ServiceType.TB_RULE_ENGINE))); } + private TbRuleEngineQueueConsumerManager getOrCreateConsumer(QueueKey queueKey) { + return consumers.computeIfAbsent(queueKey, key -> new TbRuleEngineQueueConsumerManager(ctx, key)); + } + @Scheduled(fixedDelayString = "${queue.rule-engine.stats.print-interval-ms}") public void printStats() { if (ctx.isStatsEnabled()) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index 45ccbcf32a..842b627c22 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -67,9 +67,9 @@ public class TbRuleEngineConsumerStats { private final String queueName; private final TenantId tenantId; - public TbRuleEngineConsumerStats(QueueKey queue, StatsFactory statsFactory) { - this.queueName = queue.getQueueName(); - this.tenantId = queue.getTenantId(); + public TbRuleEngineConsumerStats(QueueKey queueKey, StatsFactory statsFactory) { + this.queueName = queueKey.getQueueName(); + this.tenantId = queueKey.getTenantId(); this.statsFactory = statsFactory; String statsKey = StatsType.RULE_ENGINE.getName() + "." + queueName; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index b59086a350..d5be1243b7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -65,7 +65,6 @@ import java.util.stream.Collectors; @Slf4j public abstract class AbstractConsumerService extends TbApplicationEventListener { - protected volatile ExecutorService consumersExecutor; protected volatile ExecutorService notificationsConsumerExecutor; protected volatile boolean stopped = false; protected volatile boolean isReady = false; @@ -99,8 +98,7 @@ public abstract class AbstractConsumerService findAllQueues() { - return queueService.findAllQueues(); + @PostConstruct + public void init() { + this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer")); + this.mgmtExecutor = Executors.newFixedThreadPool(mgmtThreadPoolSize, ThingsBoardThreadFactory.forName("tb-rule-engine-mgmt")); + this.scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-scheduler")); } - @PreDestroy public void stop() { - mgmtExecutor.shutdownNow(); - consumersExecutor.shutdownNow(); // TODO: shutdown or shutdownNow? - submitExecutor.shutdownNow(); scheduler.shutdownNow(); + consumersExecutor.shutdown(); + mgmtExecutor.shutdown(); + try { + mgmtExecutor.awaitTermination(15, TimeUnit.SECONDS); + } catch (InterruptedException e) { + log.warn("Failed to await mgmtExecutor termination"); + } } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 9fbfd2fc63..6f51413ccd 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1303,7 +1303,9 @@ queue: pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}" # Time in seconds to wait in consumer thread before retries; max-pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}" # Max allowed time in seconds for pause between retries. # After a queue is deleted (or profile's isolation option was disabled), Rule Engine will continue reading related topics during this period, before deleting the actual topics - topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:30}" + topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:15}" + # Size of the thread pool that handles management operations such as subscribe, unsubscribe, queue delete, etc. + management-thread-pool-size: "${TB_QUEUE_RULE_ENGINE_MGMT_THREAD_POOL_SIZE:12}" transport: # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java index 845ac49f15..8418ba67d9 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java @@ -37,12 +37,14 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.queue.SubmitStrategy; import org.thingsboard.server.common.data.queue.SubmitStrategyType; import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.timeseries.TimeseriesDao; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; import org.thingsboard.server.service.stats.DefaultRuleEngineStatisticsService; @@ -163,7 +165,7 @@ public class BaseQueueControllerTest extends AbstractControllerTest { tenantId, ruleEngineException ))); - TbRuleEngineConsumerStats testStats = new TbRuleEngineConsumerStats(queue, statsFactory); + TbRuleEngineConsumerStats testStats = new TbRuleEngineConsumerStats(new QueueKey(ServiceType.TB_RULE_ENGINE, queue), statsFactory); testStats.log(testProcessingResult, true); int queueStatsTtlDays = 14; @@ -215,7 +217,7 @@ public class BaseQueueControllerTest extends AbstractControllerTest { tenantId, ruleEngineException ))); - TbRuleEngineConsumerStats testStats = new TbRuleEngineConsumerStats(queue, statsFactory); + TbRuleEngineConsumerStats testStats = new TbRuleEngineConsumerStats(new QueueKey(ServiceType.TB_RULE_ENGINE, queue), statsFactory); testStats.log(testProcessingResult, true); ruleEngineStatisticsService.reportQueueStats(System.currentTimeMillis(), testStats); From cf951ee2b22cde58b6fcd39a0f91d13213ce8b25 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 17 Oct 2023 11:23:58 +0300 Subject: [PATCH 16/29] Multiple improvements for Rule Engine consumer services --- .../DefaultTbRuleEngineConsumerService.java | 6 +- .../service/queue/ruleengine/QueueEvent.java | 17 +- .../TbQueueConsumerManagerTask.java | 25 +- .../queue/ruleengine/TbQueueConsumerTask.java | 43 +- .../TbRuleEngineConsumerContext.java | 19 +- .../TbRuleEngineQueueConsumerManager.java | 519 ++++++++++-------- .../server/queue/TbQueueConsumer.java | 6 +- .../AbstractTbQueueConsumerTemplate.java | 39 +- .../queue/kafka/TbKafkaConsumerTemplate.java | 1 - .../queue/memory/InMemoryTbQueueConsumer.java | 17 +- 10 files changed, 384 insertions(+), 308 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 164b48c25f..64139530b1 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -121,9 +121,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } @Override - protected void launchMainConsumers() { - consumers.values().forEach(TbRuleEngineQueueConsumerManager::launchMainConsumer); - } + protected void launchMainConsumers() {} @Override protected void stopConsumers() { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java index 4d01d15d9a..a4a3bcc59a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java @@ -1,9 +1,24 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue.ruleengine; import java.io.Serializable; public enum QueueEvent implements Serializable { - CREATED, LAUNCHED, UPDATED, PARTITION_CHANGE, STOP, DELETED + PARTITION_CHANGE, CONFIG_UPDATE, STOP, DELETE } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java index 5b4f164a10..6f1adbdcf2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java @@ -15,18 +15,33 @@ */ package org.thingsboard.server.service.queue.ruleengine; -import lombok.Data; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import lombok.Getter; +import lombok.ToString; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.Set; -@Data +@Getter +@ToString public class TbQueueConsumerManagerTask { private final QueueEvent event; - private final Queue queue; - private final Set partitions; + private Queue queue; + private Set partitions; + + public TbQueueConsumerManagerTask(QueueEvent event) { + this.event = event; + } + + public TbQueueConsumerManagerTask(QueueEvent event, Queue queue) { + this.event = event; + this.queue = queue; + } + + public TbQueueConsumerManagerTask(QueueEvent event, Set partitions) { + this.event = event; + this.partitions = partitions; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java index ed1f36074f..a0a5579951 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,51 +21,44 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.QueueKey; import java.util.Set; -import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; @Data @Slf4j public class TbQueueConsumerTask { - private final QueueKey key; - private final Object id; + private final Object key; private final TbQueueConsumer> consumer; private volatile Future task; - public void stop() { - this.consumer.stop(); + public void subscribe(Set partitions) { + log.trace("[{}] Subscribing to partitions: {}", key, partitions); + consumer.subscribe(partitions); } - public boolean stopAndAwait() { - this.consumer.stop(); - return await(); + public void initiateStop() { + log.debug("[{}] Initiating stop", key); + consumer.stop(); } - public boolean await() { - if (task != null) { - //TODO: maybe task.cancel() to interrupt the consumer? + public void awaitFinish() { + log.trace("[{}] Awaiting finish", key); + if (isRunning()) { try { - this.task.get(3, TimeUnit.MINUTES); - } catch (ExecutionException | InterruptedException | TimeoutException e) { - log.warn("[{}][{}] Failed to await for consumer to stop", key, id, e); - return false; + task.get(60, TimeUnit.SECONDS); + task = null; + } catch (Exception e) { + log.warn("[{}] Failed to await for consumer to stop", key, e); } } - return true; + log.trace("[{}] Awaited finish", key); } - public void subscribe(Set partitions) { - this.consumer.subscribe(partitions); + public boolean isRunning() { + return task != null && !task.isDone(); } - - public void unsubscribe() { - this.consumer.unsubscribe(); - } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java index c1d17746e1..325a685972 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue.ruleengine; import lombok.Data; @@ -36,8 +51,8 @@ public class TbRuleEngineConsumerContext { @Value("${queue.rule-engine.stats.enabled:true}") private boolean statsEnabled; @Value("${queue.rule-engine.prometheus-stats.enabled:false}") - boolean prometheusStatsEnabled; - @Value("${queue.rule-engine.topic-deletion-delay:30}") + private boolean prometheusStatsEnabled; + @Value("${queue.rule-engine.topic-deletion-delay:15}") private int topicDeletionDelayInSec; @Value("${queue.rule-engine.management-thread-pool-size:12}") private int mgmtThreadPoolSize; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index 2599a214f0..dc3966d0d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,8 +16,7 @@ package org.thingsboard.server.service.queue.ruleengine; import com.google.protobuf.ProtocolStringList; -import lombok.Data; -import lombok.SneakyThrows; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.id.EntityId; @@ -32,11 +31,10 @@ import org.thingsboard.server.common.msg.queue.RuleNodeInfo; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.QueueKey; -import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.service.queue.TbMsgPackCallback; import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; @@ -45,19 +43,20 @@ import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingRes import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; +import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; -@Data @Slf4j public class TbRuleEngineQueueConsumerManager { @@ -65,87 +64,83 @@ public class TbRuleEngineQueueConsumerManager { public static final String FAILED_STATUS = "failed"; private final TbRuleEngineConsumerContext ctx; - private final QueueKey key; - - private final ReentrantLock lock = new ReentrantLock(); //NonfairSync - private final ConcurrentMap consumersPerPartition = new ConcurrentHashMap<>(); + private final QueueKey queueKey; private final TbRuleEngineConsumerStats stats; + private final ReentrantLock lock = new ReentrantLock(); //NonfairSync - private volatile Set partitions = Collections.emptySet(); + @Getter private volatile Queue queue; - private volatile TbQueueConsumerTask mainConsumer; + @Getter + private volatile Set partitions; + private volatile ConsumerWrapper consumerWrapper; + + private volatile boolean stopped; private final java.util.Queue tasks = new ConcurrentLinkedQueue<>(); - public TbRuleEngineQueueConsumerManager(TbRuleEngineConsumerContext ctx, QueueKey key) { + public TbRuleEngineQueueConsumerManager(TbRuleEngineConsumerContext ctx, QueueKey queueKey) { this.ctx = ctx; - this.key = key; - this.stats = new TbRuleEngineConsumerStats(key, ctx.getStatsFactory()); + this.queueKey = queueKey; + this.stats = new TbRuleEngineConsumerStats(queueKey, ctx.getStatsFactory()); } public void init(Queue queue) { - processTask(new TbQueueConsumerManagerTask(QueueEvent.CREATED, queue, null)); + doInit(queue); } public void update(Queue queue) { - processTask(new TbQueueConsumerManagerTask(QueueEvent.UPDATED, queue, null)); + addTask(new TbQueueConsumerManagerTask(QueueEvent.CONFIG_UPDATE, queue)); } - public void subscribe(PartitionChangeEvent event) { - processTask(new TbQueueConsumerManagerTask(QueueEvent.PARTITION_CHANGE, queue, event.getPartitions())); - } - - public void launchMainConsumer() { - processTask(new TbQueueConsumerManagerTask(QueueEvent.LAUNCHED, null, null)); + public void update(Set partitions) { + addTask(new TbQueueConsumerManagerTask(QueueEvent.PARTITION_CHANGE, partitions)); } public void stop() { - processTask(new TbQueueConsumerManagerTask(QueueEvent.STOP, null, null)); + addTask(new TbQueueConsumerManagerTask(QueueEvent.STOP)); } public void delete() { - processTask(new TbQueueConsumerManagerTask(QueueEvent.DELETED, null, null)); + addTask(new TbQueueConsumerManagerTask(QueueEvent.DELETE)); } - private void processTask(TbQueueConsumerManagerTask todo) { + private void addTask(TbQueueConsumerManagerTask todo) { + if (stopped) { + return; + } tasks.add(todo); - log.info("[{}] Adding task: {}", key, todo); + log.trace("[{}] Added task: {}", queueKey, todo); tryProcessTasks(); } private void tryProcessTasks() { + if (!ctx.isReady()) { + log.debug("[{}] TbRuleEngineConsumerContext is not ready yet, will process tasks later", queueKey); + ctx.getScheduler().schedule(this::tryProcessTasks, 1, TimeUnit.SECONDS); + return; + } ctx.getMgmtExecutor().submit(() -> { if (lock.tryLock()) { try { Queue newConfiguration = null; Set newPartitions = null; - while (!tasks.isEmpty()) { + while (!stopped) { TbQueueConsumerManagerTask task = tasks.poll(); - switch (task.getEvent()) { - case CREATED: - doInit(task.getQueue()); - break; - case LAUNCHED: - if (!queue.isConsumerPerPartition()) { - doLaunchMainConsumer(); - } - break; - case UPDATED: - newConfiguration = task.getQueue(); - break; - case PARTITION_CHANGE: - newPartitions = task.getPartitions(); - break; - case STOP: - newConfiguration = null; - newPartitions = null; - doStop(); - break; - case DELETED: - newConfiguration = null; - newPartitions = null; - doDelete(); - break; + if (task == null) { + break; + } + log.trace("[{}] Processing task: {}", queueKey, task); + + if (task.getEvent() == QueueEvent.PARTITION_CHANGE) { + newPartitions = task.getPartitions(); + } else if (task.getEvent() == QueueEvent.CONFIG_UPDATE) { + newConfiguration = task.getQueue(); + } else if (task.getEvent() == QueueEvent.STOP) { + doStop(); + return; + } else if (task.getEvent() == QueueEvent.DELETE) { + doDelete(); + return; } } if (newConfiguration != null) { @@ -158,158 +153,108 @@ public class TbRuleEngineQueueConsumerManager { lock.unlock(); } } else { - log.debug("[{}] Failed to acquire lock.", key); + log.trace("[{}] Failed to acquire lock", queueKey); ctx.getScheduler().schedule(this::tryProcessTasks, 1, TimeUnit.SECONDS); } }); } - public void doInit(Queue queue) { - log.info("[{}] Init consumer with queue: {}", key, queue); + private void doInit(Queue queue) { this.queue = queue; if (queue.isConsumerPerPartition()) { - log.debug("[{}] Ignore init event since isConsumerPerPartition is enabled.", key); + consumerWrapper = new ConsumerPerPartitionWrapper(); } else { - mainConsumer = new TbQueueConsumerTask(key, "main", ctx.getQueueFactory().createToRuleEngineMsgConsumer(queue)); - } - } - - private void doLaunchMainConsumer() { - if (mainConsumer != null) { - launchConsumer(mainConsumer, queue, mainConsumer.getId(), queue.getName()); - } else { - log.warn("[{}] Can't launch main consumer since it is empty!", key); + consumerWrapper = new SingleConsumerWrapper(); } + log.debug("[{}] Initialized consumer for queue: {}", queueKey, queue); } private void doUpdate(Queue newQueue) { - log.info("[{}] Processing queue update: {}", key, newQueue); - var oldQueue = queue; + log.info("[{}] Processing queue update: {}", queueKey, newQueue); + var oldQueue = this.queue; + this.queue = newQueue; if (log.isTraceEnabled()) { - log.trace("[{}] Old queue configuration: {}", key, oldQueue); - log.trace("[{}] New queue configuration: {}", key, newQueue); - } - if (oldQueue != null) { - doStop(oldQueue); - } - doInit(newQueue); - if (!newQueue.isConsumerPerPartition()) { - doLaunchMainConsumer(); + log.trace("[{}] Old queue configuration: {}", queueKey, oldQueue); + log.trace("[{}] New queue configuration: {}", queueKey, newQueue); } - } - - private void doUpdate(Set partitions) { - log.info("[{}] Subscribing to partitions: {}", key, partitions); - if (queue.isConsumerPerPartition()) { - log.debug("[{}] Subscribing consumers per partition separately: {}", key, partitions); - Set addedPartitions = new HashSet<>(partitions); - addedPartitions.removeAll(consumersPerPartition.keySet()); - log.info("calculated addedPartitions {}", addedPartitions); - - Set removedPartitions = new HashSet<>(consumersPerPartition.keySet()); - removedPartitions.removeAll(partitions); - log.info("calculated removedPartitions {}", removedPartitions); - removedPartitions.forEach((tpi) -> { - log.info("[{}] Unsubscribing from topic: {}", queue, tpi); - consumersPerPartition.get(tpi).unsubscribe(); - }); + if (oldQueue == null) { + doInit(queue); + } else if (newQueue.isConsumerPerPartition() != oldQueue.isConsumerPerPartition()) { + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::initiateStop); + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitFinish); - removedPartitions.forEach((tpi) -> { - log.info("[{}] Removing consumer for topic: {}", queue, tpi); - consumersPerPartition.get(tpi).stopAndAwait(); - consumersPerPartition.remove(tpi); - }); - - addedPartitions.forEach((tpi) -> { - log.info("[{}] Adding consumer for topic: {}", key, tpi); - TbQueueConsumerTask consumerTask = new TbQueueConsumerTask(key, tpi, ctx.getQueueFactory().createToRuleEngineMsgConsumer(queue)); - consumersPerPartition.put(tpi, consumerTask); - //TODO: Is it ok to subscribe first? - consumerTask.subscribe(Collections.singleton(tpi)); - launchConsumer(consumerTask, queue, mainConsumer.getId(), key + "-" + tpi.getPartition().orElse(-999999)); - }); + doInit(queue); + if (partitions != null) { + doUpdate(partitions); // even if partitions number was changed, there can be no partition change event + } } else { - mainConsumer.subscribe(partitions); + // do nothing, because partitions change (if they changed) will be handled on PartitionChangeEvent, + // and changes to pollInterval/packProcessingTimeout/submitStrategy/processingStrategy will be picked up by consumer on the fly, + // and queue topic and name are immutable } } - private void doStop() { - doStop(queue); + private void doUpdate(Set partitions) { + this.partitions = partitions; + consumerWrapper.updatePartitions(partitions); } - private void doStop(Queue queue) { - if (queue.isConsumerPerPartition()) { - consumersPerPartition.values().forEach(TbQueueConsumerTask::unsubscribe); - consumersPerPartition.values().forEach(TbQueueConsumerTask::stopAndAwait); - } else if (mainConsumer != null) { - mainConsumer.unsubscribe(); - mainConsumer.stopAndAwait(); - } + private void doStop() { + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::initiateStop); + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitFinish); + log.debug("[{}] Unsubscribed and stopped consumers", queueKey); + stopped = true; } private void doDelete() { - doStop(); - //TODO: repack messages + stopped = true; + log.info("[{}] Handling queue deletion", queueKey); + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitFinish); + + List>> queueConsumers = consumerWrapper.getConsumers().stream() + .map(TbQueueConsumerTask::getConsumer).collect(Collectors.toList()); + ctx.getConsumersExecutor().submit(() -> { + drainQueue(queueConsumers); + + queueConsumers.forEach(consumer -> { + for (String topic : consumer.getFullTopicNames()) { + try { + ctx.getQueueAdmin().deleteTopic(topic); + log.info("Deleted topic {}", topic); + } catch (Exception e) { + log.error("Failed to delete topic {}", topic, e); + } + } + try { + consumer.unsubscribe(); + } catch (Exception e) { + log.error("[{}] Failed to unsubscribe consumer", queueKey, e); + } + }); + }); } - @SneakyThrows - void launchConsumer(TbQueueConsumerTask consumerTask, Queue configuration, Object consumerKey, String threadSuffix) { - log.info("[{}] Launching consumer: [{}]", key, consumerKey); - while (!ctx.isReady) { - //TODO: Remember this task. Cancel previous task if needed. - log.debug("[{}][{}] Waiting for consumer to get ready..", key, consumerKey); - Thread.sleep(1000); - } - consumerTask.setTask(ctx.getConsumersExecutor().submit(() -> consumerLoop(consumerTask.getConsumer(), configuration, threadSuffix))); + private void launchConsumer(TbQueueConsumerTask consumerTask) { + log.info("[{}] Launching consumer", consumerTask.getKey()); + Future consumerLoop = ctx.getConsumersExecutor().submit(() -> { + ThingsBoardThreadFactory.updateCurrentThreadName(consumerTask.getKey().toString()); + consumerLoop(consumerTask.getConsumer()); + }); + consumerTask.setTask(consumerLoop); } - void consumerLoop(TbQueueConsumer> consumer, Queue configuration, String threadSuffix) { - ThingsBoardThreadFactory.updateCurrentThreadName(threadSuffix); - while (!ctx.stopped && !consumer.isStopped() - //TODO: remove this. - && !consumer.isQueueDeleted()) { + private void consumerLoop(TbQueueConsumer> consumer) { + while (!stopped && !consumer.isStopped()) { try { - List> msgs = consumer.poll(queue.getPollInterval()); + List> msgs = consumer.poll(queue.getPollInterval()); if (msgs.isEmpty()) { continue; } - final TbRuleEngineSubmitStrategy submitStrategy = getSubmitStrategy(queue); - final TbRuleEngineProcessingStrategy ackStrategy = getProcessingStrategy(queue); - submitStrategy.init(msgs); - while (!ctx.isStopped() && !consumer.isStopped()) { - TbMsgPackProcessingContext packCtx = new TbMsgPackProcessingContext(queue.getName(), submitStrategy, ackStrategy.isSkipTimeoutMsgs()); - submitStrategy.submitAttempt((id, msg) -> ctx.getSubmitExecutor().submit(() -> submitMessage(configuration, stats, packCtx, id, msg))); - - final boolean timeout = !packCtx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS); - - TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(configuration.getName(), timeout, packCtx); - if (timeout) { - printFirstOrAll(configuration, packCtx, packCtx.getPendingMap(), "Timeout"); - } - if (!packCtx.getFailedMap().isEmpty()) { - printFirstOrAll(configuration, packCtx, packCtx.getFailedMap(), "Failed"); - } - packCtx.printProfilerStats(); - - TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result); - if (ctx.isStatsEnabled()) { - stats.log(result, decision.isCommit()); - } - - packCtx.cleanup(); - - if (decision.isCommit()) { - submitStrategy.stop(); - break; - } else { - submitStrategy.update(decision.getReprocessMap()); - } - } - consumer.commit(); + processMsgs(msgs, consumer, queue); } catch (Exception e) { - if (!ctx.stopped) { - log.warn("Failed to process messages from queue.", e); + if (!stopped) { + log.warn("Failed to process messages from queue", e); try { Thread.sleep(ctx.getPollDuration()); } catch (InterruptedException e2) { @@ -318,72 +263,68 @@ public class TbRuleEngineQueueConsumerManager { } } } - //TODO: refactor and move to the "doDelete" method. Use separate consumer if needed (it is still synchronous). - if (consumer.isQueueDeleted()) { - processQueueDeletion(configuration, consumer); + if (consumer.isStopped()) { + consumer.unsubscribe(); } - log.info("TB Rule Engine Consumer stopped."); + log.info("Rule Engine consumer stopped"); } - private void processQueueDeletion(Queue queue, TbQueueConsumer> consumer) { -// long finishTs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(topicDeletionDelayInSec); -// try { -// int n = 0; -// while (System.currentTimeMillis() <= finishTs) { -// List> msgs = consumer.poll(queue.getPollInterval()); -// if (msgs.isEmpty()) { -// continue; -// } -// for (TbProtoQueueMsg msg : msgs) { -// try { -// MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray()); -// EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB())); -// -// TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue.getName(), TenantId.SYS_TENANT_ID, originator); -// producerProvider.getRuleEngineMsgProducer().send(tpi, msg, null); -// n++; -// } catch (Throwable e) { -// log.debug("Failed to move message to system {}: {}", consumer.getTopic(), msg, e); -// } -// } -// consumer.commit(); -// } -// if (n > 0) { -// log.info("Moved {} messages from {} to system {}", n, consumer.getFullTopicNames(), consumer.getTopic()); -// } -// -// consumer.unsubscribe(); -// for (String topic : consumer.getFullTopicNames()) { -// try { -// queueAdmin.deleteTopic(topic); -// log.info("Deleted topic {}", topic); -// } catch (Exception e) { -// log.error("Failed to delete topic {} after unsubscribing", topic, e); -// } -// } -// } catch (Exception e) { -// log.error("Failed to process deletion of {} ({})", consumer.getTopic(), queue.getTenantId(), e); -// } + private void processMsgs(List> msgs, + TbQueueConsumer> consumer, + Queue queue) throws InterruptedException { + TbRuleEngineSubmitStrategy submitStrategy = getSubmitStrategy(queue); + TbRuleEngineProcessingStrategy ackStrategy = getProcessingStrategy(queue); + submitStrategy.init(msgs); + while (!stopped && !consumer.isStopped()) { + TbMsgPackProcessingContext packCtx = new TbMsgPackProcessingContext(queue.getName(), submitStrategy, ackStrategy.isSkipTimeoutMsgs()); + submitStrategy.submitAttempt((id, msg) -> submitMessage(packCtx, id, msg)); + + final boolean timeout = !packCtx.await(queue.getPackProcessingTimeout(), TimeUnit.MILLISECONDS); + + TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(queue.getName(), timeout, packCtx); + if (timeout) { + printFirstOrAll(packCtx, packCtx.getPendingMap(), "Timeout"); + } + if (!packCtx.getFailedMap().isEmpty()) { + printFirstOrAll(packCtx, packCtx.getFailedMap(), "Failed"); + } + packCtx.printProfilerStats(); + + TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result); + if (ctx.isStatsEnabled()) { + stats.log(result, decision.isCommit()); + } + + packCtx.cleanup(); + + if (decision.isCommit()) { + submitStrategy.stop(); + consumer.commit(); + break; + } else { + submitStrategy.update(decision.getReprocessMap()); + } + } } - TbRuleEngineSubmitStrategy getSubmitStrategy(Queue configuration) { - return ctx.getSubmitStrategyFactory().newInstance(configuration.getName(), configuration.getSubmitStrategy()); + private TbRuleEngineSubmitStrategy getSubmitStrategy(Queue queue) { + return ctx.getSubmitStrategyFactory().newInstance(queue.getName(), queue.getSubmitStrategy()); } - TbRuleEngineProcessingStrategy getProcessingStrategy(Queue configuration) { - return ctx.getProcessingStrategyFactory().newInstance(configuration.getName(), configuration.getProcessingStrategy()); + private TbRuleEngineProcessingStrategy getProcessingStrategy(Queue queue) { + return ctx.getProcessingStrategyFactory().newInstance(queue.getName(), queue.getProcessingStrategy()); } - void submitMessage(Queue configuration, TbRuleEngineConsumerStats stats, TbMsgPackProcessingContext packCtx, UUID id, TbProtoQueueMsg msg) { - log.trace("[{}] Creating callback for topic {} message: {}", id, configuration.getName(), msg.getValue()); - TransportProtos.ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); + private void submitMessage(TbMsgPackProcessingContext packCtx, UUID id, TbProtoQueueMsg msg) { + log.trace("[{}] Creating callback for topic {} message: {}", id, queue.getName(), msg.getValue()); + ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); TenantId tenantId = TenantId.fromUUID(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); - TbMsgCallback callback = ctx.prometheusStatsEnabled ? + TbMsgCallback callback = ctx.isPrometheusStatsEnabled() ? new TbMsgPackCallback(id, tenantId, packCtx, stats.getTimer(tenantId, SUCCESSFUL_STATUS), stats.getTimer(tenantId, FAILED_STATUS)) : new TbMsgPackCallback(id, tenantId, packCtx); try { if (!toRuleEngineMsg.getTbMsg().isEmpty()) { - forwardToRuleEngineActor(configuration.getName(), tenantId, toRuleEngineMsg, callback); + forwardToRuleEngineActor(queue.getName(), tenantId, toRuleEngineMsg, callback); } else { callback.onSuccess(); } @@ -392,7 +333,7 @@ public class TbRuleEngineQueueConsumerManager { } } - private void forwardToRuleEngineActor(String queueName, TenantId tenantId, TransportProtos.ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { + private void forwardToRuleEngineActor(String queueName, TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { TbMsg tbMsg = TbMsg.fromBytes(queueName, toRuleEngineMsg.getTbMsg().toByteArray(), callback); QueueToRuleEngineMsg msg; ProtocolStringList relationTypesList = toRuleEngineMsg.getRelationTypesList(); @@ -406,16 +347,15 @@ public class TbRuleEngineQueueConsumerManager { ctx.getActorContext().tell(msg); } - - private void printFirstOrAll(Queue configuration, TbMsgPackProcessingContext ctx, Map> map, String prefix) { + private void printFirstOrAll(TbMsgPackProcessingContext ctx, Map> map, String prefix) { boolean printAll = log.isTraceEnabled(); - log.info("{} to process [{}] messages", prefix, map.size()); - for (Map.Entry> pending : map.entrySet()) { - TransportProtos.ToRuleEngineMsg tmp = pending.getValue().getValue(); - TbMsg tmpMsg = TbMsg.fromBytes(configuration.getName(), tmp.getTbMsg().toByteArray(), TbMsgCallback.EMPTY); + log.info("[{}] {} to process [{}] messages", queueKey, prefix, map.size()); + for (Map.Entry> pending : map.entrySet()) { + ToRuleEngineMsg tmp = pending.getValue().getValue(); + TbMsg tmpMsg = TbMsg.fromBytes(queue.getName(), tmp.getTbMsg().toByteArray(), TbMsgCallback.EMPTY); RuleNodeInfo ruleNodeInfo = ctx.getLastVisitedRuleNode(pending.getKey()); if (printAll) { - log.trace("[{}] {} to process message: {}, Last Rule Node: {}", TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); + log.trace("[{}][{}] {} to process message: {}, Last Rule Node: {}", queueKey, TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); } else { log.info("[{}] {} to process message: {}, Last Rule Node: {}", TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); break; @@ -428,4 +368,117 @@ public class TbRuleEngineQueueConsumerManager { ctx.getStatisticsService().reportQueueStats(ts, stats); stats.reset(); } + + private void drainQueue(List>> consumers) { + long finishTs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(ctx.getTopicDeletionDelayInSec()); + try { + int n = 0; + while (System.currentTimeMillis() <= finishTs) { + for (TbQueueConsumer> consumer : consumers) { + List> msgs = consumer.poll(queue.getPollInterval()); + if (msgs.isEmpty()) { + continue; + } + for (TbProtoQueueMsg msg : msgs) { + try { + MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray()); + EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB())); + + TopicPartitionInfo tpi = ctx.getPartitionService().resolve(ServiceType.TB_RULE_ENGINE, queue.getName(), TenantId.SYS_TENANT_ID, originator); + ctx.getProducerProvider().getRuleEngineMsgProducer().send(tpi, msg, null); + n++; + } catch (Throwable e) { + log.warn("Failed to move message to system {}: {}", consumer.getTopic(), msg, e); + } + } + consumer.commit(); + } + } + if (n > 0) { + log.info("Moved {} messages from {} to system {}", n, queueKey, queue.getName()); + } + } catch (Exception e) { + log.error("[{}] Failed to drain queue", queueKey, e); + } + } + + private static String partitionsToString(Collection partitions) { + return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.joining(", ", "[", "]")); + } + + interface ConsumerWrapper { + + void updatePartitions(Set partitions); + + Collection getConsumers(); + + } + + class ConsumerPerPartitionWrapper implements ConsumerWrapper { + private final Map consumers = new HashMap<>(); + + @Override + public void updatePartitions(Set partitions) { + Set addedPartitions = new HashSet<>(partitions); + addedPartitions.removeAll(consumers.keySet()); + + Set removedPartitions = new HashSet<>(consumers.keySet()); + removedPartitions.removeAll(partitions); + log.info("[{}] Added partitions: {}, removed partitions: {}", queueKey, partitionsToString(addedPartitions), partitionsToString(removedPartitions)); + + removedPartitions.forEach((tpi) -> { + consumers.get(tpi).initiateStop(); + }); + removedPartitions.forEach((tpi) -> { + consumers.remove(tpi).awaitFinish(); + }); + + addedPartitions.forEach((tpi) -> { + String key = queueKey + "-" + tpi.getPartition().orElse(-999999); + TbQueueConsumerTask consumer = new TbQueueConsumerTask(key, ctx.getQueueFactory().createToRuleEngineMsgConsumer(queue)); + consumers.put(tpi, consumer); + consumer.subscribe(Set.of(tpi)); + launchConsumer(consumer); + }); + } + + @Override + public Collection getConsumers() { + return consumers.values(); + } + } + + class SingleConsumerWrapper implements ConsumerWrapper { + private TbQueueConsumerTask consumer; + + @Override + public void updatePartitions(Set partitions) { + log.info("[{}] New partitions: {}", queueKey, partitionsToString(partitions)); + if (partitions.isEmpty()) { + if (consumer != null && consumer.isRunning()) { + consumer.initiateStop(); + consumer.awaitFinish(); + } + consumer = null; + return; + } + + if (consumer == null) { + consumer = new TbQueueConsumerTask(queueKey, ctx.getQueueFactory().createToRuleEngineMsgConsumer(queue)); + } + consumer.subscribe(partitions); + if (!consumer.isRunning()) { + launchConsumer(consumer); + } + } + + @Override + public Collection getConsumers() { + if (consumer == null) { + return Collections.emptyList(); + } + return List.of(consumer); + } + } + } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java index 9c41f9d342..21216e164e 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java @@ -28,6 +28,8 @@ public interface TbQueueConsumer { void subscribe(Set partitions); + void stop(); + void unsubscribe(); List poll(long durationInMillis); @@ -36,10 +38,6 @@ public interface TbQueueConsumer { boolean isStopped(); - void onQueueDelete(); - - boolean isQueueDeleted(); - List getFullTopicNames(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 2ebe41850d..073371ebb5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -44,7 +44,6 @@ public abstract class AbstractTbQueueConsumerTemplate i protected volatile Set partitions; protected final ReentrantLock consumerLock = new ReentrantLock(); //NonfairSync final Queue> subscribeQueue = new ConcurrentLinkedQueue<>(); - protected volatile boolean queueDeleted = false; @Getter private final String topic; @@ -55,7 +54,7 @@ public abstract class AbstractTbQueueConsumerTemplate i @Override public void subscribe() { - log.info("enqueue topic subscribe {} ", topic); + log.debug("enqueue topic subscribe {} ", topic); if (stopped) { log.error("trying subscribe, but consumer stopped for topic {}", topic); return; @@ -65,7 +64,7 @@ public abstract class AbstractTbQueueConsumerTemplate i @Override public void subscribe(Set partitions) { - log.info("enqueue topics subscribe {} ", partitions); + log.debug("enqueue topics subscribe {} ", partitions); if (stopped) { log.error("trying subscribe, but consumer stopped for topic {}", topic); return; @@ -78,7 +77,8 @@ public abstract class AbstractTbQueueConsumerTemplate i List records; long startNanos = System.nanoTime(); if (stopped) { - return errorAndReturnEmpty(); + log.error("poll invoked but consumer stopped for topic " + topic, new RuntimeException("stacktrace")); + return emptyList(); } if (!subscribed && partitions == null && subscribeQueue.isEmpty()) { return sleepAndReturnEmpty(startNanos, durationInMillis); @@ -96,6 +96,7 @@ public abstract class AbstractTbQueueConsumerTemplate i } if (!subscribed) { List topicNames = getFullTopicNames(); + log.info("Subscribing to topics {}", topicNames); doSubscribe(topicNames); subscribed = true; } @@ -127,11 +128,6 @@ public abstract class AbstractTbQueueConsumerTemplate i return result; } - List errorAndReturnEmpty() { - log.error("poll invoked but consumer stopped for topic" + topic, new RuntimeException("stacktrace")); - return emptyList(); - } - List sleepAndReturnEmpty(final long startNanos, final long durationInMillis) { long durationNanos = TimeUnit.MILLISECONDS.toNanos(durationInMillis); long spentNanos = System.nanoTime() - startNanos; @@ -163,15 +159,20 @@ public abstract class AbstractTbQueueConsumerTemplate i } } + @Override + public void stop() { + stopped = true; + } + @Override public void unsubscribe() { - log.info("Unsubscribing from topics and stopping consumer for topics {}", partitions.stream() - .map(TopicPartitionInfo::getFullTopicName) - .collect(Collectors.joining(", "))); + log.info("Unsubscribing and stopping consumer for topics {}", getFullTopicNames()); stopped = true; consumerLock.lock(); try { - doUnsubscribe(); + if (subscribed) { + doUnsubscribe(); + } } finally { consumerLock.unlock(); } @@ -192,17 +193,11 @@ public abstract class AbstractTbQueueConsumerTemplate i abstract protected void doUnsubscribe(); - @Override - public void onQueueDelete() { - queueDeleted = true; - } - - public boolean isQueueDeleted() { - return queueDeleted; - } - @Override public List getFullTopicNames() { + if (partitions == null) { + return Collections.emptyList(); + } return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 9f58446966..ff3f0cc9b3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -73,7 +73,6 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue protected void doSubscribe(List topicNames) { if (!topicNames.isEmpty()) { topicNames.forEach(admin::createTopicIfNotExists); - log.info("subscribe topics {}", topicNames); consumer.subscribe(topicNames); } else { log.info("unsubscribe due to empty topic list"); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index 8711cbbcf1..a7f8cadd0d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -31,7 +31,6 @@ public class InMemoryTbQueueConsumer implements TbQueueCon private volatile Set partitions; private volatile boolean stopped; private volatile boolean subscribed; - private volatile boolean queueDeleted; public InMemoryTbQueueConsumer(InMemoryStorage storage, String topic) { this.storage = storage; @@ -58,9 +57,15 @@ public class InMemoryTbQueueConsumer implements TbQueueCon subscribed = true; } + @Override + public void stop() { + stopped = true; + } + @Override public void unsubscribe() { stopped = true; + subscribed = false; } @Override @@ -104,16 +109,6 @@ public class InMemoryTbQueueConsumer implements TbQueueCon return stopped; } - @Override - public void onQueueDelete() { - queueDeleted = true; - } - - @Override - public boolean isQueueDeleted() { - return queueDeleted; - } - @Override public List getFullTopicNames() { return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); From 921b2ae4b7d41b58c616b79d3d62bce70ea6182b Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 17 Oct 2023 11:24:33 +0300 Subject: [PATCH 17/29] Tests for Rule Engine consumer manager --- .../controller/TenantControllerTest.java | 40 +- .../TbRuleEngineQueueConsumerManagerTest.java | 780 ++++++++++++++++++ 2 files changed, 801 insertions(+), 19 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java index 4b594d55f9..d1aefc097f 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java @@ -42,6 +42,7 @@ import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; @@ -54,12 +55,12 @@ import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; -import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.discovery.PartitionService; @@ -67,9 +68,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; -import java.util.Deque; import java.util.HashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Random; @@ -680,26 +679,29 @@ public class TenantControllerTest extends AbstractControllerTest { .until(() -> partitionService.resolve(ServiceType.TB_RULE_ENGINE, MAIN_QUEUE_NAME, tenantId, tenantId) .getTenantId().get().isSysTenantId()); - Deque submittedMsgs = new LinkedList<>(); - await().atLeast(8, TimeUnit.SECONDS) // due to topic-deletion-delay - .atMost(20, TimeUnit.SECONDS) - .pollInterval(1, TimeUnit.SECONDS) - .untilAsserted(() -> { - TbMsg tbMsg = publishTbMsg(tenantId, tpi); - submittedMsgs.add(tbMsg.getId()); - - verify(queueAdmin, times(1)).deleteTopic(eq(isolatedTopic)); - }); - submittedMsgs.removeLast(); - for (UUID msgId : submittedMsgs) { - verify(actorContext, timeout(2000)).tell(argThat(msg -> { - return msg instanceof QueueToRuleEngineMsg && ((QueueToRuleEngineMsg) msg).getMsg().getId().equals(msgId); - })); + List submittedMsgs = new ArrayList<>(); + long timeLeft = TimeUnit.SECONDS.toMillis(7); // based on topic-deletion-delay + int msgs = 100; + for (int i = 1; i <= msgs; i++) { + TbMsg tbMsg = publishTbMsg(tenantId, tpi); + submittedMsgs.add(tbMsg.getId()); + Thread.sleep(timeLeft / msgs); } + await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> { + verify(queueAdmin, times(1)).deleteTopic(eq(isolatedTopic)); + }); + + await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> { + for (UUID msgId : submittedMsgs) { + verify(actorContext).tell(argThat(msg -> { + return msg instanceof QueueToRuleEngineMsg && ((QueueToRuleEngineMsg) msg).getMsg().getId().equals(msgId); + })); + } + }); } private TbMsg publishTbMsg(TenantId tenantId, TopicPartitionInfo tpi) { - TbMsg tbMsg = TbMsg.newMsg("POST_TELEMETRY_REQUEST", tenantId, TbMsgMetaData.EMPTY, "{\"test\":1}"); + TbMsg tbMsg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, tenantId, TbMsgMetaData.EMPTY, "{\"test\":1}"); TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java new file mode 100644 index 0000000000..276e65ec39 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java @@ -0,0 +1,780 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.ruleengine; + +import lombok.Getter; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomUtils; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.QueueId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.data.queue.ProcessingStrategy; +import org.thingsboard.server.common.data.queue.ProcessingStrategyType; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.data.queue.SubmitStrategy; +import org.thingsboard.server.common.data.queue.SubmitStrategyType; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; +import org.thingsboard.server.service.stats.RuleEngineStatisticsService; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.after; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +@Slf4j +@RunWith(MockitoJUnitRunner.class) +public class TbRuleEngineQueueConsumerManagerTest { + + @Mock + private ActorSystemContext actorContext; + @Mock + private StatsFactory statsFactory; + @Mock + private TbRuleEngineQueueFactory queueFactory; + @Mock + private RuleEngineStatisticsService statisticsService; + @Mock + private TbServiceInfoProvider serviceInfoProvider; + @Mock + private PartitionService partitionService; + @Mock + private TbQueueProducerProvider producerProvider; + private TbQueueProducer> ruleEngineMsgProducer; + @Mock + private TbQueueAdmin queueAdmin; + private TbRuleEngineConsumerContext ruleEngineConsumerContext; + + private TbRuleEngineQueueConsumerManager consumerManager; + private Queue queue; + + private Set consumers; + private boolean generateQueueMsgs; + private AtomicInteger totalConsumedMsgs; + private AtomicInteger totalProcessedMsgs; + + @Before + public void beforeEach() { + ruleEngineConsumerContext = new TbRuleEngineConsumerContext( + actorContext, statsFactory, spy(new TbRuleEngineSubmitStrategyFactory()), + spy(new TbRuleEngineProcessingStrategyFactory()), queueFactory, statisticsService, + serviceInfoProvider, partitionService, producerProvider, queueAdmin + ); + consumers = ConcurrentHashMap.newKeySet(); + generateQueueMsgs = true; + totalConsumedMsgs = new AtomicInteger(); + totalProcessedMsgs = new AtomicInteger(); + doAnswer(inv -> { + QueueToRuleEngineMsg msg = inv.getArgument(0); + msg.getMsg().getCallback().onSuccess(); + totalProcessedMsgs.incrementAndGet(); + log.trace("totalProcessedMsgs = {}", totalProcessedMsgs); + return null; + }).when(actorContext).tell(any()); + ruleEngineMsgProducer = mock(TbQueueProducer.class); + when(producerProvider.getRuleEngineMsgProducer()).thenReturn(ruleEngineMsgProducer); + ruleEngineConsumerContext.setMgmtThreadPoolSize(2); + ruleEngineConsumerContext.setTopicDeletionDelayInSec(5); + ruleEngineConsumerContext.init(); + ruleEngineConsumerContext.setReady(false); + + queue = new Queue(); + queue.setName("Test"); + queue.setTenantId(TenantId.SYS_TENANT_ID); + queue.setId(new QueueId(UUID.randomUUID())); + queue.setTopic("tb_test"); + queue.setPartitions(10); + queue.setConsumerPerPartition(true); + queue.setPollInterval(250); + queue.setPackProcessingTimeout(2000); + SubmitStrategy submitStrategy = new SubmitStrategy(); + submitStrategy.setType(SubmitStrategyType.BURST); + submitStrategy.setBatchSize(200); + queue.setSubmitStrategy(submitStrategy); + ProcessingStrategy processingStrategy = new ProcessingStrategy(); + processingStrategy.setType(ProcessingStrategyType.SKIP_ALL_FAILURES_AND_TIMED_OUT); + processingStrategy.setRetries(0); + queue.setProcessingStrategy(processingStrategy); + + doAnswer(i -> { + TestConsumer consumer = spy(new TestConsumer(queue.getTopic())); + if (generateQueueMsgs) { + consumer.setUpTestMsg(); + } + consumers.add(consumer); + return consumer; + }).when(queueFactory).createToRuleEngineMsgConsumer(any()); + + QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queue); + consumerManager = new TbRuleEngineQueueConsumerManager(ruleEngineConsumerContext, queueKey); + } + + @After + public void afterEach() { + consumerManager.stop(); + ruleEngineConsumerContext.stop(); + + if (generateQueueMsgs) { + await().atMost(1, TimeUnit.SECONDS) + .until(() -> totalProcessedMsgs.get() == totalConsumedMsgs.get()); + } + log.debug("totalConsumedMsgs = {}, totalProcessedMsgs = {}", totalConsumedMsgs.get(), totalProcessedMsgs.get()); + } + + @Test + public void testInit_consumerPerPartition() { + queue.setConsumerPerPartition(true); + consumerManager.init(queue); + + Set partitions = createTpis(2, 3, 4); + consumerManager.update(partitions); + partitions = createTpis(3, 4, 5); + consumerManager.update(partitions); + partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + // simulated multiple partition change events before consumer is ready; only latest partitions should be processed + verifyNoInteractions(queueFactory); + + ruleEngineConsumerContext.setReady(true); + await().atMost(2, TimeUnit.SECONDS) + .until(() -> consumers.size() == 3); + for (TopicPartitionInfo partition : partitions) { + TestConsumer consumer = getConsumer(partition); + verifySubscribedAndLaunched(consumer, Set.of(partition)); + } + } + + @Test + public void testInit_singleConsumer() { + queue.setConsumerPerPartition(false); + consumerManager.init(queue); + + Set partitions = createTpis(2, 3, 4); + consumerManager.update(partitions); + partitions = createTpis(3, 4, 5); + consumerManager.update(partitions); + partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + + verifyNoInteractions(queueFactory); + + ruleEngineConsumerContext.setReady(true); + await().atMost(2, TimeUnit.SECONDS) + .until(() -> consumers.size() == 1); + TestConsumer consumer = getConsumer(); + verifySubscribedAndLaunched(consumer, partitions); + } + + @Test + public void testPartitionsUpdate_singleConsumer() { + queue.setConsumerPerPartition(false); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + + Set partitions = Collections.emptySet(); + consumerManager.update(partitions); + verify(queueFactory, after(1000).never()).createToRuleEngineMsgConsumer(any()); + + partitions = createTpis(1); + consumerManager.update(partitions); + TestConsumer consumer = getConsumer(); + verifySubscribedAndLaunched(consumer, partitions); + + partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + verifySubscribedAndLaunched(consumer, partitions); + + partitions = createTpis(4, 5, 6); + consumerManager.update(partitions); + verifySubscribedAndLaunched(consumer, partitions); + + partitions = Collections.emptySet(); + consumerManager.update(partitions); + verifyUnsubscribedAndStopped(consumer); + + partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + consumer = getConsumer(); + verifySubscribedAndLaunched(consumer, partitions); + } + + @Test + public void testPartitionsUpdate_consumerPerPartition() { + queue.setConsumerPerPartition(true); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + + consumerManager.update(Collections.emptySet()); + verify(queueFactory, after(1000).never()).createToRuleEngineMsgConsumer(any()); + + consumerManager.update(createTpis(1)); + TestConsumer consumer1 = getConsumer(1); + verifySubscribedAndLaunched(consumer1, 1); + + consumerManager.update(createTpis(1, 2, 3)); + TestConsumer consumer2 = getConsumer(2); + TestConsumer consumer3 = getConsumer(3); + verifySubscribedAndLaunched(consumer2, 2); + verifySubscribedAndLaunched(consumer3, 3); + verifyNotTouched(consumer1); + + consumerManager.update(createTpis(3, 4, 5)); + TestConsumer consumer4 = getConsumer(4); + TestConsumer consumer5 = getConsumer(5); + verifySubscribedAndLaunched(consumer4, 4); + verifySubscribedAndLaunched(consumer5, 5); + verifyUnsubscribedAndStopped(consumer1); + verifyUnsubscribedAndStopped(consumer2); + verifyNotTouched(consumer3); + + consumerManager.update(Collections.emptySet()); + verifyUnsubscribedAndStopped(consumer3); + verifyUnsubscribedAndStopped(consumer4); + verifyUnsubscribedAndStopped(consumer5); + + consumerManager.update(createTpis(1, 2, 3)); + consumer1 = getConsumer(1); + consumer2 = getConsumer(2); + consumer3 = getConsumer(3); + verifySubscribedAndLaunched(consumer1, 1); + verifySubscribedAndLaunched(consumer2, 2); + verifySubscribedAndLaunched(consumer3, 3); + } + + @Test + public void testConfigUpdate_singleConsumer() { + queue.setConsumerPerPartition(false); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + Set partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + TestConsumer consumer = getConsumer(); + verifySubscribedAndLaunched(consumer, partitions); + + Queue newConfig = JacksonUtil.clone(queue); + newConfig.setPollInterval(queue.getPollInterval() / 2); + newConfig.setPartitions(queue.getPartitions() / 2); + newConfig.setPackProcessingTimeout(queue.getPackProcessingTimeout() * 2); + newConfig.getSubmitStrategy().setType(SubmitStrategyType.SEQUENTIAL_BY_ORIGINATOR); + newConfig.getProcessingStrategy().setType(ProcessingStrategyType.RETRY_ALL); + consumerManager.update(newConfig); + + await().atMost(2, TimeUnit.SECONDS) + .untilAsserted(() -> { + verify(consumer, atLeastOnce()).poll(eq((long) newConfig.getPollInterval())); + verify(ruleEngineConsumerContext.getSubmitStrategyFactory(), atLeastOnce()).newInstance(any(), eq(newConfig.getSubmitStrategy())); + verify(ruleEngineConsumerContext.getProcessingStrategyFactory(), atLeastOnce()).newInstance(any(), eq(newConfig.getProcessingStrategy())); + }); + } + + @Test + public void testConfigUpdate_consumerPerPartition() { + queue.setConsumerPerPartition(true); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + Set partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + TestConsumer consumer1 = getConsumer(1); + TestConsumer consumer2 = getConsumer(2); + TestConsumer consumer3 = getConsumer(3); + verifySubscribedAndLaunched(consumer1, 1); + verifySubscribedAndLaunched(consumer2, 2); + verifySubscribedAndLaunched(consumer3, 3); + + Queue newConfig = JacksonUtil.clone(queue); + newConfig.setPollInterval(queue.getPollInterval() / 2); + newConfig.setPartitions(queue.getPartitions() / 2); + newConfig.setPackProcessingTimeout(queue.getPackProcessingTimeout() * 2); + newConfig.getSubmitStrategy().setType(SubmitStrategyType.SEQUENTIAL_BY_ORIGINATOR); + newConfig.getProcessingStrategy().setType(ProcessingStrategyType.RETRY_ALL); + consumerManager.update(newConfig); + + await().atMost(2, TimeUnit.SECONDS) + .untilAsserted(() -> { + verify(consumer1, atLeastOnce()).poll(eq((long) newConfig.getPollInterval())); + verify(consumer2, atLeastOnce()).poll(eq((long) newConfig.getPollInterval())); + verify(consumer3, atLeastOnce()).poll(eq((long) newConfig.getPollInterval())); + }); + verifyNotTouched(consumer1); + verifyNotTouched(consumer2); + verifyNotTouched(consumer3); + } + + @Test + public void testConfigUpdate_fromSingleToConsumerPerPartition() { + queue.setConsumerPerPartition(false); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + Set partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + TestConsumer consumer = getConsumer(); + verifySubscribedAndLaunched(consumer, partitions); + + Queue newConfig = JacksonUtil.clone(queue); + newConfig.setConsumerPerPartition(true); + consumerManager.update(newConfig); + + verifyUnsubscribedAndStopped(consumer); + verifySubscribedAndLaunched(getConsumer(1), 1); + verifySubscribedAndLaunched(getConsumer(2), 2); + verifySubscribedAndLaunched(getConsumer(3), 3); + } + + @Test + public void testConfigUpdate_fromConsumerPerPartitionToSingle() { + queue.setConsumerPerPartition(true); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + Set partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + TestConsumer consumer1 = getConsumer(1); + TestConsumer consumer2 = getConsumer(2); + TestConsumer consumer3 = getConsumer(3); + verifySubscribedAndLaunched(consumer1, 1); + verifySubscribedAndLaunched(consumer2, 2); + verifySubscribedAndLaunched(consumer3, 3); + + Queue newConfig = JacksonUtil.clone(queue); + newConfig.setConsumerPerPartition(false); + consumerManager.update(newConfig); + + verifyUnsubscribedAndStopped(consumer1); + verifyUnsubscribedAndStopped(consumer2); + verifyUnsubscribedAndStopped(consumer3); + verifySubscribedAndLaunched(getConsumer(), partitions); + } + + @Test + public void testStop() { + queue.setConsumerPerPartition(true); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + consumerManager.update(createTpis(1)); + TestConsumer consumer = getConsumer(1); + verifySubscribedAndLaunched(consumer, 1); + verify(queueFactory, times(1)).createToRuleEngineMsgConsumer(any()); + + consumerManager.stop(); + consumerManager.update(createTpis(1, 2, 3, 4)); // to check that no new tasks after stop are processed + consumerManager.update(createTpis(5, 6, 7)); + + verifyUnsubscribedAndStopped(consumer); + verifyNoMoreInteractions(queueFactory); + } + + @Test + public void testDelete_consumerPerPartition() { + queue.setConsumerPerPartition(true); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + Set partitions = createTpis(1, 2); + consumerManager.update(partitions); + TestConsumer consumer1 = getConsumer(1); + TestConsumer consumer2 = getConsumer(2); + verifySubscribedAndLaunched(consumer1, 1); + verifySubscribedAndLaunched(consumer2, 2); + verifyMsgProcessed(consumer1.testMsg); + verifyMsgProcessed(consumer2.testMsg); + + consumerManager.delete(); + + await().atMost(2, TimeUnit.SECONDS) + .untilAsserted(() -> { + verify(ruleEngineMsgProducer).send(any(), any(), any()); + }); + clearInvocations(actorContext); + verify(consumer1, never()).unsubscribe(); + verify(consumer2, never()).unsubscribe(); + int msgCount1 = consumer1.msgCount; + int msgCount2 = consumer2.msgCount; + + await().atLeast(4, TimeUnit.SECONDS) // based on topicDeletionDelayInSec + .atMost(7, TimeUnit.SECONDS) + .untilAsserted(() -> { + partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .forEach(topic -> { + verify(queueAdmin).deleteTopic(eq(topic)); + }); + }); + verify(consumer1).unsubscribe(); + verify(consumer2).unsubscribe(); + + int movedMsgs1 = consumer1.msgCount - msgCount1; + int movedMsgs2 = consumer2.msgCount - msgCount2; + int totalMovedMsgs = movedMsgs1 + movedMsgs2; + assertThat(totalMovedMsgs).isGreaterThan(10); + verify(ruleEngineMsgProducer, atLeast(totalMovedMsgs)).send(any(), any(), any()); + verify(actorContext, never()).tell(any()); + generateQueueMsgs = false; + } + + @Test + public void testDelete_singleConsumer() { + queue.setConsumerPerPartition(false); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + Set partitions = createTpis(1, 2); + consumerManager.update(partitions); + TestConsumer consumer = getConsumer(); + verifySubscribedAndLaunched(consumer, partitions); + verifyMsgProcessed(consumer.testMsg); + + consumerManager.delete(); + + await().atMost(2, TimeUnit.SECONDS) + .untilAsserted(() -> { + verify(ruleEngineMsgProducer).send(any(), any(), any()); + }); + clearInvocations(actorContext); + verify(consumer, never()).unsubscribe(); + int msgCount = consumer.msgCount; + + await().atLeast(4, TimeUnit.SECONDS) + .atMost(7, TimeUnit.SECONDS) + .untilAsserted(() -> { + partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .forEach(topic -> { + verify(queueAdmin).deleteTopic(eq(topic)); + }); + }); + verify(consumer).unsubscribe(); + + int movedMsgs = consumer.msgCount - msgCount; + assertThat(movedMsgs).isGreaterThan(10); + verify(ruleEngineMsgProducer, atLeast(movedMsgs)).send(any(), any(), any()); + verify(actorContext, never()).tell(any()); + generateQueueMsgs = false; + } + + @Test + public void testManyDifferentUpdates() throws Exception { + queue.setConsumerPerPartition(RandomUtils.nextBoolean()); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + + Supplier queueConfigUpdater = () -> { + Queue oldConfig = consumerManager.getQueue(); + Queue newConfig = JacksonUtil.clone(oldConfig); + newConfig.setConsumerPerPartition(RandomUtils.nextBoolean()); + newConfig.setPollInterval(RandomUtils.nextInt(100, 501)); + newConfig.setPartitions(RandomUtils.nextInt(1, 10)); + newConfig.setPackProcessingTimeout(RandomUtils.nextLong(100, 5001)); + newConfig.getSubmitStrategy().setType(SubmitStrategyType.values()[RandomUtils.nextInt(0, SubmitStrategyType.values().length)]); + newConfig.getProcessingStrategy().setType(ProcessingStrategyType.values()[RandomUtils.nextInt(0, ProcessingStrategyType.values().length)]); + log.info("Generated new config: consumerPerPartition={}, pollInterval={}, processingStrategy={}", + newConfig.isConsumerPerPartition(), newConfig.getPollInterval(), newConfig.getProcessingStrategy().getType()); + return newConfig; + }; + Supplier> partitionsUpdater = () -> { + int partitionsCount = RandomUtils.nextInt(0, 20); + int[] partitions = IntStream.generate(() -> RandomUtils.nextInt(0, 20)) + .distinct().limit(partitionsCount) + .sorted().toArray(); + log.info("Generated new partitions: {}", Arrays.toString(partitions)); + return createTpis(partitions); + }; + + int iterations = 100; + Queue latestConfig = queue; + Set latestPartitions = Collections.emptySet(); + for (int i = 1; i <= iterations; i++) { + boolean updateQueueConfig = RandomUtils.nextBoolean(); + boolean updatePartitions = !updateQueueConfig; + if (updateQueueConfig) { + latestConfig = queueConfigUpdater.get(); + consumerManager.update(latestConfig); + } + if (updatePartitions) { + latestPartitions = partitionsUpdater.get(); + consumerManager.update(latestPartitions); + } + Thread.sleep(RandomUtils.nextLong(0, 200)); + } + if (latestPartitions.isEmpty()) { + do { + latestPartitions = partitionsUpdater.get(); + } while (latestPartitions.isEmpty()); + consumerManager.update(latestPartitions); + } + + Queue expectedConfig = latestConfig; + Set expectedPartitions = latestPartitions; + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + assertThat(consumerManager.getQueue()).isEqualTo(expectedConfig); + assertThat(consumerManager.getPartitions()).isEqualTo(expectedPartitions); + }); + + if (expectedConfig.isConsumerPerPartition()) { + await().atMost(5, TimeUnit.SECONDS).until(() -> { + for (TopicPartitionInfo partition : expectedPartitions) { + if (consumers.stream().noneMatch(consumer -> consumer.subscribed && + consumer.pollingStarted && Set.of(partition).equals(consumer.getPartitions()))) { + return false; + } + } + return consumers.size() == expectedPartitions.size(); + }); + } else { + await().atMost(5, TimeUnit.SECONDS).until(() -> { + return consumers.size() == 1 && consumers.stream() + .anyMatch(consumer -> consumer.subscribed && consumer.pollingStarted && + expectedPartitions.equals(consumer.getPartitions())); + }); + } + Mockito.reset(ruleEngineConsumerContext.getSubmitStrategyFactory()); + Mockito.reset(ruleEngineConsumerContext.getProcessingStrategyFactory()); + consumers.forEach(Mockito::clearInvocations); + + await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> { + for (TestConsumer consumer : consumers) { + verify(consumer, atLeastOnce().description("consumer " + consumer.topics)).poll(expectedConfig.getPollInterval()); + } + verify(ruleEngineConsumerContext.getSubmitStrategyFactory(), atLeastOnce()).newInstance(any(), eq(expectedConfig.getSubmitStrategy())); + verify(ruleEngineConsumerContext.getProcessingStrategyFactory(), atLeastOnce()).newInstance(any(), eq(expectedConfig.getProcessingStrategy())); + }); + } + + /* + * 2023-10-15 18:34:06,090 [main] INFO o.t.s.s.q.r.TbRuleEngineQueueConsumerManagerTest - Generated new partitions: [0, 1, 2, 3, 4, 5, 6, 8, 9, 11, 12, 13, 15, 16, 17, 18, 19] +2023-10-15 18:34:06,090 [main] INFO o.t.s.s.q.r.TbRuleEngineQueueConsumerManagerTest - Generated new config: consumerPerPartition=false, pollInterval=299, processingStrategy=RETRY_FAILED + * */ + + private void verifySubscribedAndLaunched(TestConsumer consumer, Set expectedPartitions) { + await().atMost(2, TimeUnit.SECONDS) + .until(() -> consumer.subscribed && consumer.getPartitions().equals(expectedPartitions) && consumer.pollingStarted); + verify(consumer, times(1)).subscribe(any()); + verify(consumer).subscribe(eq(expectedPartitions)); + verify(consumer).doSubscribe(argThat(topics -> topics.containsAll(expectedPartitions.stream() + .map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList())))); + verify(consumer, atLeastOnce()).poll(eq((long) queue.getPollInterval())); + verify(consumer, atLeastOnce()).doPoll(eq((long) queue.getPollInterval())); + verify(consumer, never()).unsubscribe(); + Mockito.reset(consumer); + } + + private void verifySubscribedAndLaunched(TestConsumer consumer, int... expectedPartitions) { + verifySubscribedAndLaunched(consumer, createTpis(expectedPartitions)); + } + + private void verifyUnsubscribedAndStopped(TestConsumer consumer) { + await().atMost(2, TimeUnit.SECONDS) + .until(() -> !consumer.subscribed && !consumer.topics.isEmpty()); + verify(consumer, never()).subscribe(any()); + verify(consumer, never()).doSubscribe(any()); + assertThat(consumers).doesNotContain(consumer); + Mockito.reset(consumer); + } + + private void verifyNotTouched(TestConsumer consumer) { + verify(consumer, never()).subscribe(any()); + verify(consumer, never()).subscribe(); + verify(consumer, never()).doSubscribe(any()); + verify(consumer, never()).unsubscribe(); + verify(consumer, never()).doUnsubscribe(); + } + + private void verifyMsgProcessed(TbMsg tbMsg) { + await().atMost(2, TimeUnit.SECONDS).untilAsserted(() -> { + verify(actorContext, atLeastOnce()).tell(argThat(msg -> { + return ((QueueToRuleEngineMsg) msg).getMsg().getId().equals(tbMsg.getId()); + })); + }); + } + + // for consumer-per-partition + private TestConsumer getConsumer(TopicPartitionInfo tpi) { + return await().atMost(5, TimeUnit.SECONDS) + .until(() -> consumers.stream() + .filter(consumer -> consumer.getPartitions() != null && + consumer.getPartitions().size() == 1 && + consumer.getPartitions().contains(tpi)) + .findFirst().orElse(null), Objects::nonNull); + } + + private TestConsumer getConsumer(int partition) { + return await().atMost(5, TimeUnit.SECONDS) + .until(() -> consumers.stream() + .filter(consumer -> consumer.getPartitions() != null && + consumer.getPartitions().size() == 1 && + consumer.getPartitions().stream() + .anyMatch(tpi -> tpi.getPartition().get().equals(partition))) + .findFirst().orElse(null), Objects::nonNull); + } + + // for single consumer + private TestConsumer getConsumer() { + return await().atMost(5, TimeUnit.SECONDS) + .until(() -> consumers.size() == 1 ? consumers.iterator().next() : null, Objects::nonNull); + } + + private Set createTpis(int... partitions) { + return Arrays.stream(partitions) + .mapToObj(n -> TopicPartitionInfo.builder() + .tenantId(queue.getTenantId()) + .topic(queue.getTopic()) + .partition(n) + .myPartition(true) + .build()) + .collect(Collectors.toSet()); + } + + + class TestConsumer extends AbstractTbQueueConsumerTemplate> { + + @Getter + private List topics; + + private boolean subscribed; + private boolean pollingStarted; + + private TbMsg testMsg; + private int msgCount; + + public TestConsumer(String topic) { + super(topic); + } + + @SneakyThrows + @Override + protected List doPoll(long durationInMillis) { + log.debug("doPoll({} ms)", durationInMillis); + if (!subscribed) { + throw new IllegalStateException("Cannot poll because not subscribed"); + } + pollingStarted = true; + if (testMsg != null && RandomUtils.nextBoolean()) { + Thread.sleep(100); + return List.of(testMsg); + } + return Collections.emptyList(); + } + + @Override + protected TbProtoQueueMsg decode(TbMsg tbMsg) throws IOException { + log.debug("decode()"); + UUID tenantId = UUID.randomUUID(); + return new TbProtoQueueMsg<>(UUID.randomUUID(), ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(tenantId.getMostSignificantBits()) + .setTenantIdLSB(tenantId.getLeastSignificantBits()) + .addRelationTypes("Success") + .setTbMsg(TbMsg.toByteString(tbMsg)) + .build()); + } + + @Override + protected void doSubscribe(List topicNames) { + log.debug("doSubscribe({})", topicNames); + this.topics = topicNames; + subscribed = true; + } + + @Override + protected void doCommit() { + if (!subscribed) { + throw new IllegalStateException("Cannot commit because not subscribed"); + } + msgCount++; + log.debug("doCommit() totalConsumedMsgs = {}", totalConsumedMsgs.incrementAndGet()); + } + + @Override + public void unsubscribe() { + super.unsubscribe(); + consumers.remove(this); + } + + @Override + protected void doUnsubscribe() { + log.debug("doUnsubscribe()"); + if (!subscribed) { + throw new IllegalStateException("Already unsubscribed!"); + } + subscribed = false; + } + + @Override + protected boolean isLongPollingSupported() { + return false; + } + + public Set getPartitions() { + return partitions; + } + + public void setUpTestMsg() { + testMsg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, new DeviceId(UUID.randomUUID()), new TbMsgMetaData(), "{}"); + } + } + +} From ea4eadb47f9184326f04f05e5461744376368d06 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 18 Oct 2023 14:20:41 +0300 Subject: [PATCH 18/29] Cancelling consumer loop task on stop --- .../DefaultTbRuleEngineConsumerService.java | 1 + .../service/queue/ruleengine/QueueEvent.java | 2 +- .../queue/ruleengine/TbQueueConsumerTask.java | 34 +++++++-- .../TbRuleEngineConsumerContext.java | 10 +-- .../TbRuleEngineQueueConsumerManager.java | 70 +++++++++++-------- .../TbRuleEngineQueueConsumerManagerTest.java | 10 +-- 6 files changed, 78 insertions(+), 49 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 64139530b1..3f05a5b923 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -126,6 +126,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @Override protected void stopConsumers() { consumers.values().forEach(TbRuleEngineQueueConsumerManager::stop); + consumers.values().forEach(TbRuleEngineQueueConsumerManager::awaitStop); ctx.stop(); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java index a4a3bcc59a..86955be7e7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java @@ -19,6 +19,6 @@ import java.io.Serializable; public enum QueueEvent implements Serializable { - PARTITION_CHANGE, CONFIG_UPDATE, STOP, DELETE + PARTITION_CHANGE, CONFIG_UPDATE, DELETE } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java index a0a5579951..2e3a4676e8 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java @@ -15,7 +15,8 @@ */ package org.thingsboard.server.service.queue.ruleengine; -import lombok.Data; +import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; @@ -23,16 +24,26 @@ import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import java.util.Set; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -@Data +@RequiredArgsConstructor @Slf4j public class TbQueueConsumerTask { + @Getter private final Object key; + @Getter private final TbQueueConsumer> consumer; - private volatile Future task; + + private Future task; + private CountDownLatch completionLatch; + + public void setTask(Future task) { + this.task = task; + this.completionLatch = new CountDownLatch(1); + } public void subscribe(Set partitions) { log.trace("[{}] Subscribing to partitions: {}", key, partitions); @@ -42,23 +53,32 @@ public class TbQueueConsumerTask { public void initiateStop() { log.debug("[{}] Initiating stop", key); consumer.stop(); + if (isRunning()) { + task.cancel(true); + } } - public void awaitFinish() { + public void awaitCompletion() { log.trace("[{}] Awaiting finish", key); if (isRunning()) { try { - task.get(60, TimeUnit.SECONDS); - task = null; + if (!completionLatch.await(30, TimeUnit.SECONDS)) { + throw new IllegalStateException("timeout of 30 seconds expired"); + } } catch (Exception e) { log.warn("[{}] Failed to await for consumer to stop", key, e); } + task = null; } log.trace("[{}] Awaited finish", key); } public boolean isRunning() { - return task != null && !task.isDone(); + return task != null; + } + + public void finished() { + completionLatch.countDown(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java index 325a685972..f4c8e4c2ae 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java @@ -36,7 +36,6 @@ import javax.annotation.PostConstruct; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; @Component @TbRuleEngineComponent @@ -83,12 +82,7 @@ public class TbRuleEngineConsumerContext { public void stop() { scheduler.shutdownNow(); - consumersExecutor.shutdown(); - mgmtExecutor.shutdown(); - try { - mgmtExecutor.awaitTermination(15, TimeUnit.SECONDS); - } catch (InterruptedException e) { - log.warn("Failed to await mgmtExecutor termination"); - } + consumersExecutor.shutdownNow(); + mgmtExecutor.shutdownNow(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index dc3966d0d2..3a7b2f15d8 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -85,7 +85,13 @@ public class TbRuleEngineQueueConsumerManager { } public void init(Queue queue) { - doInit(queue); + this.queue = queue; + if (queue.isConsumerPerPartition()) { + this.consumerWrapper = new ConsumerPerPartitionWrapper(); + } else { + this.consumerWrapper = new SingleConsumerWrapper(); + } + log.debug("[{}] Initialized consumer for queue: {}", queueKey, queue); } public void update(Queue queue) { @@ -96,10 +102,6 @@ public class TbRuleEngineQueueConsumerManager { addTask(new TbQueueConsumerManagerTask(QueueEvent.PARTITION_CHANGE, partitions)); } - public void stop() { - addTask(new TbQueueConsumerManagerTask(QueueEvent.STOP)); - } - public void delete() { addTask(new TbQueueConsumerManagerTask(QueueEvent.DELETE)); } @@ -135,20 +137,22 @@ public class TbRuleEngineQueueConsumerManager { newPartitions = task.getPartitions(); } else if (task.getEvent() == QueueEvent.CONFIG_UPDATE) { newConfiguration = task.getQueue(); - } else if (task.getEvent() == QueueEvent.STOP) { - doStop(); - return; } else if (task.getEvent() == QueueEvent.DELETE) { doDelete(); return; } } + if (stopped) { + return; + } if (newConfiguration != null) { doUpdate(newConfiguration); } if (newPartitions != null) { doUpdate(newPartitions); } + } catch (Exception e) { + log.error("[{}] Failed to process tasks", queueKey, e); } finally { lock.unlock(); } @@ -159,16 +163,6 @@ public class TbRuleEngineQueueConsumerManager { }); } - private void doInit(Queue queue) { - this.queue = queue; - if (queue.isConsumerPerPartition()) { - consumerWrapper = new ConsumerPerPartitionWrapper(); - } else { - consumerWrapper = new SingleConsumerWrapper(); - } - log.debug("[{}] Initialized consumer for queue: {}", queueKey, queue); - } - private void doUpdate(Queue newQueue) { log.info("[{}] Processing queue update: {}", queueKey, newQueue); var oldQueue = this.queue; @@ -179,12 +173,12 @@ public class TbRuleEngineQueueConsumerManager { } if (oldQueue == null) { - doInit(queue); + init(queue); } else if (newQueue.isConsumerPerPartition() != oldQueue.isConsumerPerPartition()) { consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::initiateStop); - consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitFinish); + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitCompletion); - doInit(queue); + init(queue); if (partitions != null) { doUpdate(partitions); // even if partitions number was changed, there can be no partition change event } @@ -200,17 +194,21 @@ public class TbRuleEngineQueueConsumerManager { consumerWrapper.updatePartitions(partitions); } - private void doStop() { + public void stop() { + log.debug("[{}] Stopping consumers", queueKey); consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::initiateStop); - consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitFinish); - log.debug("[{}] Unsubscribed and stopped consumers", queueKey); stopped = true; } + public void awaitStop() { + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitCompletion); + log.debug("[{}] Unsubscribed and stopped consumers", queueKey); + } + private void doDelete() { stopped = true; log.info("[{}] Handling queue deletion", queueKey); - consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitFinish); + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitCompletion); List>> queueConsumers = consumerWrapper.getConsumers().stream() .map(TbQueueConsumerTask::getConsumer).collect(Collectors.toList()); @@ -240,6 +238,7 @@ public class TbRuleEngineQueueConsumerManager { Future consumerLoop = ctx.getConsumersExecutor().submit(() -> { ThingsBoardThreadFactory.updateCurrentThreadName(consumerTask.getKey().toString()); consumerLoop(consumerTask.getConsumer()); + consumerTask.finished(); }); consumerTask.setTask(consumerLoop); } @@ -253,7 +252,7 @@ public class TbRuleEngineQueueConsumerManager { } processMsgs(msgs, consumer, queue); } catch (Exception e) { - if (!stopped) { + if (!consumer.isStopped()) { log.warn("Failed to process messages from queue", e); try { Thread.sleep(ctx.getPollDuration()); @@ -279,7 +278,7 @@ public class TbRuleEngineQueueConsumerManager { TbMsgPackProcessingContext packCtx = new TbMsgPackProcessingContext(queue.getName(), submitStrategy, ackStrategy.isSkipTimeoutMsgs()); submitStrategy.submitAttempt((id, msg) -> submitMessage(packCtx, id, msg)); - final boolean timeout = !packCtx.await(queue.getPackProcessingTimeout(), TimeUnit.MILLISECONDS); + final boolean timeout = !awaitPackProcessing(packCtx, queue.getPackProcessingTimeout(), true); TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(queue.getName(), timeout, packCtx); if (timeout) { @@ -307,6 +306,19 @@ public class TbRuleEngineQueueConsumerManager { } } + private boolean awaitPackProcessing(TbMsgPackProcessingContext packCtx, long processingTimeout, boolean ignoreInterrupt) throws InterruptedException { + try { + return packCtx.await(processingTimeout, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + if (ignoreInterrupt) { + log.debug("Interrupt happened while waiting for pack processing, trying to await one more time"); + return awaitPackProcessing(packCtx, processingTimeout, false); + } else { + throw new RuntimeException("Failed to await pack processing due to thread interrupt", e); + } + } + } + private TbRuleEngineSubmitStrategy getSubmitStrategy(Queue queue) { return ctx.getSubmitStrategyFactory().newInstance(queue.getName(), queue.getSubmitStrategy()); } @@ -430,7 +442,7 @@ public class TbRuleEngineQueueConsumerManager { consumers.get(tpi).initiateStop(); }); removedPartitions.forEach((tpi) -> { - consumers.remove(tpi).awaitFinish(); + consumers.remove(tpi).awaitCompletion(); }); addedPartitions.forEach((tpi) -> { @@ -457,7 +469,7 @@ public class TbRuleEngineQueueConsumerManager { if (partitions.isEmpty()) { if (consumer != null && consumer.isRunning()) { consumer.initiateStop(); - consumer.awaitFinish(); + consumer.awaitCompletion(); } consumer = null; return; diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java index 276e65ec39..20b765da6d 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java @@ -180,13 +180,16 @@ public class TbRuleEngineQueueConsumerManagerTest { @After public void afterEach() { consumerManager.stop(); + consumerManager.awaitStop(); ruleEngineConsumerContext.stop(); if (generateQueueMsgs) { - await().atMost(1, TimeUnit.SECONDS) - .until(() -> totalProcessedMsgs.get() == totalConsumedMsgs.get()); + await().atMost(2, TimeUnit.SECONDS) + .untilAsserted(() -> { + log.debug("totalConsumedMsgs = {}, totalProcessedMsgs = {}", totalConsumedMsgs.get(), totalProcessedMsgs.get()); + assertThat(totalProcessedMsgs.get()).isEqualTo(totalConsumedMsgs.get()); + }); } - log.debug("totalConsumedMsgs = {}, totalProcessedMsgs = {}", totalConsumedMsgs.get(), totalProcessedMsgs.get()); } @Test @@ -510,7 +513,6 @@ public class TbRuleEngineQueueConsumerManagerTest { verify(consumer).unsubscribe(); int movedMsgs = consumer.msgCount - msgCount; - assertThat(movedMsgs).isGreaterThan(10); verify(ruleEngineMsgProducer, atLeast(movedMsgs)).send(any(), any(), any()); verify(actorContext, never()).tell(any()); generateQueueMsgs = false; From 5108e98f38631ff4f828cb014965b4836ef475a2 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 19 Oct 2023 11:03:53 +0300 Subject: [PATCH 19/29] Minor refactoring --- .../service/queue/ruleengine/TbRuleEngineConsumerContext.java | 2 +- application/src/main/resources/thingsboard.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java index f4c8e4c2ae..b18df6b62b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java @@ -74,7 +74,7 @@ public class TbRuleEngineConsumerContext { private volatile boolean isReady = false; @PostConstruct - public void init() { + void init() { this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer")); this.mgmtExecutor = Executors.newFixedThreadPool(mgmtThreadPoolSize, ThingsBoardThreadFactory.forName("tb-rule-engine-mgmt")); this.scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-scheduler")); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 6f51413ccd..be01c95fad 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1304,7 +1304,7 @@ queue: max-pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}" # Max allowed time in seconds for pause between retries. # After a queue is deleted (or profile's isolation option was disabled), Rule Engine will continue reading related topics during this period, before deleting the actual topics topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:15}" - # Size of the thread pool that handles management operations such as subscribe, unsubscribe, queue delete, etc. + # Size of the thread pool that handles such operations as partition changes, config updates, queue deletion management-thread-pool-size: "${TB_QUEUE_RULE_ENGINE_MGMT_THREAD_POOL_SIZE:12}" transport: # For high priority notifications that require minimum latency and processing time From 049f40a62e4cba6ffcd863eaada4a0ad6a7f360d Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 19 Oct 2023 14:00:04 +0300 Subject: [PATCH 20/29] Clear interruption status after exiting consumer while loop --- .../service/queue/ruleengine/TbQueueConsumerTask.java | 7 ++++--- .../ruleengine/TbRuleEngineQueueConsumerManager.java | 8 ++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java index 2e3a4676e8..9c0df63ad5 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java @@ -41,8 +41,8 @@ public class TbQueueConsumerTask { private CountDownLatch completionLatch; public void setTask(Future task) { - this.task = task; this.completionLatch = new CountDownLatch(1); + this.task = task; } public void subscribe(Set partitions) { @@ -63,14 +63,14 @@ public class TbQueueConsumerTask { if (isRunning()) { try { if (!completionLatch.await(30, TimeUnit.SECONDS)) { + task = null; throw new IllegalStateException("timeout of 30 seconds expired"); } + log.trace("[{}] Awaited finish", key); } catch (Exception e) { log.warn("[{}] Failed to await for consumer to stop", key, e); } - task = null; } - log.trace("[{}] Awaited finish", key); } public boolean isRunning() { @@ -79,6 +79,7 @@ public class TbQueueConsumerTask { public void finished() { completionLatch.countDown(); + task = null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index 3a7b2f15d8..71a7c45d45 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -237,7 +237,11 @@ public class TbRuleEngineQueueConsumerManager { log.info("[{}] Launching consumer", consumerTask.getKey()); Future consumerLoop = ctx.getConsumersExecutor().submit(() -> { ThingsBoardThreadFactory.updateCurrentThreadName(consumerTask.getKey().toString()); - consumerLoop(consumerTask.getConsumer()); + try { + consumerLoop(consumerTask.getConsumer()); + } catch (Throwable e) { + log.error("Failure in consumer loop", e); + } consumerTask.finished(); }); consumerTask.setTask(consumerLoop); @@ -262,7 +266,7 @@ public class TbRuleEngineQueueConsumerManager { } } } - if (consumer.isStopped()) { + if (Thread.interrupted() || consumer.isStopped()) { consumer.unsubscribe(); } log.info("Rule Engine consumer stopped"); From 2d1640a0025798cafb3a967562c3773978b936d5 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 19 Oct 2023 15:29:46 +0300 Subject: [PATCH 21/29] Don't interrupt consumer loop task on stop --- .../queue/ruleengine/TbQueueConsumerTask.java | 23 ++++--------------- .../TbRuleEngineQueueConsumerManager.java | 18 ++------------- 2 files changed, 6 insertions(+), 35 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java index 9c0df63ad5..59d55285f0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.queue.ruleengine; import lombok.Getter; import lombok.RequiredArgsConstructor; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; @@ -24,7 +25,6 @@ import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import java.util.Set; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -37,13 +37,8 @@ public class TbQueueConsumerTask { @Getter private final TbQueueConsumer> consumer; + @Setter private Future task; - private CountDownLatch completionLatch; - - public void setTask(Future task) { - this.completionLatch = new CountDownLatch(1); - this.task = task; - } public void subscribe(Set partitions) { log.trace("[{}] Subscribing to partitions: {}", key, partitions); @@ -53,23 +48,18 @@ public class TbQueueConsumerTask { public void initiateStop() { log.debug("[{}] Initiating stop", key); consumer.stop(); - if (isRunning()) { - task.cancel(true); - } } public void awaitCompletion() { log.trace("[{}] Awaiting finish", key); if (isRunning()) { try { - if (!completionLatch.await(30, TimeUnit.SECONDS)) { - task = null; - throw new IllegalStateException("timeout of 30 seconds expired"); - } + task.get(30, TimeUnit.SECONDS); log.trace("[{}] Awaited finish", key); } catch (Exception e) { log.warn("[{}] Failed to await for consumer to stop", key, e); } + task = null; } } @@ -77,9 +67,4 @@ public class TbQueueConsumerTask { return task != null; } - public void finished() { - completionLatch.countDown(); - task = null; - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index 71a7c45d45..5a59cb124e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -242,7 +242,6 @@ public class TbRuleEngineQueueConsumerManager { } catch (Throwable e) { log.error("Failure in consumer loop", e); } - consumerTask.finished(); }); consumerTask.setTask(consumerLoop); } @@ -266,7 +265,7 @@ public class TbRuleEngineQueueConsumerManager { } } } - if (Thread.interrupted() || consumer.isStopped()) { + if (consumer.isStopped()) { consumer.unsubscribe(); } log.info("Rule Engine consumer stopped"); @@ -282,7 +281,7 @@ public class TbRuleEngineQueueConsumerManager { TbMsgPackProcessingContext packCtx = new TbMsgPackProcessingContext(queue.getName(), submitStrategy, ackStrategy.isSkipTimeoutMsgs()); submitStrategy.submitAttempt((id, msg) -> submitMessage(packCtx, id, msg)); - final boolean timeout = !awaitPackProcessing(packCtx, queue.getPackProcessingTimeout(), true); + final boolean timeout = !packCtx.await(queue.getPackProcessingTimeout(), TimeUnit.MILLISECONDS); TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(queue.getName(), timeout, packCtx); if (timeout) { @@ -310,19 +309,6 @@ public class TbRuleEngineQueueConsumerManager { } } - private boolean awaitPackProcessing(TbMsgPackProcessingContext packCtx, long processingTimeout, boolean ignoreInterrupt) throws InterruptedException { - try { - return packCtx.await(processingTimeout, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - if (ignoreInterrupt) { - log.debug("Interrupt happened while waiting for pack processing, trying to await one more time"); - return awaitPackProcessing(packCtx, processingTimeout, false); - } else { - throw new RuntimeException("Failed to await pack processing due to thread interrupt", e); - } - } - } - private TbRuleEngineSubmitStrategy getSubmitStrategy(Queue queue) { return ctx.getSubmitStrategyFactory().newInstance(queue.getName(), queue.getSubmitStrategy()); } From be7854fb5baee31af0a9455c749755c80134b17e Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 19 Oct 2023 18:58:41 +0300 Subject: [PATCH 22/29] Add 'isolated' to topic and consumer group name for isolated queues --- .../thingsboard/server/common/msg/queue/TopicPartitionInfo.java | 2 +- .../server/queue/provider/KafkaMonolithQueueFactory.java | 2 +- .../server/queue/provider/KafkaTbRuleEngineQueueFactory.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java index 7122d5ef2a..d2144e8111 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -42,7 +42,7 @@ public class TopicPartitionInfo { this.myPartition = myPartition; String tmp = topic; if (tenantId != null && !tenantId.isNullUid()) { - tmp += "." + tenantId.getId().toString(); + tmp += ".isolated." + tenantId.getId().toString(); } if (partition != null) { tmp += "." + partition; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 22a16de64f..55050ce08c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -187,7 +187,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(configuration.getTopic()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId("re-" + queueName + (configuration.getTenantId().isSysTenantId() ? "" : ("-" + configuration.getTenantId())) + "-consumer"); + consumerBuilder.groupId("re-" + queueName + (configuration.getTenantId().isSysTenantId() ? "" : ("-isolated-" + configuration.getTenantId())) + "-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(ruleEngineAdmin); consumerBuilder.statsService(consumerStatsService); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 2e3bf784d7..cb063b6a7f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -166,7 +166,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(configuration.getTopic()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId("re-" + queueName + (configuration.getTenantId().isSysTenantId() ? "" : ("-" + configuration.getTenantId())) + "-consumer"); + consumerBuilder.groupId("re-" + queueName + (configuration.getTenantId().isSysTenantId() ? "" : ("-isolated-" + configuration.getTenantId())) + "-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(ruleEngineAdmin); consumerBuilder.statsService(consumerStatsService); From 23cd7f08654fddf66b6d3fa7f882f0e0ce50e336 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 20 Oct 2023 15:03:24 +0300 Subject: [PATCH 23/29] Fix testNotificationRuleProcessing_exceededRateLimits --- .../notification/NotificationRuleApiTest.java | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java index 0a2854a28d..33aaf2af35 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java @@ -420,11 +420,11 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { notificationRulesCache.evict(TenantId.SYS_TENANT_ID); int n = 10; - updateDefaultTenantProfile(profileConfiguration -> { - profileConfiguration.getProfileConfiguration().get().setTenantEntityExportRateLimit(n + ":600"); - profileConfiguration.getProfileConfiguration().get().setCustomerServerRestLimitsConfiguration(n + ":600"); - profileConfiguration.getProfileConfiguration().get().setTenantNotificationRequestsPerRuleRateLimit(n + ":600"); - profileConfiguration.getProfileConfiguration().get().setTransportDeviceTelemetryMsgRateLimit(n + ":600"); + updateDefaultTenantProfileConfig(profileConfiguration -> { + profileConfiguration.setTenantEntityExportRateLimit(n + ":600"); + profileConfiguration.setCustomerServerRestLimitsConfiguration(n + ":600"); + profileConfiguration.setTenantNotificationRequestsPerRuleRateLimit(n + ":600"); + profileConfiguration.setTransportDeviceTelemetryMsgRateLimit(n + ":600"); }); loginTenantAdmin(); NotificationRule rule = createNotificationRule(AlarmCommentNotificationRuleTriggerConfig.builder() @@ -434,11 +434,14 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { rateLimitService.checkRateLimit(LimitedApi.ENTITY_EXPORT, tenantId); rateLimitService.checkRateLimit(LimitedApi.REST_REQUESTS_PER_CUSTOMER, tenantId, customerId); rateLimitService.checkRateLimit(LimitedApi.NOTIFICATION_REQUESTS_PER_RULE, tenantId, rule.getId()); + Thread.sleep(100); } loginTenantAdmin(); - List notifications = await().atMost(30, TimeUnit.SECONDS) - .until(() -> getMyNotifications(true, 10), list -> list.size() == 3); + List notifications = await().atMost(15, TimeUnit.SECONDS) + .until(() -> getMyNotifications(true, 10).stream() + .filter(notification -> notification.getType() == NotificationType.RATE_LIMITS) + .collect(Collectors.toList()), list -> list.size() == 3); assertThat(notifications).allSatisfy(notification -> { assertThat(notification.getSubject()).isEqualTo("Rate limits exceeded"); }); @@ -455,12 +458,14 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { }); loginSysAdmin(); - notifications = await().atMost(30, TimeUnit.SECONDS) - .until(() -> getMyNotifications(true, 10), list -> list.size() == 1); - assertThat(notifications).allSatisfy(notification -> { + notifications = await().atMost(15, TimeUnit.SECONDS) + .until(() -> getMyNotifications(true, 10).stream() + .filter(notification -> notification.getType() == NotificationType.RATE_LIMITS) + .collect(Collectors.toList()), list -> list.size() == 1); + assertThat(notifications).singleElement().satisfies(notification -> { assertThat(notification.getSubject()).isEqualTo("Rate limits exceeded for tenant " + TEST_TENANT_NAME); + assertThat(notification.getText()).isEqualTo("Rate limits for entity version creation exceeded"); }); - assertThat(notifications.get(0).getText()).isEqualTo("Rate limits for entity version creation exceeded"); } @Test From aea4d267ffebaa8dca0ffe4b11a1336a4cb35df3 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 20 Oct 2023 15:26:25 +0300 Subject: [PATCH 24/29] Minor improvements --- .../queue/ruleengine/TbRuleEngineConsumerContext.java | 3 ++- .../home/components/profile/tenant-profile.component.ts | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java index b18df6b62b..da2a5d0db4 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java @@ -19,6 +19,7 @@ import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.stats.StatsFactory; @@ -76,7 +77,7 @@ public class TbRuleEngineConsumerContext { @PostConstruct void init() { this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer")); - this.mgmtExecutor = Executors.newFixedThreadPool(mgmtThreadPoolSize, ThingsBoardThreadFactory.forName("tb-rule-engine-mgmt")); + this.mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(mgmtThreadPoolSize, "tb-rule-engine-mgmt"); this.scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-scheduler")); } diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts index 26d6611a3c..3f048ab2e3 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts @@ -56,7 +56,7 @@ export class TenantProfileComponent extends EntityComponent { const mainQueue = [ { id: guid(), - consumerPerPartition: true, + consumerPerPartition: false, name: 'Main', packProcessingTimeout: 10000, partitions: 1, @@ -84,7 +84,7 @@ export class TenantProfileComponent extends EntityComponent { topic: 'tb_rule_engine.hp', pollInterval: 2000, partitions: 1, - consumerPerPartition: true, + consumerPerPartition: false, packProcessingTimeout: 10000, submitStrategy: { type: 'BURST', @@ -108,7 +108,7 @@ export class TenantProfileComponent extends EntityComponent { topic: 'tb_rule_engine.sq', pollInterval: 2000, partitions: 1, - consumerPerPartition: true, + consumerPerPartition: false, packProcessingTimeout: 10000, submitStrategy: { type: 'SEQUENTIAL_BY_ORIGINATOR', From 6d7cb95492c1a90fc55dab13a5f3d95a563bf690 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 20 Oct 2023 15:46:53 +0300 Subject: [PATCH 25/29] Bigger timeout in tests for msg processing --- .../queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java index 20b765da6d..38582d0fce 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java @@ -184,7 +184,7 @@ public class TbRuleEngineQueueConsumerManagerTest { ruleEngineConsumerContext.stop(); if (generateQueueMsgs) { - await().atMost(2, TimeUnit.SECONDS) + await().atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> { log.debug("totalConsumedMsgs = {}, totalProcessedMsgs = {}", totalConsumedMsgs.get(), totalProcessedMsgs.get()); assertThat(totalProcessedMsgs.get()).isEqualTo(totalConsumedMsgs.get()); From 802890cdc0b99703f1431dad5c4e5ec643dc130c Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Mon, 23 Oct 2023 19:37:38 +0300 Subject: [PATCH 26/29] Fixed testIsolatedTenantProfile - correct expected number of downlink msgs --- .../server/edge/TenantProfileEdgeTest.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java index 7ec32f8aff..6f665ce462 100644 --- a/application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java @@ -32,6 +32,7 @@ import org.thingsboard.server.gen.edge.v1.TenantUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import java.util.List; +import java.util.Optional; import static org.assertj.core.api.Assertions.assertThat; @@ -75,17 +76,21 @@ public class TenantProfileEdgeTest extends AbstractEdgeTest { TenantProfileQueueConfiguration mainQueueConfiguration = createQueueConfig(DataConstants.MAIN_QUEUE_NAME, DataConstants.MAIN_QUEUE_TOPIC); TenantProfileQueueConfiguration isolatedQueueConfiguration = createQueueConfig("IsolatedHighPriority", "tb_rule_engine.isolated_hp"); edgeTenantProfile.getProfileData().setQueueConfiguration(List.of(mainQueueConfiguration, isolatedQueueConfiguration)); - edgeImitator.expectMessageAmount(1); + edgeImitator.expectMessageAmount(3); edgeTenantProfile = doPost("/api/tenantProfile", edgeTenantProfile, TenantProfile.class); Assert.assertTrue(edgeImitator.waitForMessages()); - AbstractMessage latestMessage = edgeImitator.getLatestMessage(); - Assert.assertTrue(latestMessage instanceof TenantProfileUpdateMsg); - TenantProfileUpdateMsg tenantProfileUpdateMsg = (TenantProfileUpdateMsg) latestMessage; + + Optional tenantProfileUpdateMsgOpt = edgeImitator.findMessageByType(TenantProfileUpdateMsg.class); + Assert.assertTrue(tenantProfileUpdateMsgOpt.isPresent()); + TenantProfileUpdateMsg tenantProfileUpdateMsg = tenantProfileUpdateMsgOpt.get(); Assert.assertEquals(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE, tenantProfileUpdateMsg.getMsgType()); Assert.assertEquals(edgeTenantProfile.getUuidId().getMostSignificantBits(), tenantProfileUpdateMsg.getIdMSB()); Assert.assertEquals(edgeTenantProfile.getUuidId().getLeastSignificantBits(), tenantProfileUpdateMsg.getIdLSB()); Assert.assertEquals(edgeTenantProfile.getDescription(), tenantProfileUpdateMsg.getDescription()); + List queueUpdateMsgs = edgeImitator.findAllMessagesByType(QueueUpdateMsg.class); + Assert.assertEquals(2, queueUpdateMsgs.size()); + loginTenantAdmin(); edgeImitator.expectMessageAmount(21); @@ -95,7 +100,7 @@ public class TenantProfileEdgeTest extends AbstractEdgeTest { Assert.assertTrue(edgeImitator.getDownlinkMsgs().get(0) instanceof TenantUpdateMsg); Assert.assertTrue(edgeImitator.getDownlinkMsgs().get(1) instanceof TenantProfileUpdateMsg); - List queueUpdateMsgs = edgeImitator.findAllMessagesByType(QueueUpdateMsg.class); + queueUpdateMsgs = edgeImitator.findAllMessagesByType(QueueUpdateMsg.class); Assert.assertEquals(2, queueUpdateMsgs.size()); for (QueueUpdateMsg queueUpdateMsg : queueUpdateMsgs) { Assert.assertEquals(tenantId.getId().getMostSignificantBits(), queueUpdateMsg.getTenantIdMSB()); From 0571f7e6ddbe62605bc703cfbecfa4c177e2411a Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 24 Oct 2023 12:24:01 +0300 Subject: [PATCH 27/29] Fix compatibility between CE and PE --- .../server/service/queue/ProtoUtils.java | 22 ++++++- .../server/service/queue/ProtoUtilsTest.java | 40 +++--------- common/cluster-api/src/main/proto/queue.proto | 60 +++++++++--------- .../server/common/data/EntityType.java | 63 ++++++++++--------- .../msg/plugin/ComponentLifecycleMsg.java | 2 + 5 files changed, 98 insertions(+), 89 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java b/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java index 3406265747..e39949e891 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java @@ -15,33 +15,51 @@ */ package org.thingsboard.server.service.queue; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.gen.transport.TransportProtos; +import java.util.Arrays; import java.util.UUID; public class ProtoUtils { + private static final EntityType[] entityTypeByProtoNumber; + + static { + int arraySize = Arrays.stream(EntityType.values()).mapToInt(EntityType::getProtoNumber).max().orElse(0); + entityTypeByProtoNumber = new EntityType[arraySize + 1]; + Arrays.stream(EntityType.values()).forEach(entityType -> entityTypeByProtoNumber[entityType.getProtoNumber()] = entityType); + } + public static TransportProtos.ComponentLifecycleMsgProto toProto(ComponentLifecycleMsg msg) { return TransportProtos.ComponentLifecycleMsgProto.newBuilder() .setTenantIdMSB(msg.getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(msg.getTenantId().getId().getLeastSignificantBits()) - .setEntityType(TransportProtos.EntityType.forNumber(msg.getEntityId().getEntityType().ordinal())) + .setEntityType(toProto(msg.getEntityId().getEntityType())) .setEntityIdMSB(msg.getEntityId().getId().getMostSignificantBits()) .setEntityIdLSB(msg.getEntityId().getId().getLeastSignificantBits()) .setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(msg.getEvent().ordinal())) .build(); } + public static TransportProtos.EntityTypeProto toProto(EntityType entityType) { + return TransportProtos.EntityTypeProto.forNumber(entityType.getProtoNumber()); + } + public static ComponentLifecycleMsg fromProto(TransportProtos.ComponentLifecycleMsgProto proto) { return new ComponentLifecycleMsg( TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), - EntityIdFactory.getByTypeAndUuid(proto.getEntityTypeValue(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())), + EntityIdFactory.getByTypeAndUuid(fromProto(proto.getEntityType()), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())), ComponentLifecycleEvent.values()[proto.getEventValue()] ); } + public static EntityType fromProto(TransportProtos.EntityTypeProto entityType) { + return entityTypeByProtoNumber[entityType.getNumber()]; + } + } diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ProtoUtilsTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ProtoUtilsTest.java index c63055a6f1..e235076635 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ProtoUtilsTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ProtoUtilsTest.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.queue; import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; @@ -31,41 +32,18 @@ class ProtoUtilsTest { TenantId tenantId = TenantId.fromUUID(UUID.fromString("35e10f77-16e7-424d-ae46-ee780f87ac4f")); EntityId entityId = new RuleChainId(UUID.fromString("c640b635-4f0f-41e6-b10b-25a86003094e")); @Test - void toProtoComponentLifecycleMsg() { + void protoComponentLifecycleSerialization() { ComponentLifecycleMsg msg = new ComponentLifecycleMsg(tenantId, entityId, ComponentLifecycleEvent.UPDATED); - - TransportProtos.ComponentLifecycleMsgProto proto = ProtoUtils.toProto(msg); - - assertThat(proto).as("to proto").isEqualTo(TransportProtos.ComponentLifecycleMsgProto.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setEntityType(TransportProtos.EntityType.forNumber(entityId.getEntityType().ordinal())) - .setEntityIdMSB(entityId.getId().getMostSignificantBits()) - .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) - .setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(ComponentLifecycleEvent.UPDATED.ordinal())) - .build() - ); - - assertThat(ProtoUtils.fromProto(proto)).as("from proto").isEqualTo(msg); + assertThat(ProtoUtils.fromProto(ProtoUtils.toProto(msg))).as("deserialized").isEqualTo(msg); + msg = new ComponentLifecycleMsg(tenantId, entityId, ComponentLifecycleEvent.STARTED); + assertThat(ProtoUtils.fromProto(ProtoUtils.toProto(msg))).as("deserialized").isEqualTo(msg); } @Test - void fromProtoComponentLifecycleMsg() { - TransportProtos.ComponentLifecycleMsgProto proto = TransportProtos.ComponentLifecycleMsgProto.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setEntityType(TransportProtos.EntityType.forNumber(entityId.getEntityType().ordinal())) - .setEntityIdMSB(entityId.getId().getMostSignificantBits()) - .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) - .setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(ComponentLifecycleEvent.STARTED.ordinal())) - .build(); - - ComponentLifecycleMsg msg = ProtoUtils.fromProto(proto); - - assertThat(msg).as("from proto").isEqualTo( - new ComponentLifecycleMsg(tenantId, entityId, ComponentLifecycleEvent.STARTED)); - - assertThat(ProtoUtils.toProto(msg)).as("to proto").isEqualTo(proto); + void protoEntityTypeSerialization() { + for(EntityType entityType : EntityType.values()){ + assertThat(ProtoUtils.fromProto(ProtoUtils.toProto(entityType))).as(entityType.getNormalName()).isEqualTo(entityType); + } } } \ No newline at end of file diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index d8439d0cd0..e560dfccf5 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -23,33 +23,37 @@ option java_outer_classname = "TransportProtos"; /** * Common data structures */ -enum EntityType { - TENANT = 0; - CUSTOMER = 1; - USER = 2; - DASHBOARD = 3; - ASSET = 4; - DEVICE = 5; - ALARM = 6; - RULE_CHAIN = 7; - RULE_NODE = 8; - ENTITY_VIEW = 9; - WIDGETS_BUNDLE = 10; - WIDGET_TYPE = 11; - TENANT_PROFILE = 12; - DEVICE_PROFILE = 13; - ASSET_PROFILE = 14; - API_USAGE_STATE = 15; - TB_RESOURCE = 16; - OTA_PACKAGE = 17; - EDGE = 18; - RPC = 19; - QUEUE = 20; - NOTIFICATION_TARGET = 21; - NOTIFICATION_TEMPLATE = 22; - NOTIFICATION_REQUEST = 23; - NOTIFICATION = 24; - NOTIFICATION_RULE = 25; +enum EntityTypeProto { + UNSPECIFIED = 0; + TENANT = 1; + CUSTOMER = 2; + USER = 3; + DASHBOARD = 4; + ASSET = 5; + DEVICE = 6; + ALARM = 7; + // next 2 reserved for PE; + RULE_CHAIN = 10; + RULE_NODE = 11; + // next 2 reserved for PE; + ENTITY_VIEW = 14; + WIDGETS_BUNDLE = 15; + WIDGET_TYPE = 16; + // next 2 reserved for PE; + TENANT_PROFILE = 19; + DEVICE_PROFILE = 20; + ASSET_PROFILE = 21; + API_USAGE_STATE = 22; + TB_RESOURCE = 23; + OTA_PACKAGE = 24; + EDGE = 25; + RPC = 26; + QUEUE = 27; + NOTIFICATION_TARGET = 28; + NOTIFICATION_TEMPLATE = 29; + NOTIFICATION_REQUEST = 30; + NOTIFICATION = 31; + NOTIFICATION_RULE = 32; } /** @@ -776,7 +780,7 @@ enum ComponentLifecycleEvent { message ComponentLifecycleMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; - EntityType entityType = 3; + EntityTypeProto entityType = 3; int64 entityIdMSB = 4; int64 entityIdLSB = 5; ComponentLifecycleEvent event = 6; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index c4344e8789..c755659966 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -26,39 +26,46 @@ import java.util.stream.Collectors; * @author Andrew Shvayka */ public enum EntityType { - // In sync with EntityType proto - TENANT, - CUSTOMER, - USER, - DASHBOARD, - ASSET, - DEVICE, - ALARM, - RULE_CHAIN, - RULE_NODE, - ENTITY_VIEW { + TENANT(1), + CUSTOMER(2), + USER(3), + DASHBOARD(4), + ASSET(5), + DEVICE(6), + ALARM (7), + RULE_CHAIN (10), + RULE_NODE (11), + + ENTITY_VIEW (14) { // backward compatibility for TbOriginatorTypeSwitchNode to return correct rule node connection. @Override - public String getNormalName() { + public String getNormalName () { return "Entity View"; } }, - WIDGETS_BUNDLE, - WIDGET_TYPE, - TENANT_PROFILE, - DEVICE_PROFILE, - ASSET_PROFILE, - API_USAGE_STATE, - TB_RESOURCE, - OTA_PACKAGE, - EDGE, - RPC, - QUEUE, - NOTIFICATION_TARGET, - NOTIFICATION_TEMPLATE, - NOTIFICATION_REQUEST, - NOTIFICATION, - NOTIFICATION_RULE; + WIDGETS_BUNDLE (15), + WIDGET_TYPE (16), + TENANT_PROFILE (19), + DEVICE_PROFILE (20), + ASSET_PROFILE (21), + API_USAGE_STATE (22), + TB_RESOURCE (23), + OTA_PACKAGE (24), + EDGE (25), + RPC (26), + QUEUE (27), + NOTIFICATION_TARGET (28), + NOTIFICATION_TEMPLATE (29), + NOTIFICATION_REQUEST (30), + NOTIFICATION (31), + NOTIFICATION_RULE (32); + + @Getter + private final int protoNumber; // Corresponds to EntityTypeProto + + private EntityType(int protoNumber) { + this.protoNumber = protoNumber; + } public static final List NORMAL_NAMES = EnumSet.allOf(EntityType.class).stream() .map(EntityType::getNormalName).collect(Collectors.toUnmodifiableList()); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index f110a6f209..f293b5dc24 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -32,6 +32,8 @@ import java.util.Optional; */ @Data public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { + private static final long serialVersionUID = -5303421482781273062L; + private final TenantId tenantId; private final EntityId entityId; private final ComponentLifecycleEvent event; From b2098fc11258fd37af36057b2f308cb8eadc0345 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 24 Oct 2023 12:30:17 +0300 Subject: [PATCH 28/29] Fix compatibility between CE and PE --- common/cluster-api/src/main/proto/queue.proto | 40 +++++++++---------- .../server/common/data/EntityType.java | 38 +++++++++--------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index e560dfccf5..8202f1c749 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -32,28 +32,28 @@ enum EntityTypeProto { ASSET = 5; DEVICE = 6; ALARM = 7; + // next 3 reserved for PE; + RULE_CHAIN = 11; + RULE_NODE = 12; // next 2 reserved for PE; - RULE_CHAIN = 10; - RULE_NODE = 11; + ENTITY_VIEW = 15; + WIDGETS_BUNDLE = 16; + WIDGET_TYPE = 17; // next 2 reserved for PE; - ENTITY_VIEW = 14; - WIDGETS_BUNDLE = 15; - WIDGET_TYPE = 16; - // next 2 reserved for PE; - TENANT_PROFILE = 19; - DEVICE_PROFILE = 20; - ASSET_PROFILE = 21; - API_USAGE_STATE = 22; - TB_RESOURCE = 23; - OTA_PACKAGE = 24; - EDGE = 25; - RPC = 26; - QUEUE = 27; - NOTIFICATION_TARGET = 28; - NOTIFICATION_TEMPLATE = 29; - NOTIFICATION_REQUEST = 30; - NOTIFICATION = 31; - NOTIFICATION_RULE = 32; + TENANT_PROFILE = 20; + DEVICE_PROFILE = 21; + ASSET_PROFILE = 22; + API_USAGE_STATE = 23; + TB_RESOURCE = 24; + OTA_PACKAGE = 25; + EDGE = 26; + RPC = 27; + QUEUE = 28; + NOTIFICATION_TARGET = 29; + NOTIFICATION_TEMPLATE = 30; + NOTIFICATION_REQUEST = 31; + NOTIFICATION = 32; + NOTIFICATION_RULE = 33; } /** diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index c755659966..ce87edf43a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -33,32 +33,32 @@ public enum EntityType { ASSET(5), DEVICE(6), ALARM (7), - RULE_CHAIN (10), - RULE_NODE (11), + RULE_CHAIN (11), + RULE_NODE (12), - ENTITY_VIEW (14) { + ENTITY_VIEW (15) { // backward compatibility for TbOriginatorTypeSwitchNode to return correct rule node connection. @Override public String getNormalName () { return "Entity View"; } }, - WIDGETS_BUNDLE (15), - WIDGET_TYPE (16), - TENANT_PROFILE (19), - DEVICE_PROFILE (20), - ASSET_PROFILE (21), - API_USAGE_STATE (22), - TB_RESOURCE (23), - OTA_PACKAGE (24), - EDGE (25), - RPC (26), - QUEUE (27), - NOTIFICATION_TARGET (28), - NOTIFICATION_TEMPLATE (29), - NOTIFICATION_REQUEST (30), - NOTIFICATION (31), - NOTIFICATION_RULE (32); + WIDGETS_BUNDLE (16), + WIDGET_TYPE (17), + TENANT_PROFILE (20), + DEVICE_PROFILE (21), + ASSET_PROFILE (22), + API_USAGE_STATE (23), + TB_RESOURCE (24), + OTA_PACKAGE (25), + EDGE (26), + RPC (27), + QUEUE (28), + NOTIFICATION_TARGET (29), + NOTIFICATION_TEMPLATE (30), + NOTIFICATION_REQUEST (31), + NOTIFICATION (32), + NOTIFICATION_RULE (33); @Getter private final int protoNumber; // Corresponds to EntityTypeProto From 9ca568d0700daab0b77ee1727b33abbdca670589 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 24 Oct 2023 13:23:14 +0300 Subject: [PATCH 29/29] Improvements for queue-related tests --- .../controller/TenantControllerTest.java | 2 +- .../TbRuleEngineQueueConsumerManagerTest.java | 21 ++++++------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java index d1aefc097f..27a9872ba5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java @@ -687,7 +687,7 @@ public class TenantControllerTest extends AbstractControllerTest { submittedMsgs.add(tbMsg.getId()); Thread.sleep(timeLeft / msgs); } - await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> { + await().atMost(15, TimeUnit.SECONDS).untilAsserted(() -> { verify(queueAdmin, times(1)).deleteTopic(eq(isolatedTopic)); }); diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java index 38582d0fce..05eacc68e3 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java @@ -456,8 +456,7 @@ public class TbRuleEngineQueueConsumerManagerTest { clearInvocations(actorContext); verify(consumer1, never()).unsubscribe(); verify(consumer2, never()).unsubscribe(); - int msgCount1 = consumer1.msgCount; - int msgCount2 = consumer2.msgCount; + int msgCount = totalConsumedMsgs.get(); await().atLeast(4, TimeUnit.SECONDS) // based on topicDeletionDelayInSec .atMost(7, TimeUnit.SECONDS) @@ -471,10 +470,8 @@ public class TbRuleEngineQueueConsumerManagerTest { verify(consumer1).unsubscribe(); verify(consumer2).unsubscribe(); - int movedMsgs1 = consumer1.msgCount - msgCount1; - int movedMsgs2 = consumer2.msgCount - msgCount2; - int totalMovedMsgs = movedMsgs1 + movedMsgs2; - assertThat(totalMovedMsgs).isGreaterThan(10); + int totalMovedMsgs = totalConsumedMsgs.get() - msgCount; + assertThat(totalMovedMsgs).isNotZero(); verify(ruleEngineMsgProducer, atLeast(totalMovedMsgs)).send(any(), any(), any()); verify(actorContext, never()).tell(any()); generateQueueMsgs = false; @@ -499,7 +496,7 @@ public class TbRuleEngineQueueConsumerManagerTest { }); clearInvocations(actorContext); verify(consumer, never()).unsubscribe(); - int msgCount = consumer.msgCount; + int msgCount = totalConsumedMsgs.get(); await().atLeast(4, TimeUnit.SECONDS) .atMost(7, TimeUnit.SECONDS) @@ -512,7 +509,8 @@ public class TbRuleEngineQueueConsumerManagerTest { }); verify(consumer).unsubscribe(); - int movedMsgs = consumer.msgCount - msgCount; + int movedMsgs = totalConsumedMsgs.get() - msgCount; + assertThat(movedMsgs).isNotZero(); verify(ruleEngineMsgProducer, atLeast(movedMsgs)).send(any(), any(), any()); verify(actorContext, never()).tell(any()); generateQueueMsgs = false; @@ -607,11 +605,6 @@ public class TbRuleEngineQueueConsumerManagerTest { }); } - /* - * 2023-10-15 18:34:06,090 [main] INFO o.t.s.s.q.r.TbRuleEngineQueueConsumerManagerTest - Generated new partitions: [0, 1, 2, 3, 4, 5, 6, 8, 9, 11, 12, 13, 15, 16, 17, 18, 19] -2023-10-15 18:34:06,090 [main] INFO o.t.s.s.q.r.TbRuleEngineQueueConsumerManagerTest - Generated new config: consumerPerPartition=false, pollInterval=299, processingStrategy=RETRY_FAILED - * */ - private void verifySubscribedAndLaunched(TestConsumer consumer, Set expectedPartitions) { await().atMost(2, TimeUnit.SECONDS) .until(() -> consumer.subscribed && consumer.getPartitions().equals(expectedPartitions) && consumer.pollingStarted); @@ -701,7 +694,6 @@ public class TbRuleEngineQueueConsumerManagerTest { private boolean pollingStarted; private TbMsg testMsg; - private int msgCount; public TestConsumer(String topic) { super(topic); @@ -746,7 +738,6 @@ public class TbRuleEngineQueueConsumerManagerTest { if (!subscribed) { throw new IllegalStateException("Cannot commit because not subscribed"); } - msgCount++; log.debug("doCommit() totalConsumedMsgs = {}", totalConsumedMsgs.incrementAndGet()); }