From 8c94ec5c5277610c1dd494c75c1c677898885210 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Mon, 16 Oct 2023 18:05:12 +0300 Subject: [PATCH 01/23] Edge: do not sync User login actions. Edge: do not save particular edge events in case edge is not activated or disconnected --- .../edge/EdgeEventSourcingListener.java | 42 +++++++++++--- .../edge/rpc/processor/BaseEdgeProcessor.java | 58 +++++++++++++++++++ .../server/edge/AbstractEdgeTest.java | 49 +++++++--------- .../server/edge/DeviceEdgeTest.java | 2 - .../thingsboard/server/edge/UserEdgeTest.java | 8 --- .../dao/eventsourcing/SaveEntityEvent.java | 1 + .../server/dao/user/UserServiceImpl.java | 7 ++- 7 files changed, 119 insertions(+), 48 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java index 43b05094a4..707dbec964 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.service.edge; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @@ -36,6 +38,7 @@ import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.RelationActionEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; +import org.thingsboard.server.dao.user.UserServiceImpl; import javax.annotation.PostConstruct; @@ -75,7 +78,7 @@ public class EdgeEventSourcingListener { return; } try { - if (!isValidEdgeEventEntity(event.getEntity())) { + if (!isValidEdgeEventEntity(event.getEntity(), event.getOldEntity())) { return; } log.trace("[{}] SaveEntityEvent called: {}", event.getTenantId(), event); @@ -83,7 +86,7 @@ public class EdgeEventSourcingListener { tbClusterService.sendNotificationMsgToEdge(event.getTenantId(), null, event.getEntityId(), null, null, action); } catch (Exception e) { - log.error("[{}] failed to process SaveEntityEvent: {}", event.getTenantId(), event); + log.error("[{}] failed to process SaveEntityEvent: {}", event.getTenantId(), event, e); } } @@ -97,7 +100,7 @@ public class EdgeEventSourcingListener { tbClusterService.sendNotificationMsgToEdge(event.getTenantId(), event.getEdgeId(), event.getEntityId(), JacksonUtil.toString(event.getEntity()), null, EdgeEventActionType.DELETED); } catch (Exception e) { - log.error("[{}] failed to process DeleteEntityEvent: {}", event.getTenantId(), event); + log.error("[{}] failed to process DeleteEntityEvent: {}", event.getTenantId(), event, e); } } @@ -111,7 +114,7 @@ public class EdgeEventSourcingListener { tbClusterService.sendNotificationMsgToEdge(event.getTenantId(), event.getEdgeId(), event.getEntityId(), event.getBody(), null, edgeTypeByActionType(event.getActionType())); } catch (Exception e) { - log.error("[{}] failed to process ActionEntityEvent: {}", event.getTenantId(), event); + log.error("[{}] failed to process ActionEntityEvent: {}", event.getTenantId(), event, e); } } @@ -134,11 +137,11 @@ public class EdgeEventSourcingListener { tbClusterService.sendNotificationMsgToEdge(event.getTenantId(), null, null, JacksonUtil.toString(relation), EdgeEventType.RELATION, edgeTypeByActionType(event.getActionType())); } catch (Exception e) { - log.error("[{}] failed to process RelationActionEvent: {}", event.getTenantId(), event); + log.error("[{}] failed to process RelationActionEvent: {}", event.getTenantId(), event, e); } } - private boolean isValidEdgeEventEntity(Object entity) { + private boolean isValidEdgeEventEntity(Object entity, Object oldEntity) { if (entity instanceof OtaPackageInfo) { OtaPackageInfo otaPackageInfo = (OtaPackageInfo) entity; return otaPackageInfo.hasUrl() || otaPackageInfo.isHasData(); @@ -147,7 +150,15 @@ public class EdgeEventSourcingListener { return RuleChainType.EDGE.equals(ruleChain.getType()); } else if (entity instanceof User) { User user = (User) entity; - return !Authority.SYS_ADMIN.equals(user.getAuthority()); + if (Authority.SYS_ADMIN.equals(user.getAuthority())) { + return false; + } + if (oldEntity != null) { + User oldUser = (User) oldEntity; + cleanUpUserAdditionalInfo(oldUser); + cleanUpUserAdditionalInfo(user); + return !user.equals(oldUser); + } } else if (entity instanceof AlarmApiCallResult) { AlarmApiCallResult alarmApiCallResult = (AlarmApiCallResult) entity; return alarmApiCallResult.isModified(); @@ -155,4 +166,21 @@ public class EdgeEventSourcingListener { // Default: If the entity doesn't match any of the conditions, consider it as valid. return true; } + + private void cleanUpUserAdditionalInfo(User user) { + // reset FAILED_LOGIN_ATTEMPTS and LAST_LOGIN_TS - edge is not interested in this information + if (user.getAdditionalInfo() instanceof NullNode) { + user.setAdditionalInfo(null); + } + if (user.getAdditionalInfo() instanceof ObjectNode) { + ObjectNode additionalInfo = ((ObjectNode) user.getAdditionalInfo()); + additionalInfo.remove(UserServiceImpl.FAILED_LOGIN_ATTEMPTS); + additionalInfo.remove(UserServiceImpl.LAST_LOGIN_TS); + if (additionalInfo.isEmpty()) { + user.setAdditionalInfo(null); + } else { + user.setAdditionalInfo(additionalInfo); + } + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index 328a64efe2..2d86c1e184 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -24,6 +24,7 @@ import org.springframework.context.annotation.Lazy; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EdgeUtils; @@ -46,6 +47,7 @@ import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -110,11 +112,13 @@ import org.thingsboard.server.service.entitiy.TbNotificationEntityService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import org.thingsboard.server.service.state.DefaultDeviceStateService; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -305,6 +309,60 @@ public abstract class BaseEdgeProcessor { EdgeEventActionType action, EntityId entityId, JsonNode body) { + ListenableFuture> future = attributesService.find(tenantId, edgeId, DataConstants.SERVER_SCOPE, DefaultDeviceStateService.ACTIVITY_STATE); + return Futures.transformAsync(future, activeOpt -> { + if (activeOpt.isEmpty()) { + log.trace("Edge is not activated. Skipping event. tenantId [{}], edgeId [{}], type[{}], " + + "action [{}], entityId [{}], body [{}]", + tenantId, edgeId, type, action, entityId, body); + return Futures.immediateFuture(null); + } + if (activeOpt.get().getBooleanValue().isPresent() && activeOpt.get().getBooleanValue().get()) { + return doSaveEdgeEvent(tenantId, edgeId, type, action, entityId, body); + } else { + if (doSaveIfEdgeIsOffline(type, action)) { + return doSaveEdgeEvent(tenantId, edgeId, type, action, entityId, body); + } else { + log.trace("Edge is not active at the moment. Skipping event. tenantId [{}], edgeId [{}], type[{}], " + + "action [{}], entityId [{}], body [{}]", + tenantId, edgeId, type, action, entityId, body); + return Futures.immediateFuture(null); + } + } + }, dbCallbackExecutorService); + } + + private boolean doSaveIfEdgeIsOffline(EdgeEventType type, + EdgeEventActionType action) { + switch (action) { + case TIMESERIES_UPDATED: + case ALARM_ACK: + case ALARM_CLEAR: + case ALARM_ASSIGNED: + case ALARM_UNASSIGNED: + case CREDENTIALS_REQUEST: + return true; + } + switch (type) { + case ALARM: + case RULE_CHAIN: + case RULE_CHAIN_METADATA: + case USER: + case CUSTOMER: + case TENANT: + case TENANT_PROFILE: + case WIDGETS_BUNDLE: + case WIDGET_TYPE: + case ADMIN_SETTINGS: + case OTA_PACKAGE: + case QUEUE: + case RELATION: + return true; + } + return false; + } + + private ListenableFuture doSaveEdgeEvent(TenantId tenantId, EdgeId edgeId, EdgeEventType type, EdgeEventActionType action, EntityId entityId, JsonNode body) { log.debug("Pushing event to edge queue. tenantId [{}], edgeId [{}], type[{}], " + "action [{}], entityId [{}], body [{}]", tenantId, edgeId, type, action, entityId, body); diff --git a/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java index 689847416c..43544dd5cf 100644 --- a/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java @@ -130,7 +130,7 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { installation(); edgeImitator = new EdgeImitator("localhost", 7070, edge.getRoutingKey(), edge.getSecret()); - edgeImitator.expectMessageAmount(26); + edgeImitator.expectMessageAmount(21); edgeImitator.connect(); requestEdgeRuleChainMetadata(); @@ -163,9 +163,7 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { @After public void teardownEdgeTest() { try { - edgeImitator.expectMessageAmount(2); loginTenantAdmin(); - Assert.assertTrue(edgeImitator.waitForMessages()); doDelete("/api/edge/" + edge.getId().toString()) .andExpect(status().isOk()); @@ -228,33 +226,33 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { // 1 message from queue fetcher validateQueues(); - // 2 messages - 1 from rule chain fetcher and 1 from rule chain controller + // 1 from rule chain fetcher UUID ruleChainUUID = validateRuleChains(); // 1 from request message validateRuleChainMetadataUpdates(ruleChainUUID); - // 4 messages - 4 messages from fetcher - 2 from system level ('mail', 'mailTemplates') and 2 from admin level ('mail', 'mailTemplates') + // 4 messages + // - 2 from fetcher - system level ('mail', 'mailTemplates') + // - 2 from fetcher - admin level ('mail', 'mailTemplates') validateAdminSettings(); - // 5 messages + // 4 messages // - 1 from default profile fetcher // - 2 from device profile fetcher (default and thermostat) // - 1 from device fetcher - // - 1 from device controller (thermostat) validateDeviceProfiles(); - // 4 messages + // 3 messages // - 1 from default profile fetcher // - 1 message from asset profile fetcher // - 1 message from asset fetcher - // - 1 message from asset controller validateAssetProfiles(); - // 2 messages - 1 from device fetcher and 1 from device controller + // 1 from device fetcher validateDevices(); - // 2 messages - 1 from asset fetcher and 1 from asset controller + // 1 from asset fetcher validateAssets(); // 1 message from public customer fetcher @@ -308,8 +306,7 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { // default msg device profile from fetcher // thermostat msg from device profile fetcher // thermostat msg from device fetcher - // thermostat msg from creation of device - Assert.assertEquals(5, deviceProfileUpdateMsgList.size()); + Assert.assertEquals(4, deviceProfileUpdateMsgList.size()); Optional thermostatProfileUpdateMsgOpt = deviceProfileUpdateMsgList.stream().filter(dfum -> THERMOSTAT_DEVICE_PROFILE_NAME.equals(dfum.getName())).findAny(); Assert.assertTrue(thermostatProfileUpdateMsgOpt.isPresent()); @@ -326,10 +323,9 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { } private void validateDevices() throws Exception { - List deviceUpdateMsgs = edgeImitator.findAllMessagesByType(DeviceUpdateMsg.class); - Assert.assertEquals(2, deviceUpdateMsgs.size()); - validateDevice(deviceUpdateMsgs.get(0)); - validateDevice(deviceUpdateMsgs.get(1)); + Optional deviceUpdateMsgOpt = edgeImitator.findMessageByType(DeviceUpdateMsg.class); + Assert.assertTrue(deviceUpdateMsgOpt.isPresent()); + validateDevice(deviceUpdateMsgOpt.get()); } private void validateDevice(DeviceUpdateMsg deviceUpdateMsg) throws Exception { @@ -345,10 +341,9 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { } private void validateAssets() throws Exception { - List assetUpdateMsgs = edgeImitator.findAllMessagesByType(AssetUpdateMsg.class); - Assert.assertEquals(2, assetUpdateMsgs.size()); - validateAsset(assetUpdateMsgs.get(0)); - validateAsset(assetUpdateMsgs.get(1)); + Optional assetUpdateMsgOpt = edgeImitator.findMessageByType(AssetUpdateMsg.class); + Assert.assertTrue(assetUpdateMsgOpt.isPresent()); + validateAsset(assetUpdateMsgOpt.get()); } private void validateAsset(AssetUpdateMsg assetUpdateMsg) throws Exception { @@ -365,12 +360,10 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { } private UUID validateRuleChains() throws Exception { - List ruleChainUpdateMsgs = edgeImitator.findAllMessagesByType(RuleChainUpdateMsg.class); - Assert.assertEquals(2, ruleChainUpdateMsgs.size()); - RuleChainUpdateMsg ruleChainCreateMsg = ruleChainUpdateMsgs.get(0); - RuleChainUpdateMsg ruleChainUpdateMsg = ruleChainUpdateMsgs.get(1); - validateRuleChain(ruleChainCreateMsg, UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE); - validateRuleChain(ruleChainUpdateMsg, UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE); + Optional ruleChainUpdateMsgOpt = edgeImitator.findMessageByType(RuleChainUpdateMsg.class); + Assert.assertTrue(ruleChainUpdateMsgOpt.isPresent()); + RuleChainUpdateMsg ruleChainUpdateMsg = ruleChainUpdateMsgOpt.get(); + validateRuleChain(ruleChainUpdateMsg, UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE); return new UUID(ruleChainUpdateMsg.getIdMSB(), ruleChainUpdateMsg.getIdLSB()); } @@ -429,7 +422,7 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { private void validateAssetProfiles() throws Exception { List assetProfileUpdateMsgs = edgeImitator.findAllMessagesByType(AssetProfileUpdateMsg.class); - Assert.assertEquals(4, assetProfileUpdateMsgs.size()); + Assert.assertEquals(3, assetProfileUpdateMsgs.size()); AssetProfileUpdateMsg assetProfileUpdateMsg = assetProfileUpdateMsgs.get(0); Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, assetProfileUpdateMsg.getMsgType()); UUID assetProfileUUID = new UUID(assetProfileUpdateMsg.getIdMSB(), assetProfileUpdateMsg.getIdLSB()); diff --git a/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java index aed99b4cba..ca8dd5dbf0 100644 --- a/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java @@ -274,9 +274,7 @@ public class DeviceEdgeTest extends AbstractEdgeTest { tenantProfile.getProfileData().setConfiguration(profileConfiguration); doPost("/api/tenantProfile/", tenantProfile, TenantProfile.class); - edgeImitator.expectMessageAmount(2); loginTenantAdmin(); - Assert.assertTrue(edgeImitator.waitForMessages()); UUID uuid = Uuids.timeBased(); diff --git a/application/src/test/java/org/thingsboard/server/edge/UserEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/UserEdgeTest.java index 3fd39328d5..3dc398b9f3 100644 --- a/application/src/test/java/org/thingsboard/server/edge/UserEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/UserEdgeTest.java @@ -79,9 +79,7 @@ public class UserEdgeTest extends AbstractEdgeTest { Assert.assertEquals(savedTenantAdmin.getLastName(), userUpdateMsg.getLastName()); // update user credentials - edgeImitator.expectMessageAmount(2); login(savedTenantAdmin.getEmail(), "tenant"); - Assert.assertTrue(edgeImitator.waitForMessages()); edgeImitator.expectMessageAmount(1); ChangePasswordRequest changePasswordRequest = new ChangePasswordRequest(); @@ -96,9 +94,7 @@ public class UserEdgeTest extends AbstractEdgeTest { Assert.assertEquals(savedTenantAdmin.getUuidId().getLeastSignificantBits(), userCredentialsUpdateMsg.getUserIdLSB()); Assert.assertTrue(passwordEncoder.matches(changePasswordRequest.getNewPassword(), userCredentialsUpdateMsg.getPassword())); - edgeImitator.expectMessageAmount(2); loginTenantAdmin(); - Assert.assertTrue(edgeImitator.waitForMessages()); // delete user edgeImitator.expectMessageAmount(1); @@ -164,9 +160,7 @@ public class UserEdgeTest extends AbstractEdgeTest { Assert.assertEquals(savedCustomerUser.getLastName(), userUpdateMsg.getLastName()); // update user credentials - edgeImitator.expectMessageAmount(2); login(savedCustomerUser.getEmail(), "customer"); - Assert.assertTrue(edgeImitator.waitForMessages()); edgeImitator.expectMessageAmount(1); ChangePasswordRequest changePasswordRequest = new ChangePasswordRequest(); @@ -181,9 +175,7 @@ public class UserEdgeTest extends AbstractEdgeTest { Assert.assertEquals(savedCustomerUser.getUuidId().getLeastSignificantBits(), userCredentialsUpdateMsg.getUserIdLSB()); Assert.assertTrue(passwordEncoder.matches(changePasswordRequest.getNewPassword(), userCredentialsUpdateMsg.getPassword())); - edgeImitator.expectMessageAmount(2); loginTenantAdmin(); - Assert.assertTrue(edgeImitator.waitForMessages()); // delete user edgeImitator.expectMessageAmount(1); diff --git a/dao/src/main/java/org/thingsboard/server/dao/eventsourcing/SaveEntityEvent.java b/dao/src/main/java/org/thingsboard/server/dao/eventsourcing/SaveEntityEvent.java index 205f592d43..cc2e854f59 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/eventsourcing/SaveEntityEvent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/eventsourcing/SaveEntityEvent.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.id.TenantId; public class SaveEntityEvent { private final TenantId tenantId; private final T entity; + private final T oldEntity; private final EntityId entityId; private final Boolean added; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index 4db1eb668a..a584fef86c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -70,8 +70,8 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic public static final String USER_PASSWORD_HISTORY = "userPasswordHistory"; - private static final String LAST_LOGIN_TS = "lastLoginTs"; - private static final String FAILED_LOGIN_ATTEMPTS = "failedLoginAttempts"; + public static final String LAST_LOGIN_TS = "lastLoginTs"; + public static final String FAILED_LOGIN_ATTEMPTS = "failedLoginAttempts"; private static final int DEFAULT_TOKEN_LENGTH = 30; public static final String INCORRECT_USER_ID = "Incorrect userId "; @@ -126,7 +126,7 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic @Override public User saveUser(TenantId tenantId, User user) { log.trace("Executing saveUser [{}]", user); - userValidator.validate(user, User::getTenantId); + User oldUser = userValidator.validate(user, User::getTenantId); if (!userLoginCaseSensitive) { user.setEmail(user.getEmail().toLowerCase()); } @@ -143,6 +143,7 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic eventPublisher.publishEvent(SaveEntityEvent.builder() .tenantId(tenantId == null ? TenantId.SYS_TENANT_ID : tenantId) .entity(savedUser) + .oldEntity(oldUser) .entityId(savedUser.getId()) .added(user.getId() == null).build()); return savedUser; From 3946b058cdda60fbdfb03d92d0c60fb3d1399605 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 17 Oct 2023 18:09:38 +0300 Subject: [PATCH 02/23] Do not push ALARM update to edge automatically - only using push to edge rule node --- .../service/edge/EdgeEventSourcingListener.java | 10 +++++----- .../thingsboard/server/edge/AlarmEdgeTest.java | 15 --------------- ui-ngx/src/app/core/http/entity.service.ts | 10 +++++++--- 3 files changed, 12 insertions(+), 23 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java index 707dbec964..006af577c9 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java @@ -25,6 +25,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmApiCallResult; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; @@ -78,7 +79,7 @@ public class EdgeEventSourcingListener { return; } try { - if (!isValidEdgeEventEntity(event.getEntity(), event.getOldEntity())) { + if (!isValidSaveEntityEventForEdgeProcessing(event.getEntity(), event.getOldEntity())) { return; } log.trace("[{}] SaveEntityEvent called: {}", event.getTenantId(), event); @@ -141,7 +142,7 @@ public class EdgeEventSourcingListener { } } - private boolean isValidEdgeEventEntity(Object entity, Object oldEntity) { + private boolean isValidSaveEntityEventForEdgeProcessing(Object entity, Object oldEntity) { if (entity instanceof OtaPackageInfo) { OtaPackageInfo otaPackageInfo = (OtaPackageInfo) entity; return otaPackageInfo.hasUrl() || otaPackageInfo.isHasData(); @@ -159,9 +160,8 @@ public class EdgeEventSourcingListener { cleanUpUserAdditionalInfo(user); return !user.equals(oldUser); } - } else if (entity instanceof AlarmApiCallResult) { - AlarmApiCallResult alarmApiCallResult = (AlarmApiCallResult) entity; - return alarmApiCallResult.isModified(); + } else if (entity instanceof AlarmApiCallResult || entity instanceof Alarm) { + return false; } // Default: If the entity doesn't match any of the conditions, consider it as valid. return true; diff --git a/application/src/test/java/org/thingsboard/server/edge/AlarmEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/AlarmEdgeTest.java index afccc634ed..259572851c 100644 --- a/application/src/test/java/org/thingsboard/server/edge/AlarmEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/AlarmEdgeTest.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.google.protobuf.AbstractMessage; import org.junit.Assert; import org.junit.Test; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; @@ -102,20 +101,6 @@ public class AlarmEdgeTest extends AbstractEdgeTest { Assert.assertEquals(savedAlarm.getStatus().name(), alarmUpdateMsg.getStatus()); Assert.assertEquals(savedAlarm.getSeverity().name(), alarmUpdateMsg.getSeverity()); - // update alarm - String updatedDetails = "{\"testKey\":\"testValue\"}"; - savedAlarm.setDetails(JacksonUtil.OBJECT_MAPPER.readTree(updatedDetails)); - edgeImitator.expectMessageAmount(1); - savedAlarm = doPost("/api/alarm", savedAlarm, Alarm.class); - Assert.assertTrue(edgeImitator.waitForMessages()); - latestMessage = edgeImitator.getLatestMessage(); - Assert.assertTrue(latestMessage instanceof AlarmUpdateMsg); - alarmUpdateMsg = (AlarmUpdateMsg) latestMessage; - Assert.assertEquals(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE, alarmUpdateMsg.getMsgType()); - Assert.assertEquals(savedAlarm.getUuidId().getMostSignificantBits(), alarmUpdateMsg.getIdMSB()); - Assert.assertEquals(savedAlarm.getUuidId().getLeastSignificantBits(), alarmUpdateMsg.getIdLSB()); - Assert.assertEquals(updatedDetails, alarmUpdateMsg.getDetails()); - // ack alarm edgeImitator.expectMessageAmount(1); doPost("/api/alarm/" + savedAlarm.getUuidId() + "/ack"); diff --git a/ui-ngx/src/app/core/http/entity.service.ts b/ui-ngx/src/app/core/http/entity.service.ts index 12f6300b93..cc1dd13345 100644 --- a/ui-ngx/src/app/core/http/entity.service.ts +++ b/ui-ngx/src/app/core/http/entity.service.ts @@ -92,6 +92,7 @@ import { NotificationService } from '@core/http/notification.service'; import { TenantProfileService } from '@core/http/tenant-profile.service'; import { NotificationType } from '@shared/models/notification.models'; import { UserId } from '@shared/models/id/user-id'; +import { AlarmService } from '@core/http/alarm.service'; @Injectable({ providedIn: 'root' @@ -119,7 +120,8 @@ export class EntityService { private assetProfileService: AssetProfileService, private utils: UtilsService, private queueService: QueueService, - private notificationService: NotificationService + private notificationService: NotificationService, + private alarmService: AlarmService ) { } private getEntityObservable(entityType: EntityType, entityId: string, @@ -155,7 +157,7 @@ export class EntityService { observable = this.ruleChainService.getRuleChain(entityId, config); break; case EntityType.ALARM: - console.error('Get Alarm Entity is not implemented!'); + observable = this.alarmService.getAlarm(entityId, config); break; case EntityType.OTA_PACKAGE: observable = this.otaPackageService.getOtaPackageInfo(entityId, config); @@ -238,7 +240,9 @@ export class EntityService { entityIds); break; case EntityType.ALARM: - console.error('Get Alarm Entity is not implemented!'); + observable = this.getEntitiesByIdsObservable( + (id) => this.alarmService.getAlarm(id, config), + entityIds); break; case EntityType.DEVICE_PROFILE: observable = this.getEntitiesByIdsObservable( From 04cfa2021dcc989d6130a43334a435dfaeb2f0d9 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 17 Oct 2023 18:50:42 +0300 Subject: [PATCH 03/23] Fixed edge controller tests --- .../edge/rpc/processor/BaseEdgeProcessor.java | 3 ++- .../server/controller/EdgeControllerTest.java | 11 +++++++++-- .../server/controller/EdgeEventControllerTest.java | 13 ++++++++++--- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index 2d86c1e184..a888ddc27b 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -309,7 +309,8 @@ public abstract class BaseEdgeProcessor { EdgeEventActionType action, EntityId entityId, JsonNode body) { - ListenableFuture> future = attributesService.find(tenantId, edgeId, DataConstants.SERVER_SCOPE, DefaultDeviceStateService.ACTIVITY_STATE); + ListenableFuture> future = + attributesService.find(tenantId, edgeId, DataConstants.SERVER_SCOPE, DefaultDeviceStateService.ACTIVITY_STATE); return Futures.transformAsync(future, activeOpt -> { if (activeOpt.isEmpty()) { log.trace("Edge is not activated. Skipping event. tenantId [{}], edgeId [{}], type[{}], " + diff --git a/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java index 1afeaff0cb..b9e0b8cd6c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.controller; import com.datastax.oss.driver.api.core.uuid.Uuids; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -34,8 +35,10 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Primary; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.StringUtils; @@ -852,6 +855,11 @@ public class EdgeControllerTest extends AbstractControllerTest { Edge edge = doPost("/api/edge", constructEdge("Test Sync Edge", "test"), Edge.class); + // simulate edge activation + ObjectNode attributes = JacksonUtil.newObjectNode(); + attributes.put("active", true); + doPost("/api/plugins/telemetry/EDGE/" + edge.getId() + "/attributes/" + DataConstants.SERVER_SCOPE, attributes); + doPost("/api/edge/" + edge.getId().getId().toString() + "/device/" + savedDevice.getId().getId().toString(), Device.class); doPost("/api/edge/" + edge.getId().getId().toString() @@ -860,13 +868,12 @@ public class EdgeControllerTest extends AbstractControllerTest { EdgeImitator edgeImitator = new EdgeImitator(EDGE_HOST, EDGE_PORT, edge.getRoutingKey(), edge.getSecret()); edgeImitator.ignoreType(UserCredentialsUpdateMsg.class); - edgeImitator.expectMessageAmount(25); + edgeImitator.expectMessageAmount(24); edgeImitator.connect(); assertThat(edgeImitator.waitForMessages()).as("await for messages on first connect").isTrue(); verifyFetchersMsgs(edgeImitator); // verify queue msgs - Assert.assertTrue(popRuleChainMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE, "Edge Root Rule Chain")); Assert.assertTrue(popDeviceProfileMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "default")); Assert.assertTrue(popDeviceMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "Test Sync Edge Device 1")); Assert.assertTrue(popAssetProfileMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "test")); diff --git a/application/src/test/java/org/thingsboard/server/controller/EdgeEventControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdgeEventControllerTest.java index 04842df6a0..3303c64c27 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdgeEventControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdgeEventControllerTest.java @@ -16,6 +16,7 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import org.awaitility.Awaitility; import org.junit.After; @@ -26,6 +27,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.TestPropertySource; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.edge.Edge; @@ -86,6 +89,11 @@ public class EdgeEventControllerTest extends AbstractControllerTest { Edge edge = constructEdge("TestEdge", "default"); edge = doPost("/api/edge", edge, Edge.class); + // simulate edge activation + ObjectNode attributes = JacksonUtil.newObjectNode(); + attributes.put("active", true); + doPost("/api/plugins/telemetry/EDGE/" + edge.getId() + "/attributes/" + DataConstants.SERVER_SCOPE, attributes); + Device device = constructDevice("TestDevice", "default"); Device savedDevice = doPost("/api/device", device, Device.class); @@ -99,14 +107,13 @@ public class EdgeEventControllerTest extends AbstractControllerTest { EntityRelation relation = new EntityRelation(savedAsset.getId(), savedDevice.getId(), EntityRelation.CONTAINS_TYPE); - awaitForNumberOfEdgeEvents(edgeId, 3); + awaitForNumberOfEdgeEvents(edgeId, 2); doPost("/api/relation", relation); - awaitForNumberOfEdgeEvents(edgeId, 4); + awaitForNumberOfEdgeEvents(edgeId, 3); List edgeEvents = findEdgeEvents(edgeId); - Assert.assertTrue(popEdgeEvent(edgeEvents, EdgeEventType.RULE_CHAIN)); // root rule chain Assert.assertTrue(popEdgeEvent(edgeEvents, EdgeEventType.DEVICE)); // TestDevice Assert.assertTrue(popEdgeEvent(edgeEvents, EdgeEventType.ASSET)); // TestAsset Assert.assertTrue(popEdgeEvent(edgeEvents, EdgeEventType.RELATION)); From c5c037b874a32519c79490e7d1f75ad8e832dd52 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 18 Oct 2023 08:47:03 +0300 Subject: [PATCH 04/23] Update AlarmControllerTest with latest business logic - no edge notification during alarm udpate --- .../thingsboard/server/controller/AlarmControllerTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java index 0e6eb16713..2f859055bd 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java @@ -136,7 +136,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(AlarmSeverity.MAJOR, updatedAlarm.getSeverity()); AlarmInfo foundAlarm = doGet("/api/alarm/info/" + updatedAlarm.getId(), AlarmInfo.class); - testNotifyEntityAllOneTime(foundAlarm, updatedAlarm.getId(), updatedAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, updatedAlarm.getId(), updatedAlarm.getOriginator(), tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.UPDATED); } @@ -153,7 +153,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(AlarmSeverity.MAJOR, updatedAlarm.getSeverity()); AlarmInfo foundAlarm = doGet("/api/alarm/info/" + updatedAlarm.getId(), AlarmInfo.class); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.UPDATED); alarm = updatedAlarm; From 9b6b161d37e79506f9895c53f4abd58d5dfdeda5 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 18 Oct 2023 10:15:51 +0300 Subject: [PATCH 05/23] Revert changes for getting alarms by ids --- ui-ngx/src/app/core/http/entity.service.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ui-ngx/src/app/core/http/entity.service.ts b/ui-ngx/src/app/core/http/entity.service.ts index cc1dd13345..b4eab83b08 100644 --- a/ui-ngx/src/app/core/http/entity.service.ts +++ b/ui-ngx/src/app/core/http/entity.service.ts @@ -240,9 +240,7 @@ export class EntityService { entityIds); break; case EntityType.ALARM: - observable = this.getEntitiesByIdsObservable( - (id) => this.alarmService.getAlarm(id, config), - entityIds); + console.error('Get Alarm Entity is not implemented!'); break; case EntityType.DEVICE_PROFILE: observable = this.getEntitiesByIdsObservable( From 45fd970898930b91a839fcf4fd7f847f00163454 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 18 Sep 2023 12:49:19 +0300 Subject: [PATCH 06/23] Refactoring in progress --- .../DefaultTbRuleEngineConsumerService.java | 201 ++++++++--------- .../queue/TbQueueConsumerLauncher.java | 37 ++++ .../queue/TbQueueConsumerManagerTask.java | 17 ++ .../queue/TbRuleEngineConsumerStats.java | 5 +- .../TbRuleEngineQueueConsumerManager.java | 205 ++++++++++++++++++ .../provider/TbRuleEngineQueueFactory.java | 1 - 6 files changed, 356 insertions(+), 110 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerLauncher.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerManagerTask.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineQueueConsumerManager.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index c715b7b03c..ae04de456b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -68,6 +68,7 @@ import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrateg import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; import org.thingsboard.server.service.stats.RuleEngineStatisticsService; +import org.threadly.concurrent.wrapper.KeyDistributedExecutor; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -116,12 +117,12 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< private final QueueService queueService; private final TbQueueProducerProvider producerProvider; private final TbQueueAdmin queueAdmin; - private final ConcurrentMap>> consumers = new ConcurrentHashMap<>(); - private final ConcurrentMap consumerConfigurations = new ConcurrentHashMap<>(); - private final ConcurrentMap consumerStats = new ConcurrentHashMap<>(); - private final ConcurrentMap topicsConsumerPerPartition = new ConcurrentHashMap<>(); - final ExecutorService submitExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-submit")); - final ScheduledExecutorService repartitionExecutor = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-repartition")); + + private final ConcurrentMap consumerMap = new ConcurrentHashMap<>(); + private final ExecutorService consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer")); + private final ExecutorService submitExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-submit")); + private final ScheduledExecutorService repartitionExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-repartition")); + public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory, TbRuleEngineSubmitStrategyFactory submitStrategyFactory, @@ -153,7 +154,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @PostConstruct public void init() { - super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer"); + super.init("tb-rule-engine-notifications-consumer"); // TODO: restore init of the main consumer? List queues = queueService.findAllQueues(); for (Queue configuration : queues) { if (partitionService.isManagedByCurrentService(configuration.getTenantId())) { @@ -163,19 +164,14 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } private void initConsumer(Queue configuration) { - QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, configuration); - consumerConfigurations.putIfAbsent(queueKey, configuration); - consumerStats.putIfAbsent(queueKey, new TbRuleEngineConsumerStats(configuration, statsFactory)); - if (!configuration.isConsumerPerPartition()) { - consumers.computeIfAbsent(queueKey, queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration)); - } else { - topicsConsumerPerPartition.computeIfAbsent(queueKey, k -> new TbTopicWithConsumerPerPartition(k.getQueueName())); - } + consumerMap.computeIfAbsent(new QueueKey(ServiceType.TB_RULE_ENGINE, configuration), + key -> new TbRuleEngineQueueConsumerManager(repartitionExecutor, consumersExecutor, statsFactory, tbRuleEngineQueueFactory, key)).init(configuration); } @PreDestroy public void stop() { super.destroy(); + consumersExecutor.shutdownNow(); // TODO: shutdown or shutdownNow? submitExecutor.shutdownNow(); repartitionExecutor.shutdownNow(); } @@ -183,119 +179,107 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { if (event.getServiceType().equals(getServiceType())) { - event.getPartitionsMap().forEach((queueKey, partitions) -> { - String serviceQueue = queueKey.getQueueName(); - log.info("[{}] Subscribing to partitions: {}", serviceQueue, partitions); - Queue configuration = consumerConfigurations.get(queueKey); - if (configuration == null) { - return; - } - if (!configuration.isConsumerPerPartition()) { - consumers.get(queueKey).subscribe(partitions); - } else { - log.info("[{}] Subscribing consumer per partition: {}", serviceQueue, partitions); - subscribeConsumerPerPartition(queueKey, partitions); - } - }); - } - } - - void subscribeConsumerPerPartition(QueueKey queue, Set partitions) { - topicsConsumerPerPartition.get(queue).getSubscribeQueue().add(partitions); - scheduleTopicRepartition(queue); - } - - private void scheduleTopicRepartition(QueueKey queue) { - repartitionExecutor.schedule(() -> repartitionTopicWithConsumerPerPartition(queue), 1, TimeUnit.SECONDS); - } - - void repartitionTopicWithConsumerPerPartition(final QueueKey queueKey) { - if (stopped) { - return; - } - TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.get(queueKey); - java.util.Queue> subscribeQueue = tbTopicWithConsumerPerPartition.getSubscribeQueue(); - if (subscribeQueue.isEmpty()) { - return; - } - if (tbTopicWithConsumerPerPartition.getLock().tryLock()) { - try { - Set partitions = null; - while (!subscribeQueue.isEmpty()) { - partitions = subscribeQueue.poll(); - } - if (partitions == null) { - return; - } - - Set addedPartitions = new HashSet<>(partitions); - ConcurrentMap>> consumers = tbTopicWithConsumerPerPartition.getConsumers(); - addedPartitions.removeAll(consumers.keySet()); - log.info("calculated addedPartitions {}", addedPartitions); - - Set removedPartitions = new HashSet<>(consumers.keySet()); - removedPartitions.removeAll(partitions); - log.info("calculated removedPartitions {}", removedPartitions); - - removedPartitions.forEach((tpi) -> { - removeConsumerForTopicByTpi(queueKey.getQueueName(), consumers, tpi); - }); - - addedPartitions.forEach((tpi) -> { - log.info("[{}] Adding consumer for topic: {}", queueKey, tpi); - Queue configuration = consumerConfigurations.get(queueKey); - TbQueueConsumer> consumer = tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration); - consumers.put(tpi, consumer); - launchConsumer(consumer, consumerConfigurations.get(queueKey), consumerStats.get(queueKey), "" + queueKey + "-" + tpi.getPartition().orElse(-999999)); - consumer.subscribe(Collections.singleton(tpi)); - }); - } finally { - tbTopicWithConsumerPerPartition.getLock().unlock(); + var consumer = consumerMap.get(event.getQueueKey()); + if (consumer != null) { + consumer.subscribe(event); + } else { + log.warn("Received invalid partition change event for {} that is not managed by this service", event.getQueueKey()); } - } else { - scheduleTopicRepartition(queueKey); //reschedule later } - } +// void subscribeConsumerPerPartition(QueueKey queue, Set partitions) { +// topicsConsumerPerPartition.get(queue).getSubscribeQueue().add(partitions); +// scheduleTopicRepartition(queue); +// } +// +// private void scheduleTopicRepartition(QueueKey queue) { +// repartitionExecutor.schedule(() -> repartitionTopicWithConsumerPerPartition(queue), 1, TimeUnit.SECONDS); +// } +// +// void repartitionTopicWithConsumerPerPartition(final QueueKey queueKey) { +// if (stopped) { +// return; +// } +// TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.get(queueKey); +// java.util.Queue> subscribeQueue = tbTopicWithConsumerPerPartition.getSubscribeQueue(); +// if (subscribeQueue.isEmpty()) { +// return; +// } +// if (tbTopicWithConsumerPerPartition.getLock().tryLock()) { +// try { +// Set partitions = null; +// while (!subscribeQueue.isEmpty()) { +// partitions = subscribeQueue.poll(); +// } +// if (partitions == null) { +// return; +// } +// +// Set addedPartitions = new HashSet<>(partitions); +// ConcurrentMap>> consumers = tbTopicWithConsumerPerPartition.getConsumers(); +// addedPartitions.removeAll(consumers.keySet()); +// log.info("calculated addedPartitions {}", addedPartitions); +// +// Set removedPartitions = new HashSet<>(consumers.keySet()); +// removedPartitions.removeAll(partitions); +// log.info("calculated removedPartitions {}", removedPartitions); +// +// removedPartitions.forEach((tpi) -> { +// removeConsumerForTopicByTpi(queueKey.getQueueName(), consumers, tpi); +// }); +// +// addedPartitions.forEach((tpi) -> { +// log.info("[{}] Adding consumer for topic: {}", queueKey, tpi); +// Queue configuration = consumerConfigurations.get(queueKey); +// TbQueueConsumer> consumer = tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration); +// consumers.put(tpi, consumer); +// launchConsumer(consumer, queueKey, tpi.getFullTopicName(), queueKey + "-" + tpi.getPartition().orElse(-999999)); +// consumer.subscribe(Collections.singleton(tpi)); +// }); +// } finally { +// tbTopicWithConsumerPerPartition.getLock().unlock(); +// } +// } else { +// scheduleTopicRepartition(queueKey); //reschedule later +// } +// } + void removeConsumerForTopicByTpi(String queue, ConcurrentMap>> consumers, TopicPartitionInfo tpi) { log.info("[{}] Removing consumer for topic: {}", queue, tpi); - consumers.get(tpi).unsubscribe(); - consumers.remove(tpi); + consumers.remove(tpi).stop(); } @Override protected void launchMainConsumers() { - consumers.forEach((queue, consumer) -> launchConsumer(consumer, consumerConfigurations.get(queue), consumerStats.get(queue), queue.getQueueName())); + consumers.forEach((queue, consumer) -> launchConsumer(consumer, queue, queue, queue.getQueueName())); } @Override protected void stopMainConsumers() { - consumers.values().forEach(TbQueueConsumer::unsubscribe); - topicsConsumerPerPartition.values().forEach(tbTopicWithConsumerPerPartition -> tbTopicWithConsumerPerPartition.getConsumers().keySet() - .forEach((tpi) -> removeConsumerForTopicByTpi(tbTopicWithConsumerPerPartition.getTopic(), tbTopicWithConsumerPerPartition.getConsumers(), tpi))); + consumerMap.values().forEach(TbRuleEngineQueueConsumerManager::stop); } - void launchConsumer(TbQueueConsumer> consumer, Queue configuration, TbRuleEngineConsumerStats stats, String threadSuffix) { + void launchConsumer(TbQueueConsumer> consumer, QueueKey queueKey, Object consumerKey, String threadSuffix) { if (isReady) { - consumersExecutor.execute(() -> consumerLoop(consumer, configuration, stats, threadSuffix)); + log.info("[{}] Launching consumer", consumerKey); + consumersExecutor.execute(consumerKey, () -> consumerLoop(consumer, queueKey, threadSuffix)); } else { - scheduleLaunchConsumer(consumer, configuration, stats, threadSuffix); + scheduleLaunchConsumer(consumer, queueKey, consumerKey, threadSuffix); } } - private void scheduleLaunchConsumer(TbQueueConsumer> consumer, Queue configuration, TbRuleEngineConsumerStats stats, String threadSuffix) { + private void scheduleLaunchConsumer(TbQueueConsumer> consumer, QueueKey queueKey, Object consumerKey, String threadSuffix) { repartitionExecutor.schedule(() -> { - if (isReady) { - consumersExecutor.execute(() -> consumerLoop(consumer, configuration, stats, threadSuffix)); - } else { - scheduleLaunchConsumer(consumer, configuration, stats, threadSuffix); - } + launchConsumer(consumer, queueKey, consumerKey, threadSuffix); }, 10, TimeUnit.SECONDS); } - void consumerLoop(TbQueueConsumer> consumer, org.thingsboard.server.common.data.queue.Queue configuration, TbRuleEngineConsumerStats stats, String threadSuffix) { + void consumerLoop(TbQueueConsumer> consumer, QueueKey queueKey, String threadSuffix) { + Queue configuration = consumerConfigurations.get(queueKey); + TbRuleEngineConsumerStats stats = consumerStats.get(queueKey); updateCurrentThreadName(threadSuffix); + while (!stopped && !consumer.isStopped() && !consumer.isQueueDeleted()) { try { List> msgs = consumer.poll(configuration.getPollInterval()); @@ -347,7 +331,9 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } } - if (consumer.isQueueDeleted()) { + if (consumer.isStopped()) { + consumer.unsubscribe(); + } else if (consumer.isQueueDeleted()) { processQueueDeletion(configuration, consumer); } log.info("TB Rule Engine Consumer stopped."); @@ -460,20 +446,20 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< ReentrantLock lock = consumerPerPartition.getLock(); try { lock.lock(); - consumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::unsubscribe); + consumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::stop); } finally { lock.unlock(); } } else { TbQueueConsumer> consumer = consumers.remove(queueKey); - consumer.unsubscribe(); + consumer.stop(); } } initConsumer(queue); if (!queue.isConsumerPerPartition()) { - launchConsumer(consumers.get(queueKey), consumerConfigurations.get(queueKey), consumerStats.get(queueKey), queueName); + launchConsumer(consumers.get(queueKey), queueKey, queueKey, queueName); } } @@ -502,6 +488,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } } } + partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), new ArrayList<>(partitionService.getOtherServices(ServiceType.TB_RULE_ENGINE))); } private void forwardToRuleEngineActor(String queueName, TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerLauncher.java b/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerLauncher.java new file mode 100644 index 0000000000..f5688eb207 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerLauncher.java @@ -0,0 +1,37 @@ +package org.thingsboard.server.service.queue; + +import lombok.Data; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.QueueKey; + +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +@Data +public class TbQueueConsumerLauncher { + + private final TbQueueConsumer> consumer; + private volatile Future task; + + public void stop() { + this.consumer.stop(); + } + + public void awaitStopped() throws ExecutionException, InterruptedException, TimeoutException { + if (task != null) { + this.task.get(3, TimeUnit.MINUTES); + } + } + + public void subscribe(Set partitions) { + this.consumer.subscribe(partitions); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerManagerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerManagerTask.java new file mode 100644 index 0000000000..11316a9e05 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerManagerTask.java @@ -0,0 +1,17 @@ +package org.thingsboard.server.service.queue; + +import lombok.Data; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; + +import java.util.Set; + +@Data +public class TbQueueConsumerManagerTask { + + private final ComponentLifecycleEvent event; + private final Queue queue; + private final Set partitions; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index 2904c299ce..45ccbcf32a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsType; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; import java.util.ArrayList; @@ -66,8 +67,8 @@ public class TbRuleEngineConsumerStats { private final String queueName; private final TenantId tenantId; - public TbRuleEngineConsumerStats(Queue queue, StatsFactory statsFactory) { - this.queueName = queue.getName(); + public TbRuleEngineConsumerStats(QueueKey queue, StatsFactory statsFactory) { + this.queueName = queue.getQueueName(); this.tenantId = queue.getTenantId(); this.statsFactory = statsFactory; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineQueueConsumerManager.java new file mode 100644 index 0000000000..2c6d191c0a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineQueueConsumerManager.java @@ -0,0 +1,205 @@ +package org.thingsboard.server.service.queue; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +@Data +@Slf4j +public class TbRuleEngineQueueConsumerManager { + + private final ScheduledExecutorService scheduler; + private final ExecutorService consumerExecutor; + private final StatsFactory statsFactory; + private final TbRuleEngineQueueFactory queueFactory; + private final QueueKey key; + private final ReentrantLock lock = new ReentrantLock(); //NonfairSync + private final ConcurrentMap consumers = new ConcurrentHashMap<>(); + private final TbRuleEngineConsumerStats stats; + + private volatile Set partitions = Collections.emptySet(); + private volatile Queue queue; + private volatile TbQueueConsumerLauncher mainConsumer; + + private final java.util.Queue tasks = new ConcurrentLinkedQueue<>(); + + public TbRuleEngineQueueConsumerManager(ScheduledExecutorService scheduler, ExecutorService consumerExecutor, StatsFactory statsFactory, TbRuleEngineQueueFactory queueFactory, QueueKey key) { + this.scheduler = scheduler; + this.consumerExecutor = consumerExecutor; + this.statsFactory = statsFactory; + this.queueFactory = queueFactory; + this.key = key; + this.stats = new TbRuleEngineConsumerStats(key, statsFactory); + } + + public void init(Queue queue) { + processTask(new TbQueueConsumerManagerTask(ComponentLifecycleEvent.CREATED, queue, null)); + } + + private void processTask(TbQueueConsumerManagerTask todo) { + tasks.add(todo); + log.info("[{}] Adding task: {}", key, todo); + tryProcessTasks(); + } + + private void tryProcessTasks() { + consumerExecutor.submit(() -> { + if (lock.tryLock()) { + try { + TbQueueConsumerManagerTask lastUpdateTask = null; + while (!tasks.isEmpty()) { + TbQueueConsumerManagerTask task = tasks.poll(); + switch (task.getEvent()) { + case CREATED: + doInit(task.getQueue()); + break; + case UPDATED: + lastUpdateTask = task; + break; + case DELETED: + lastUpdateTask = null; + doDelete(); + break; + } + } + if (lastUpdateTask != null) { + doUpdate(lastUpdateTask.getQueue(), lastUpdateTask.getPartitions()); + } + } finally { + lock.unlock(); + } + } else { + log.debug("[{}] Failed to acquire lock.", key); + scheduler.schedule(this::tryProcessTasks, 1, TimeUnit.SECONDS); + } + }); + } + + public void doInit(Queue queue) { + log.info("[{}] Init consumer with queue: {}", key, queue); + this.queue = queue; + if (!queue.isConsumerPerPartition()) { + mainConsumer = new TbQueueConsumerLauncher(queueFactory.createToRuleEngineMsgConsumer(queue)); + } + } + + private void doUpdate(Queue newQueue, Set partitions) { + if (newQueue.isConsumerPerPartition()) { + + } else { + for (var oldConsumer : consumers.values()) { + oldConsumer.stop(); + } + for (var oldConsumer : consumers.entrySet()) { + try { + oldConsumer.getValue().awaitStopped(); + } catch (Exception e) { + log.info("[{}][{}] Failed to stop the consumer during update", key, oldConsumer.getKey().getPartition().orElse(-1), e); + } + } + if (mainConsumer == null) { + mainConsumer = new TbQueueConsumerLauncher(queueFactory.createToRuleEngineMsgConsumer(queue)); + //TODO: launch + } + mainConsumer.subscribe(partitions); + + } + } + + private void doDelete() { + } + + public void subscribe(PartitionChangeEvent event) { + log.info("[{}] Subscribing to partitions: {}", key, event.getPartitions()); + if (!queue.isConsumerPerPartition()) { + mainConsumer.subscribe(event.getPartitions()); + } else { + log.info("[{}] Subscribing consumer per partition: {}", key, event.getPartitions()); + subscribeConsumerPerPartition(event.getQueueKey(), event.getPartitions()); + } + } + + void subscribeConsumerPerPartition(QueueKey queue, Set partitions) { + topicsConsumerPerPartition.get(queue).getSubscribeQueue().add(partitions); + scheduleTopicRepartition(queue); + } + + private void scheduleTopicRepartition(QueueKey queue) { + repartitionExecutor.schedule(() -> repartitionTopicWithConsumerPerPartition(queue), 1, TimeUnit.SECONDS); + } + + void repartitionTopicWithConsumerPerPartition(final QueueKey queueKey) { + if (stopped) { + return; + } + TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.get(queueKey); + java.util.Queue> subscribeQueue = tbTopicWithConsumerPerPartition.getSubscribeQueue(); + if (subscribeQueue.isEmpty()) { + return; + } + if (tbTopicWithConsumerPerPartition.getLock().tryLock()) { + try { + Set partitions = null; + while (!subscribeQueue.isEmpty()) { + partitions = subscribeQueue.poll(); + } + if (partitions == null) { + return; + } + + Set addedPartitions = new HashSet<>(partitions); + ConcurrentMap>> consumers = tbTopicWithConsumerPerPartition.getConsumers(); + addedPartitions.removeAll(consumers.keySet()); + log.info("calculated addedPartitions {}", addedPartitions); + + Set removedPartitions = new HashSet<>(consumers.keySet()); + removedPartitions.removeAll(partitions); + log.info("calculated removedPartitions {}", removedPartitions); + + removedPartitions.forEach((tpi) -> { + removeConsumerForTopicByTpi(queueKey.getQueueName(), consumers, tpi); + }); + + addedPartitions.forEach((tpi) -> { + log.info("[{}] Adding consumer for topic: {}", queueKey, tpi); + Queue configuration = consumerConfigurations.get(queueKey); + TbQueueConsumer> consumer = tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration); + consumers.put(tpi, consumer); + launchConsumer(consumer, queueKey, tpi.getFullTopicName(), queueKey + "-" + tpi.getPartition().orElse(-999999)); + consumer.subscribe(Collections.singleton(tpi)); + }); + } finally { + tbTopicWithConsumerPerPartition.getLock().unlock(); + } + } else { + scheduleTopicRepartition(queueKey); //reschedule later + } + } + + public void stop() { +// consumers.values().forEach(TbQueueConsumer::stop); +// topicsConsumerPerPartition.values().forEach(tbTopicWithConsumerPerPartition -> tbTopicWithConsumerPerPartition.getConsumers().keySet() +// .forEach((tpi) -> removeConsumerForTopicByTpi(tbTopicWithConsumerPerPartition.getTopic(), tbTopicWithConsumerPerPartition.getConsumers(), tpi))); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 386447045d..bb1943a7d5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -82,7 +82,6 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory * @return * @param configuration */ - //TODO 2.5 ybondarenko: make sure you use queueName to distinct consumers where necessary TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration); /** From 3a0afd8a5ab9e5ed518da6bc1f63f909695875c7 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 2 Oct 2023 18:51:03 +0300 Subject: [PATCH 07/23] Refactoring of the consumer service --- .../DefaultTbRuleEngineConsumerService.java | 426 ++--------------- .../queue/TbQueueConsumerLauncher.java | 37 -- .../queue/TbQueueConsumerManagerTask.java | 17 - .../TbRuleEngineQueueConsumerManager.java | 205 --------- .../service/queue/ruleengine/QueueEvent.java | 9 + .../TbQueueConsumerManagerTask.java | 32 ++ .../queue/ruleengine/TbQueueConsumerTask.java | 70 +++ .../TbRuleEngineConsumerContext.java | 75 +++ .../TbRuleEngineQueueConsumerManager.java | 431 ++++++++++++++++++ .../common/util/ThingsBoardThreadFactory.java | 12 + 10 files changed, 659 insertions(+), 655 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerLauncher.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerManagerTask.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineQueueConsumerManager.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index ae04de456b..b37e5abff3 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -15,44 +15,25 @@ */ package org.thingsboard.server.service.queue; -import com.google.protobuf.ProtocolStringList; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.rpc.RpcError; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.gen.MsgProtos; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; -import org.thingsboard.server.common.msg.queue.RuleEngineException; -import org.thingsboard.server.common.msg.queue.RuleNodeInfo; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; -import org.thingsboard.server.common.stats.StatsFactory; -import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.QueueKey; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.TbRuleEngineComponent; @@ -60,102 +41,48 @@ import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; -import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; -import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; -import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; -import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; -import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; -import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; +import org.thingsboard.server.service.queue.ruleengine.TbRuleEngineConsumerContext; +import org.thingsboard.server.service.queue.ruleengine.TbRuleEngineQueueConsumerManager; import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; -import org.thingsboard.server.service.stats.RuleEngineStatisticsService; -import org.threadly.concurrent.wrapper.KeyDistributedExecutor; import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; @Service @TbRuleEngineComponent @Slf4j public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService implements TbRuleEngineConsumerService { - public static final String SUCCESSFUL_STATUS = "successful"; - public static final String FAILED_STATUS = "failed"; - public static final String THREAD_TOPIC_SEPARATOR = " | "; - @Value("${queue.rule-engine.poll-interval}") - private long pollDuration; - @Value("${queue.rule-engine.pack-processing-timeout}") - private long packProcessingTimeout; - @Value("${queue.rule-engine.stats.enabled:true}") - private boolean statsEnabled; - @Value("${queue.rule-engine.prometheus-stats.enabled:false}") - boolean prometheusStatsEnabled; - @Value("${queue.rule-engine.topic-deletion-delay:30}") - private int topicDeletionDelayInSec; - - private final StatsFactory statsFactory; - private final TbRuleEngineSubmitStrategyFactory submitStrategyFactory; - private final TbRuleEngineProcessingStrategyFactory processingStrategyFactory; - private final TbRuleEngineQueueFactory tbRuleEngineQueueFactory; - private final RuleEngineStatisticsService statisticsService; + private final TbRuleEngineConsumerContext ctx; private final TbRuleEngineDeviceRpcService tbDeviceRpcService; - private final TbServiceInfoProvider serviceInfoProvider; - private final QueueService queueService; - private final TbQueueProducerProvider producerProvider; - private final TbQueueAdmin queueAdmin; - - private final ConcurrentMap consumerMap = new ConcurrentHashMap<>(); - private final ExecutorService consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer")); - private final ExecutorService submitExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-submit")); - private final ScheduledExecutorService repartitionExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-repartition")); + private final ConcurrentMap consumers = new ConcurrentHashMap<>(); - public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory, - TbRuleEngineSubmitStrategyFactory submitStrategyFactory, + public DefaultTbRuleEngineConsumerService(TbRuleEngineConsumerContext ctx, TbRuleEngineQueueFactory tbRuleEngineQueueFactory, - RuleEngineStatisticsService statisticsService, ActorSystemContext actorContext, DataDecodingEncodingService encodingService, TbRuleEngineDeviceRpcService tbDeviceRpcService, - StatsFactory statsFactory, TbDeviceProfileCache deviceProfileCache, TbAssetProfileCache assetProfileCache, TbTenantProfileCache tenantProfileCache, TbApiUsageStateService apiUsageStateService, - PartitionService partitionService, ApplicationEventPublisher eventPublisher, - TbServiceInfoProvider serviceInfoProvider, QueueService queueService, - TbQueueProducerProvider producerProvider, TbQueueAdmin queueAdmin) { - super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, eventPublisher, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer(), Optional.empty()); - this.statisticsService = statisticsService; - this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; - this.submitStrategyFactory = submitStrategyFactory; - this.processingStrategyFactory = processingStrategyFactory; + PartitionService partitionService, ApplicationEventPublisher eventPublisher) { + super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, + eventPublisher, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer(), Optional.empty()); + this.ctx = ctx; this.tbDeviceRpcService = tbDeviceRpcService; - this.statsFactory = statsFactory; - this.serviceInfoProvider = serviceInfoProvider; - this.queueService = queueService; - this.producerProvider = producerProvider; - this.queueAdmin = queueAdmin; } @PostConstruct public void init() { super.init("tb-rule-engine-notifications-consumer"); // TODO: restore init of the main consumer? - List queues = queueService.findAllQueues(); + List queues = ctx.findAllQueues(); for (Queue configuration : queues) { if (partitionService.isManagedByCurrentService(configuration.getTenantId())) { initConsumer(configuration); @@ -164,22 +91,14 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } private void initConsumer(Queue configuration) { - consumerMap.computeIfAbsent(new QueueKey(ServiceType.TB_RULE_ENGINE, configuration), - key -> new TbRuleEngineQueueConsumerManager(repartitionExecutor, consumersExecutor, statsFactory, tbRuleEngineQueueFactory, key)).init(configuration); - } - - @PreDestroy - public void stop() { - super.destroy(); - consumersExecutor.shutdownNow(); // TODO: shutdown or shutdownNow? - submitExecutor.shutdownNow(); - repartitionExecutor.shutdownNow(); + consumers.computeIfAbsent(new QueueKey(ServiceType.TB_RULE_ENGINE, configuration), + key -> new TbRuleEngineQueueConsumerManager(ctx, key)).init(configuration); } @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { if (event.getServiceType().equals(getServiceType())) { - var consumer = consumerMap.get(event.getQueueKey()); + var consumer = consumers.get(event.getQueueKey()); if (consumer != null) { consumer.subscribe(event); } else { @@ -188,207 +107,14 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } } -// void subscribeConsumerPerPartition(QueueKey queue, Set partitions) { -// topicsConsumerPerPartition.get(queue).getSubscribeQueue().add(partitions); -// scheduleTopicRepartition(queue); -// } -// -// private void scheduleTopicRepartition(QueueKey queue) { -// repartitionExecutor.schedule(() -> repartitionTopicWithConsumerPerPartition(queue), 1, TimeUnit.SECONDS); -// } -// -// void repartitionTopicWithConsumerPerPartition(final QueueKey queueKey) { -// if (stopped) { -// return; -// } -// TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.get(queueKey); -// java.util.Queue> subscribeQueue = tbTopicWithConsumerPerPartition.getSubscribeQueue(); -// if (subscribeQueue.isEmpty()) { -// return; -// } -// if (tbTopicWithConsumerPerPartition.getLock().tryLock()) { -// try { -// Set partitions = null; -// while (!subscribeQueue.isEmpty()) { -// partitions = subscribeQueue.poll(); -// } -// if (partitions == null) { -// return; -// } -// -// Set addedPartitions = new HashSet<>(partitions); -// ConcurrentMap>> consumers = tbTopicWithConsumerPerPartition.getConsumers(); -// addedPartitions.removeAll(consumers.keySet()); -// log.info("calculated addedPartitions {}", addedPartitions); -// -// Set removedPartitions = new HashSet<>(consumers.keySet()); -// removedPartitions.removeAll(partitions); -// log.info("calculated removedPartitions {}", removedPartitions); -// -// removedPartitions.forEach((tpi) -> { -// removeConsumerForTopicByTpi(queueKey.getQueueName(), consumers, tpi); -// }); -// -// addedPartitions.forEach((tpi) -> { -// log.info("[{}] Adding consumer for topic: {}", queueKey, tpi); -// Queue configuration = consumerConfigurations.get(queueKey); -// TbQueueConsumer> consumer = tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration); -// consumers.put(tpi, consumer); -// launchConsumer(consumer, queueKey, tpi.getFullTopicName(), queueKey + "-" + tpi.getPartition().orElse(-999999)); -// consumer.subscribe(Collections.singleton(tpi)); -// }); -// } finally { -// tbTopicWithConsumerPerPartition.getLock().unlock(); -// } -// } else { -// scheduleTopicRepartition(queueKey); //reschedule later -// } -// } - - void removeConsumerForTopicByTpi(String queue, ConcurrentMap>> consumers, TopicPartitionInfo tpi) { - log.info("[{}] Removing consumer for topic: {}", queue, tpi); - consumers.remove(tpi).stop(); - } - @Override protected void launchMainConsumers() { - consumers.forEach((queue, consumer) -> launchConsumer(consumer, queue, queue, queue.getQueueName())); + consumers.values().forEach(TbRuleEngineQueueConsumerManager::launchMainConsumer); } @Override protected void stopMainConsumers() { - consumerMap.values().forEach(TbRuleEngineQueueConsumerManager::stop); - } - - void launchConsumer(TbQueueConsumer> consumer, QueueKey queueKey, Object consumerKey, String threadSuffix) { - if (isReady) { - log.info("[{}] Launching consumer", consumerKey); - consumersExecutor.execute(consumerKey, () -> consumerLoop(consumer, queueKey, threadSuffix)); - } else { - scheduleLaunchConsumer(consumer, queueKey, consumerKey, threadSuffix); - } - } - - private void scheduleLaunchConsumer(TbQueueConsumer> consumer, QueueKey queueKey, Object consumerKey, String threadSuffix) { - repartitionExecutor.schedule(() -> { - launchConsumer(consumer, queueKey, consumerKey, threadSuffix); - }, 10, TimeUnit.SECONDS); - } - - void consumerLoop(TbQueueConsumer> consumer, QueueKey queueKey, String threadSuffix) { - Queue configuration = consumerConfigurations.get(queueKey); - TbRuleEngineConsumerStats stats = consumerStats.get(queueKey); - updateCurrentThreadName(threadSuffix); - - while (!stopped && !consumer.isStopped() && !consumer.isQueueDeleted()) { - try { - List> msgs = consumer.poll(configuration.getPollInterval()); - if (msgs.isEmpty()) { - continue; - } - final TbRuleEngineSubmitStrategy submitStrategy = getSubmitStrategy(configuration); - final TbRuleEngineProcessingStrategy ackStrategy = getAckStrategy(configuration); - submitStrategy.init(msgs); - while (!stopped && !consumer.isStopped()) { - TbMsgPackProcessingContext ctx = new TbMsgPackProcessingContext(configuration.getName(), submitStrategy, ackStrategy.isSkipTimeoutMsgs()); - submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> submitMessage(configuration, stats, ctx, id, msg))); - - final boolean timeout = !ctx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS); - - TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(configuration.getName(), timeout, ctx); - if (timeout) { - printFirstOrAll(configuration, ctx, ctx.getPendingMap(), "Timeout"); - } - if (!ctx.getFailedMap().isEmpty()) { - printFirstOrAll(configuration, ctx, ctx.getFailedMap(), "Failed"); - } - ctx.printProfilerStats(); - - TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result); - if (statsEnabled) { - stats.log(result, decision.isCommit()); - } - - ctx.cleanup(); - - if (decision.isCommit()) { - submitStrategy.stop(); - break; - } else { - submitStrategy.update(decision.getReprocessMap()); - } - } - consumer.commit(); - } catch (Exception e) { - if (!stopped) { - log.warn("Failed to process messages from queue.", e); - try { - Thread.sleep(pollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - } - - if (consumer.isStopped()) { - consumer.unsubscribe(); - } else if (consumer.isQueueDeleted()) { - processQueueDeletion(configuration, consumer); - } - log.info("TB Rule Engine Consumer stopped."); - } - - void updateCurrentThreadName(String threadSuffix) { - String name = Thread.currentThread().getName(); - int spliteratorIndex = name.indexOf(THREAD_TOPIC_SEPARATOR); - if (spliteratorIndex > 0) { - name = name.substring(0, spliteratorIndex); - } - name = name + THREAD_TOPIC_SEPARATOR + threadSuffix; - Thread.currentThread().setName(name); - } - - TbRuleEngineProcessingStrategy getAckStrategy(Queue configuration) { - return processingStrategyFactory.newInstance(configuration.getName(), configuration.getProcessingStrategy()); - } - - TbRuleEngineSubmitStrategy getSubmitStrategy(Queue configuration) { - return submitStrategyFactory.newInstance(configuration.getName(), configuration.getSubmitStrategy()); - } - - void submitMessage(Queue configuration, TbRuleEngineConsumerStats stats, TbMsgPackProcessingContext ctx, UUID id, TbProtoQueueMsg msg) { - log.trace("[{}] Creating callback for topic {} message: {}", id, configuration.getName(), msg.getValue()); - ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); - TenantId tenantId = TenantId.fromUUID(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); - TbMsgCallback callback = prometheusStatsEnabled ? - new TbMsgPackCallback(id, tenantId, ctx, stats.getTimer(tenantId, SUCCESSFUL_STATUS), stats.getTimer(tenantId, FAILED_STATUS)) : - new TbMsgPackCallback(id, tenantId, ctx); - try { - if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { - forwardToRuleEngineActor(configuration.getName(), tenantId, toRuleEngineMsg, callback); - } else { - callback.onSuccess(); - } - } catch (Exception e) { - callback.onFailure(new RuleEngineException(e.getMessage(), e)); - } - } - - private void printFirstOrAll(Queue configuration, TbMsgPackProcessingContext ctx, Map> map, String prefix) { - boolean printAll = log.isTraceEnabled(); - log.info("{} to process [{}] messages", prefix, map.size()); - for (Map.Entry> pending : map.entrySet()) { - ToRuleEngineMsg tmp = pending.getValue().getValue(); - TbMsg tmpMsg = TbMsg.fromBytes(configuration.getName(), tmp.getTbMsg().toByteArray(), TbMsgCallback.EMPTY); - RuleNodeInfo ruleNodeInfo = ctx.getLastVisitedRuleNode(pending.getKey()); - if (printAll) { - log.trace("[{}] {} to process message: {}, Last Rule Node: {}", TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); - } else { - log.info("[{}] {} to process message: {}, Last Rule Node: {}", TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); - break; - } - } + consumers.values().forEach(TbRuleEngineQueueConsumerManager::stop); } @Override @@ -398,18 +124,18 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @Override protected long getNotificationPollDuration() { - return pollDuration; + return ctx.getPollDuration(); } @Override protected long getNotificationPackProcessingTimeout() { - return packProcessingTimeout; + return ctx.getPackProcessingTimeout(); } @Override protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) throws Exception { ToRuleEngineNotificationMsg nfMsg = msg.getValue(); - if (nfMsg.getComponentLifecycleMsg() != null && !nfMsg.getComponentLifecycleMsg().isEmpty()) { + if (!nfMsg.getComponentLifecycleMsg().isEmpty()) { handleComponentLifecycleMsg(id, nfMsg.getComponentLifecycleMsg()); callback.onSuccess(); } else if (nfMsg.hasFromDeviceRpcResponse()) { @@ -420,10 +146,10 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< tbDeviceRpcService.processRpcResponseFromDevice(response); callback.onSuccess(); } else if (nfMsg.hasQueueUpdateMsg()) { - repartitionExecutor.execute(() -> updateQueue(nfMsg.getQueueUpdateMsg())); + ctx.getScheduler().execute(() -> updateQueue(nfMsg.getQueueUpdateMsg())); callback.onSuccess(); } else if (nfMsg.hasQueueDeleteMsg()) { - repartitionExecutor.execute(() -> deleteQueue(nfMsg.getQueueDeleteMsg())); + ctx.getScheduler().execute(() -> deleteQueue(nfMsg.getQueueDeleteMsg())); callback.onSuccess(); } else { log.trace("Received notification with missing handler"); @@ -438,33 +164,13 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< QueueId queueId = new QueueId(new UUID(queueUpdateMsg.getQueueIdMSB(), queueUpdateMsg.getQueueIdLSB())); String queueName = queueUpdateMsg.getQueueName(); QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueName, tenantId); - Queue queue = queueService.findQueueById(tenantId, queueId); - Queue oldQueue = consumerConfigurations.remove(queueKey); - if (oldQueue != null) { - if (oldQueue.isConsumerPerPartition()) { - TbTopicWithConsumerPerPartition consumerPerPartition = topicsConsumerPerPartition.remove(queueKey); - ReentrantLock lock = consumerPerPartition.getLock(); - try { - lock.lock(); - consumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::stop); - } finally { - lock.unlock(); - } - } else { - TbQueueConsumer> consumer = consumers.remove(queueKey); - consumer.stop(); - } - } - - initConsumer(queue); - - if (!queue.isConsumerPerPartition()) { - launchConsumer(consumers.get(queueKey), queueKey, queueKey, queueName); - } + Queue queue = ctx.getQueueService().findQueueById(tenantId, queueId); + consumers.computeIfAbsent(queueKey, key -> new TbRuleEngineQueueConsumerManager(ctx, key)).update(queue); } partitionService.updateQueue(queueUpdateMsg); - partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), new ArrayList<>(partitionService.getOtherServices(ServiceType.TB_RULE_ENGINE))); + partitionService.recalculatePartitions(ctx.getServiceInfoProvider().getServiceInfo(), + new ArrayList<>(partitionService.getOtherServices(ServiceType.TB_RULE_ENGINE))); } private void deleteQueue(TransportProtos.QueueDeleteMsg queueDeleteMsg) { @@ -473,90 +179,18 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueDeleteMsg.getQueueName(), tenantId); partitionService.removeQueue(queueDeleteMsg); - Queue queue = consumerConfigurations.remove(queueKey); - if (queue != null) { - if (queue.isConsumerPerPartition()) { - TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.remove(queueKey); - if (tbTopicWithConsumerPerPartition != null) { - tbTopicWithConsumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::onQueueDelete); - tbTopicWithConsumerPerPartition.getConsumers().clear(); - } - } else { - TbQueueConsumer> consumer = consumers.remove(queueKey); - if (consumer != null) { - consumer.onQueueDelete(); - } - } - } - partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), new ArrayList<>(partitionService.getOtherServices(ServiceType.TB_RULE_ENGINE))); - } - - private void forwardToRuleEngineActor(String queueName, TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { - TbMsg tbMsg = TbMsg.fromBytes(queueName, toRuleEngineMsg.getTbMsg().toByteArray(), callback); - QueueToRuleEngineMsg msg; - ProtocolStringList relationTypesList = toRuleEngineMsg.getRelationTypesList(); - Set relationTypes = null; - if (relationTypesList != null) { - if (relationTypesList.size() == 1) { - relationTypes = Collections.singleton(relationTypesList.get(0)); - } else { - relationTypes = new HashSet<>(relationTypesList); - } - } - msg = new QueueToRuleEngineMsg(tenantId, tbMsg, relationTypes, toRuleEngineMsg.getFailureMessage()); - actorContext.tell(msg); - } - - private void processQueueDeletion(Queue queue, TbQueueConsumer> consumer) { - long finishTs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(topicDeletionDelayInSec); - try { - int n = 0; - while (System.currentTimeMillis() <= finishTs) { - List> msgs = consumer.poll(queue.getPollInterval()); - if (msgs.isEmpty()) { - continue; - } - for (TbProtoQueueMsg msg : msgs) { - try { - MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray()); - EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB())); - - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue.getName(), TenantId.SYS_TENANT_ID, originator); - producerProvider.getRuleEngineMsgProducer().send(tpi, msg, null); - n++; - } catch (Throwable e) { - log.debug("Failed to move message to system {}: {}", consumer.getTopic(), msg, e); - } - } - consumer.commit(); - } - if (n > 0) { - log.info("Moved {} messages from {} to system {}", n, consumer.getFullTopicNames(), consumer.getTopic()); - } - - consumer.unsubscribe(); - for (String topic : consumer.getFullTopicNames()) { - try { - queueAdmin.deleteTopic(topic); - log.info("Deleted topic {}", topic); - } catch (Exception e) { - log.error("Failed to delete topic {} after unsubscribing", topic, e); - } - } - } catch (Exception e) { - log.error("Failed to process deletion of {} ({})", consumer.getTopic(), queue.getTenantId(), e); + var manager = consumers.remove(queueKey); + if (manager != null) { + manager.delete(); } + partitionService.recalculatePartitions(ctx.getServiceInfoProvider().getServiceInfo(), new ArrayList<>(partitionService.getOtherServices(ServiceType.TB_RULE_ENGINE))); } @Scheduled(fixedDelayString = "${queue.rule-engine.stats.print-interval-ms}") public void printStats() { - if (statsEnabled) { + if (ctx.isStatsEnabled()) { long ts = System.currentTimeMillis(); - consumerStats.forEach((queue, stats) -> { - stats.printStats(); - statisticsService.reportQueueStats(ts, stats); - stats.reset(); - }); + consumers.values().forEach(manager -> manager.printStats(ts)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerLauncher.java b/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerLauncher.java deleted file mode 100644 index f5688eb207..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerLauncher.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.thingsboard.server.service.queue; - -import lombok.Data; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.QueueKey; - -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -@Data -public class TbQueueConsumerLauncher { - - private final TbQueueConsumer> consumer; - private volatile Future task; - - public void stop() { - this.consumer.stop(); - } - - public void awaitStopped() throws ExecutionException, InterruptedException, TimeoutException { - if (task != null) { - this.task.get(3, TimeUnit.MINUTES); - } - } - - public void subscribe(Set partitions) { - this.consumer.subscribe(partitions); - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerManagerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerManagerTask.java deleted file mode 100644 index 11316a9e05..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbQueueConsumerManagerTask.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.thingsboard.server.service.queue; - -import lombok.Data; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; - -import java.util.Set; - -@Data -public class TbQueueConsumerManagerTask { - - private final ComponentLifecycleEvent event; - private final Queue queue; - private final Set partitions; - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineQueueConsumerManager.java deleted file mode 100644 index 2c6d191c0a..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineQueueConsumerManager.java +++ /dev/null @@ -1,205 +0,0 @@ -package org.thingsboard.server.service.queue; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.common.stats.StatsFactory; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.QueueKey; -import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; -import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; - -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; - -@Data -@Slf4j -public class TbRuleEngineQueueConsumerManager { - - private final ScheduledExecutorService scheduler; - private final ExecutorService consumerExecutor; - private final StatsFactory statsFactory; - private final TbRuleEngineQueueFactory queueFactory; - private final QueueKey key; - private final ReentrantLock lock = new ReentrantLock(); //NonfairSync - private final ConcurrentMap consumers = new ConcurrentHashMap<>(); - private final TbRuleEngineConsumerStats stats; - - private volatile Set partitions = Collections.emptySet(); - private volatile Queue queue; - private volatile TbQueueConsumerLauncher mainConsumer; - - private final java.util.Queue tasks = new ConcurrentLinkedQueue<>(); - - public TbRuleEngineQueueConsumerManager(ScheduledExecutorService scheduler, ExecutorService consumerExecutor, StatsFactory statsFactory, TbRuleEngineQueueFactory queueFactory, QueueKey key) { - this.scheduler = scheduler; - this.consumerExecutor = consumerExecutor; - this.statsFactory = statsFactory; - this.queueFactory = queueFactory; - this.key = key; - this.stats = new TbRuleEngineConsumerStats(key, statsFactory); - } - - public void init(Queue queue) { - processTask(new TbQueueConsumerManagerTask(ComponentLifecycleEvent.CREATED, queue, null)); - } - - private void processTask(TbQueueConsumerManagerTask todo) { - tasks.add(todo); - log.info("[{}] Adding task: {}", key, todo); - tryProcessTasks(); - } - - private void tryProcessTasks() { - consumerExecutor.submit(() -> { - if (lock.tryLock()) { - try { - TbQueueConsumerManagerTask lastUpdateTask = null; - while (!tasks.isEmpty()) { - TbQueueConsumerManagerTask task = tasks.poll(); - switch (task.getEvent()) { - case CREATED: - doInit(task.getQueue()); - break; - case UPDATED: - lastUpdateTask = task; - break; - case DELETED: - lastUpdateTask = null; - doDelete(); - break; - } - } - if (lastUpdateTask != null) { - doUpdate(lastUpdateTask.getQueue(), lastUpdateTask.getPartitions()); - } - } finally { - lock.unlock(); - } - } else { - log.debug("[{}] Failed to acquire lock.", key); - scheduler.schedule(this::tryProcessTasks, 1, TimeUnit.SECONDS); - } - }); - } - - public void doInit(Queue queue) { - log.info("[{}] Init consumer with queue: {}", key, queue); - this.queue = queue; - if (!queue.isConsumerPerPartition()) { - mainConsumer = new TbQueueConsumerLauncher(queueFactory.createToRuleEngineMsgConsumer(queue)); - } - } - - private void doUpdate(Queue newQueue, Set partitions) { - if (newQueue.isConsumerPerPartition()) { - - } else { - for (var oldConsumer : consumers.values()) { - oldConsumer.stop(); - } - for (var oldConsumer : consumers.entrySet()) { - try { - oldConsumer.getValue().awaitStopped(); - } catch (Exception e) { - log.info("[{}][{}] Failed to stop the consumer during update", key, oldConsumer.getKey().getPartition().orElse(-1), e); - } - } - if (mainConsumer == null) { - mainConsumer = new TbQueueConsumerLauncher(queueFactory.createToRuleEngineMsgConsumer(queue)); - //TODO: launch - } - mainConsumer.subscribe(partitions); - - } - } - - private void doDelete() { - } - - public void subscribe(PartitionChangeEvent event) { - log.info("[{}] Subscribing to partitions: {}", key, event.getPartitions()); - if (!queue.isConsumerPerPartition()) { - mainConsumer.subscribe(event.getPartitions()); - } else { - log.info("[{}] Subscribing consumer per partition: {}", key, event.getPartitions()); - subscribeConsumerPerPartition(event.getQueueKey(), event.getPartitions()); - } - } - - void subscribeConsumerPerPartition(QueueKey queue, Set partitions) { - topicsConsumerPerPartition.get(queue).getSubscribeQueue().add(partitions); - scheduleTopicRepartition(queue); - } - - private void scheduleTopicRepartition(QueueKey queue) { - repartitionExecutor.schedule(() -> repartitionTopicWithConsumerPerPartition(queue), 1, TimeUnit.SECONDS); - } - - void repartitionTopicWithConsumerPerPartition(final QueueKey queueKey) { - if (stopped) { - return; - } - TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.get(queueKey); - java.util.Queue> subscribeQueue = tbTopicWithConsumerPerPartition.getSubscribeQueue(); - if (subscribeQueue.isEmpty()) { - return; - } - if (tbTopicWithConsumerPerPartition.getLock().tryLock()) { - try { - Set partitions = null; - while (!subscribeQueue.isEmpty()) { - partitions = subscribeQueue.poll(); - } - if (partitions == null) { - return; - } - - Set addedPartitions = new HashSet<>(partitions); - ConcurrentMap>> consumers = tbTopicWithConsumerPerPartition.getConsumers(); - addedPartitions.removeAll(consumers.keySet()); - log.info("calculated addedPartitions {}", addedPartitions); - - Set removedPartitions = new HashSet<>(consumers.keySet()); - removedPartitions.removeAll(partitions); - log.info("calculated removedPartitions {}", removedPartitions); - - removedPartitions.forEach((tpi) -> { - removeConsumerForTopicByTpi(queueKey.getQueueName(), consumers, tpi); - }); - - addedPartitions.forEach((tpi) -> { - log.info("[{}] Adding consumer for topic: {}", queueKey, tpi); - Queue configuration = consumerConfigurations.get(queueKey); - TbQueueConsumer> consumer = tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration); - consumers.put(tpi, consumer); - launchConsumer(consumer, queueKey, tpi.getFullTopicName(), queueKey + "-" + tpi.getPartition().orElse(-999999)); - consumer.subscribe(Collections.singleton(tpi)); - }); - } finally { - tbTopicWithConsumerPerPartition.getLock().unlock(); - } - } else { - scheduleTopicRepartition(queueKey); //reschedule later - } - } - - public void stop() { -// consumers.values().forEach(TbQueueConsumer::stop); -// topicsConsumerPerPartition.values().forEach(tbTopicWithConsumerPerPartition -> tbTopicWithConsumerPerPartition.getConsumers().keySet() -// .forEach((tpi) -> removeConsumerForTopicByTpi(tbTopicWithConsumerPerPartition.getTopic(), tbTopicWithConsumerPerPartition.getConsumers(), tpi))); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java new file mode 100644 index 0000000000..4d01d15d9a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java @@ -0,0 +1,9 @@ +package org.thingsboard.server.service.queue.ruleengine; + +import java.io.Serializable; + +public enum QueueEvent implements Serializable { + + CREATED, LAUNCHED, UPDATED, PARTITION_CHANGE, STOP, DELETED + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java new file mode 100644 index 0000000000..5b4f164a10 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.ruleengine; + +import lombok.Data; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; + +import java.util.Set; + +@Data +public class TbQueueConsumerManagerTask { + + private final QueueEvent event; + private final Queue queue; + private final Set partitions; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java new file mode 100644 index 0000000000..3f549b273c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java @@ -0,0 +1,70 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.ruleengine; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.QueueKey; + +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +@Data +@Slf4j +public class TbQueueConsumerTask { + + private final QueueKey key; + private final Object id; + private final TbQueueConsumer> consumer; + private volatile Future task; + + public void stop() { + this.consumer.stop(); + } + + public boolean stopAndAwait() { + this.consumer.stop(); + return await(); + } + + public boolean await() { + if (task != null) { + try { + this.task.get(3, TimeUnit.MINUTES); + } catch (ExecutionException | InterruptedException | TimeoutException e) { + log.warn("[{}][{}] Failed to await for consumer to stop", key, id, e); + return false; + } + } + return true; + } + + public void subscribe(Set partitions) { + this.consumer.subscribe(partitions); + } + + + public void unsubscribe() { + this.consumer.unsubscribe(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java new file mode 100644 index 0000000000..49f1ad1fab --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java @@ -0,0 +1,75 @@ +package org.thingsboard.server.service.queue.ruleengine; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.dao.queue.QueueService; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; +import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService; +import org.thingsboard.server.service.stats.RuleEngineStatisticsService; + +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +@Component +@TbRuleEngineComponent +@Slf4j +@Data +public class TbRuleEngineConsumerContext { + + @Value("${queue.rule-engine.poll-interval}") + private long pollDuration; + @Value("${queue.rule-engine.pack-processing-timeout}") + private long packProcessingTimeout; + @Value("${queue.rule-engine.stats.enabled:true}") + private boolean statsEnabled; + @Value("${queue.rule-engine.prometheus-stats.enabled:false}") + boolean prometheusStatsEnabled; + @Value("${queue.rule-engine.topic-deletion-delay:30}") + private int topicDeletionDelayInSec; + + protected volatile boolean stopped = false; + protected volatile boolean isReady = false; + + private final ActorSystemContext actorContext; + private final StatsFactory statsFactory; + private final TbRuleEngineSubmitStrategyFactory submitStrategyFactory; + private final TbRuleEngineProcessingStrategyFactory processingStrategyFactory; + private final TbRuleEngineQueueFactory queueFactory; + private final RuleEngineStatisticsService statisticsService; + private final TbServiceInfoProvider serviceInfoProvider; + private final QueueService queueService; + private final TbQueueProducerProvider producerProvider; + private final TbQueueAdmin queueAdmin; + + //TODO: add reasonable limit for mgmt pool. + private final ExecutorService mgmtExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-mgmt")); + private final ExecutorService consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer")); + private final ExecutorService submitExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-submit")); + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-scheduler")); + + public List findAllQueues() { + return queueService.findAllQueues(); + } + + @PreDestroy + public void stop() { + consumersExecutor.shutdownNow(); // TODO: shutdown or shutdownNow? + submitExecutor.shutdownNow(); + scheduler.shutdownNow(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java new file mode 100644 index 0000000000..2599a214f0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -0,0 +1,431 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.ruleengine; + +import com.google.protobuf.ProtocolStringList; +import lombok.Data; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.gen.MsgProtos; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.RuleNodeInfo; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.service.queue.TbMsgPackCallback; +import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; +import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +@Data +@Slf4j +public class TbRuleEngineQueueConsumerManager { + + public static final String SUCCESSFUL_STATUS = "successful"; + public static final String FAILED_STATUS = "failed"; + + private final TbRuleEngineConsumerContext ctx; + private final QueueKey key; + + private final ReentrantLock lock = new ReentrantLock(); //NonfairSync + private final ConcurrentMap consumersPerPartition = new ConcurrentHashMap<>(); + private final TbRuleEngineConsumerStats stats; + + private volatile Set partitions = Collections.emptySet(); + private volatile Queue queue; + private volatile TbQueueConsumerTask mainConsumer; + + private final java.util.Queue tasks = new ConcurrentLinkedQueue<>(); + + public TbRuleEngineQueueConsumerManager(TbRuleEngineConsumerContext ctx, QueueKey key) { + this.ctx = ctx; + this.key = key; + this.stats = new TbRuleEngineConsumerStats(key, ctx.getStatsFactory()); + } + + public void init(Queue queue) { + processTask(new TbQueueConsumerManagerTask(QueueEvent.CREATED, queue, null)); + } + + public void update(Queue queue) { + processTask(new TbQueueConsumerManagerTask(QueueEvent.UPDATED, queue, null)); + } + + public void subscribe(PartitionChangeEvent event) { + processTask(new TbQueueConsumerManagerTask(QueueEvent.PARTITION_CHANGE, queue, event.getPartitions())); + } + + public void launchMainConsumer() { + processTask(new TbQueueConsumerManagerTask(QueueEvent.LAUNCHED, null, null)); + } + + public void stop() { + processTask(new TbQueueConsumerManagerTask(QueueEvent.STOP, null, null)); + } + + public void delete() { + processTask(new TbQueueConsumerManagerTask(QueueEvent.DELETED, null, null)); + } + + private void processTask(TbQueueConsumerManagerTask todo) { + tasks.add(todo); + log.info("[{}] Adding task: {}", key, todo); + tryProcessTasks(); + } + + private void tryProcessTasks() { + ctx.getMgmtExecutor().submit(() -> { + if (lock.tryLock()) { + try { + Queue newConfiguration = null; + Set newPartitions = null; + while (!tasks.isEmpty()) { + TbQueueConsumerManagerTask task = tasks.poll(); + switch (task.getEvent()) { + case CREATED: + doInit(task.getQueue()); + break; + case LAUNCHED: + if (!queue.isConsumerPerPartition()) { + doLaunchMainConsumer(); + } + break; + case UPDATED: + newConfiguration = task.getQueue(); + break; + case PARTITION_CHANGE: + newPartitions = task.getPartitions(); + break; + case STOP: + newConfiguration = null; + newPartitions = null; + doStop(); + break; + case DELETED: + newConfiguration = null; + newPartitions = null; + doDelete(); + break; + } + } + if (newConfiguration != null) { + doUpdate(newConfiguration); + } + if (newPartitions != null) { + doUpdate(newPartitions); + } + } finally { + lock.unlock(); + } + } else { + log.debug("[{}] Failed to acquire lock.", key); + ctx.getScheduler().schedule(this::tryProcessTasks, 1, TimeUnit.SECONDS); + } + }); + } + + public void doInit(Queue queue) { + log.info("[{}] Init consumer with queue: {}", key, queue); + this.queue = queue; + if (queue.isConsumerPerPartition()) { + log.debug("[{}] Ignore init event since isConsumerPerPartition is enabled.", key); + } else { + mainConsumer = new TbQueueConsumerTask(key, "main", ctx.getQueueFactory().createToRuleEngineMsgConsumer(queue)); + } + } + + private void doLaunchMainConsumer() { + if (mainConsumer != null) { + launchConsumer(mainConsumer, queue, mainConsumer.getId(), queue.getName()); + } else { + log.warn("[{}] Can't launch main consumer since it is empty!", key); + } + } + + private void doUpdate(Queue newQueue) { + log.info("[{}] Processing queue update: {}", key, newQueue); + var oldQueue = queue; + if (log.isTraceEnabled()) { + log.trace("[{}] Old queue configuration: {}", key, oldQueue); + log.trace("[{}] New queue configuration: {}", key, newQueue); + } + if (oldQueue != null) { + doStop(oldQueue); + } + doInit(newQueue); + if (!newQueue.isConsumerPerPartition()) { + doLaunchMainConsumer(); + } + } + + private void doUpdate(Set partitions) { + log.info("[{}] Subscribing to partitions: {}", key, partitions); + if (queue.isConsumerPerPartition()) { + log.debug("[{}] Subscribing consumers per partition separately: {}", key, partitions); + Set addedPartitions = new HashSet<>(partitions); + addedPartitions.removeAll(consumersPerPartition.keySet()); + log.info("calculated addedPartitions {}", addedPartitions); + + Set removedPartitions = new HashSet<>(consumersPerPartition.keySet()); + removedPartitions.removeAll(partitions); + log.info("calculated removedPartitions {}", removedPartitions); + + removedPartitions.forEach((tpi) -> { + log.info("[{}] Unsubscribing from topic: {}", queue, tpi); + consumersPerPartition.get(tpi).unsubscribe(); + }); + + removedPartitions.forEach((tpi) -> { + log.info("[{}] Removing consumer for topic: {}", queue, tpi); + consumersPerPartition.get(tpi).stopAndAwait(); + consumersPerPartition.remove(tpi); + }); + + addedPartitions.forEach((tpi) -> { + log.info("[{}] Adding consumer for topic: {}", key, tpi); + TbQueueConsumerTask consumerTask = new TbQueueConsumerTask(key, tpi, ctx.getQueueFactory().createToRuleEngineMsgConsumer(queue)); + consumersPerPartition.put(tpi, consumerTask); + //TODO: Is it ok to subscribe first? + consumerTask.subscribe(Collections.singleton(tpi)); + launchConsumer(consumerTask, queue, mainConsumer.getId(), key + "-" + tpi.getPartition().orElse(-999999)); + }); + } else { + mainConsumer.subscribe(partitions); + } + } + + private void doStop() { + doStop(queue); + } + + private void doStop(Queue queue) { + if (queue.isConsumerPerPartition()) { + consumersPerPartition.values().forEach(TbQueueConsumerTask::unsubscribe); + consumersPerPartition.values().forEach(TbQueueConsumerTask::stopAndAwait); + } else if (mainConsumer != null) { + mainConsumer.unsubscribe(); + mainConsumer.stopAndAwait(); + } + } + + private void doDelete() { + doStop(); + //TODO: repack messages + } + + @SneakyThrows + void launchConsumer(TbQueueConsumerTask consumerTask, Queue configuration, Object consumerKey, String threadSuffix) { + log.info("[{}] Launching consumer: [{}]", key, consumerKey); + while (!ctx.isReady) { + //TODO: Remember this task. Cancel previous task if needed. + log.debug("[{}][{}] Waiting for consumer to get ready..", key, consumerKey); + Thread.sleep(1000); + } + consumerTask.setTask(ctx.getConsumersExecutor().submit(() -> consumerLoop(consumerTask.getConsumer(), configuration, threadSuffix))); + } + + void consumerLoop(TbQueueConsumer> consumer, Queue configuration, String threadSuffix) { + ThingsBoardThreadFactory.updateCurrentThreadName(threadSuffix); + while (!ctx.stopped && !consumer.isStopped() + //TODO: remove this. + && !consumer.isQueueDeleted()) { + try { + List> msgs = consumer.poll(queue.getPollInterval()); + if (msgs.isEmpty()) { + continue; + } + final TbRuleEngineSubmitStrategy submitStrategy = getSubmitStrategy(queue); + final TbRuleEngineProcessingStrategy ackStrategy = getProcessingStrategy(queue); + submitStrategy.init(msgs); + while (!ctx.isStopped() && !consumer.isStopped()) { + TbMsgPackProcessingContext packCtx = new TbMsgPackProcessingContext(queue.getName(), submitStrategy, ackStrategy.isSkipTimeoutMsgs()); + submitStrategy.submitAttempt((id, msg) -> ctx.getSubmitExecutor().submit(() -> submitMessage(configuration, stats, packCtx, id, msg))); + + final boolean timeout = !packCtx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS); + + TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(configuration.getName(), timeout, packCtx); + if (timeout) { + printFirstOrAll(configuration, packCtx, packCtx.getPendingMap(), "Timeout"); + } + if (!packCtx.getFailedMap().isEmpty()) { + printFirstOrAll(configuration, packCtx, packCtx.getFailedMap(), "Failed"); + } + packCtx.printProfilerStats(); + + TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result); + if (ctx.isStatsEnabled()) { + stats.log(result, decision.isCommit()); + } + + packCtx.cleanup(); + + if (decision.isCommit()) { + submitStrategy.stop(); + break; + } else { + submitStrategy.update(decision.getReprocessMap()); + } + } + consumer.commit(); + } catch (Exception e) { + if (!ctx.stopped) { + log.warn("Failed to process messages from queue.", e); + try { + Thread.sleep(ctx.getPollDuration()); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + } + //TODO: refactor and move to the "doDelete" method. Use separate consumer if needed (it is still synchronous). + if (consumer.isQueueDeleted()) { + processQueueDeletion(configuration, consumer); + } + log.info("TB Rule Engine Consumer stopped."); + } + + private void processQueueDeletion(Queue queue, TbQueueConsumer> consumer) { +// long finishTs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(topicDeletionDelayInSec); +// try { +// int n = 0; +// while (System.currentTimeMillis() <= finishTs) { +// List> msgs = consumer.poll(queue.getPollInterval()); +// if (msgs.isEmpty()) { +// continue; +// } +// for (TbProtoQueueMsg msg : msgs) { +// try { +// MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray()); +// EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB())); +// +// TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue.getName(), TenantId.SYS_TENANT_ID, originator); +// producerProvider.getRuleEngineMsgProducer().send(tpi, msg, null); +// n++; +// } catch (Throwable e) { +// log.debug("Failed to move message to system {}: {}", consumer.getTopic(), msg, e); +// } +// } +// consumer.commit(); +// } +// if (n > 0) { +// log.info("Moved {} messages from {} to system {}", n, consumer.getFullTopicNames(), consumer.getTopic()); +// } +// +// consumer.unsubscribe(); +// for (String topic : consumer.getFullTopicNames()) { +// try { +// queueAdmin.deleteTopic(topic); +// log.info("Deleted topic {}", topic); +// } catch (Exception e) { +// log.error("Failed to delete topic {} after unsubscribing", topic, e); +// } +// } +// } catch (Exception e) { +// log.error("Failed to process deletion of {} ({})", consumer.getTopic(), queue.getTenantId(), e); +// } + } + + TbRuleEngineSubmitStrategy getSubmitStrategy(Queue configuration) { + return ctx.getSubmitStrategyFactory().newInstance(configuration.getName(), configuration.getSubmitStrategy()); + } + + TbRuleEngineProcessingStrategy getProcessingStrategy(Queue configuration) { + return ctx.getProcessingStrategyFactory().newInstance(configuration.getName(), configuration.getProcessingStrategy()); + } + + void submitMessage(Queue configuration, TbRuleEngineConsumerStats stats, TbMsgPackProcessingContext packCtx, UUID id, TbProtoQueueMsg msg) { + log.trace("[{}] Creating callback for topic {} message: {}", id, configuration.getName(), msg.getValue()); + TransportProtos.ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); + TenantId tenantId = TenantId.fromUUID(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); + TbMsgCallback callback = ctx.prometheusStatsEnabled ? + new TbMsgPackCallback(id, tenantId, packCtx, stats.getTimer(tenantId, SUCCESSFUL_STATUS), stats.getTimer(tenantId, FAILED_STATUS)) : + new TbMsgPackCallback(id, tenantId, packCtx); + try { + if (!toRuleEngineMsg.getTbMsg().isEmpty()) { + forwardToRuleEngineActor(configuration.getName(), tenantId, toRuleEngineMsg, callback); + } else { + callback.onSuccess(); + } + } catch (Exception e) { + callback.onFailure(new RuleEngineException(e.getMessage(), e)); + } + } + + private void forwardToRuleEngineActor(String queueName, TenantId tenantId, TransportProtos.ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { + TbMsg tbMsg = TbMsg.fromBytes(queueName, toRuleEngineMsg.getTbMsg().toByteArray(), callback); + QueueToRuleEngineMsg msg; + ProtocolStringList relationTypesList = toRuleEngineMsg.getRelationTypesList(); + Set relationTypes; + if (relationTypesList.size() == 1) { + relationTypes = Collections.singleton(relationTypesList.get(0)); + } else { + relationTypes = new HashSet<>(relationTypesList); + } + msg = new QueueToRuleEngineMsg(tenantId, tbMsg, relationTypes, toRuleEngineMsg.getFailureMessage()); + ctx.getActorContext().tell(msg); + } + + + private void printFirstOrAll(Queue configuration, TbMsgPackProcessingContext ctx, Map> map, String prefix) { + boolean printAll = log.isTraceEnabled(); + log.info("{} to process [{}] messages", prefix, map.size()); + for (Map.Entry> pending : map.entrySet()) { + TransportProtos.ToRuleEngineMsg tmp = pending.getValue().getValue(); + TbMsg tmpMsg = TbMsg.fromBytes(configuration.getName(), tmp.getTbMsg().toByteArray(), TbMsgCallback.EMPTY); + RuleNodeInfo ruleNodeInfo = ctx.getLastVisitedRuleNode(pending.getKey()); + if (printAll) { + log.trace("[{}] {} to process message: {}, Last Rule Node: {}", TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); + } else { + log.info("[{}] {} to process message: {}, Last Rule Node: {}", TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); + break; + } + } + } + + public void printStats(long ts) { + stats.printStats(); + ctx.getStatisticsService().reportQueueStats(ts, stats); + stats.reset(); + } +} diff --git a/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardThreadFactory.java b/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardThreadFactory.java index 1f41c48f43..9eaca3ca47 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardThreadFactory.java +++ b/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardThreadFactory.java @@ -22,6 +22,7 @@ import java.util.concurrent.atomic.AtomicInteger; * Copy of Executors.DefaultThreadFactory but with ability to set name of the pool */ public class ThingsBoardThreadFactory implements ThreadFactory { + public static final String THREAD_TOPIC_SEPARATOR = " | "; private static final AtomicInteger poolNumber = new AtomicInteger(1); private final ThreadGroup group; private final AtomicInteger threadNumber = new AtomicInteger(1); @@ -40,6 +41,17 @@ public class ThingsBoardThreadFactory implements ThreadFactory { "-thread-"; } + public static void updateCurrentThreadName(String threadSuffix) { + String name = Thread.currentThread().getName(); + int spliteratorIndex = name.indexOf(THREAD_TOPIC_SEPARATOR); + if (spliteratorIndex > 0) { + name = name.substring(0, spliteratorIndex); + } + name = name + THREAD_TOPIC_SEPARATOR + threadSuffix; + Thread.currentThread().setName(name); + } + + @Override public Thread newThread(Runnable r) { Thread t = new Thread(group, r, From c56132d6e89d00043b0d644850891663ef0b9122 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 3 Oct 2023 15:04:41 +0300 Subject: [PATCH 08/23] Minor todos --- .../server/service/queue/ruleengine/TbQueueConsumerTask.java | 1 + .../service/queue/ruleengine/TbRuleEngineConsumerContext.java | 3 +++ 2 files changed, 4 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java index 3f549b273c..ed1f36074f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java @@ -49,6 +49,7 @@ public class TbQueueConsumerTask { public boolean await() { if (task != null) { + //TODO: maybe task.cancel() to interrupt the consumer? try { this.task.get(3, TimeUnit.MINUTES); } catch (ExecutionException | InterruptedException | TimeoutException e) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java index 49f1ad1fab..07307a3f10 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java @@ -42,6 +42,7 @@ public class TbRuleEngineConsumerContext { @Value("${queue.rule-engine.topic-deletion-delay:30}") private int topicDeletionDelayInSec; + //TODO: check if they are set correctly. protected volatile boolean stopped = false; protected volatile boolean isReady = false; @@ -59,6 +60,7 @@ public class TbRuleEngineConsumerContext { //TODO: add reasonable limit for mgmt pool. private final ExecutorService mgmtExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-mgmt")); private final ExecutorService consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer")); + //TODO: do we actually need this? private final ExecutorService submitExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-submit")); private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-scheduler")); @@ -68,6 +70,7 @@ public class TbRuleEngineConsumerContext { @PreDestroy public void stop() { + mgmtExecutor.shutdownNow(); consumersExecutor.shutdownNow(); // TODO: shutdown or shutdownNow? submitExecutor.shutdownNow(); scheduler.shutdownNow(); From 92e0c65a3740f179a9ec651a392fdcfbbfa6f175 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Sat, 14 Oct 2023 21:50:40 +0300 Subject: [PATCH 09/23] Minor refactoring --- .../queue/DefaultTbCoreConsumerService.java | 10 +++- .../DefaultTbRuleEngineConsumerService.java | 59 +++++++++++++------ .../queue/TbRuleEngineConsumerStats.java | 6 +- .../processing/AbstractConsumerService.java | 11 +--- .../TbRuleEngineConsumerContext.java | 45 +++++++------- .../src/main/resources/thingsboard.yml | 4 +- .../controller/BaseQueueControllerTest.java | 6 +- 7 files changed, 85 insertions(+), 56 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index fd5a252cc5..b2d7700441 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -142,8 +142,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> usageStatsConsumer; private final TbQueueConsumer> firmwareStatesConsumer; + protected volatile ExecutorService consumersExecutor; protected volatile ExecutorService usageStatsExecutor; - private volatile ExecutorService firmwareStatesExecutor; public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, @@ -186,7 +186,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService implements TbRuleEngineConsumerService { private final TbRuleEngineConsumerContext ctx; + private final QueueService queueService; private final TbRuleEngineDeviceRpcService tbDeviceRpcService; private final ConcurrentMap consumers = new ConcurrentHashMap<>(); @@ -68,6 +72,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< ActorSystemContext actorContext, DataDecodingEncodingService encodingService, TbRuleEngineDeviceRpcService tbDeviceRpcService, + QueueService queueService, TbDeviceProfileCache deviceProfileCache, TbAssetProfileCache assetProfileCache, TbTenantProfileCache tenantProfileCache, @@ -77,12 +82,13 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< eventPublisher, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer(), Optional.empty()); this.ctx = ctx; this.tbDeviceRpcService = tbDeviceRpcService; + this.queueService = queueService; } @PostConstruct public void init() { - super.init("tb-rule-engine-notifications-consumer"); // TODO: restore init of the main consumer? - List queues = ctx.findAllQueues(); + super.init("tb-rule-engine-notifications-consumer"); + List queues = queueService.findAllQueues(); for (Queue configuration : queues) { if (partitionService.isManagedByCurrentService(configuration.getTenantId())) { initConsumer(configuration); @@ -91,30 +97,38 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } private void initConsumer(Queue configuration) { - consumers.computeIfAbsent(new QueueKey(ServiceType.TB_RULE_ENGINE, configuration), - key -> new TbRuleEngineQueueConsumerManager(ctx, key)).init(configuration); + getOrCreateConsumer(new QueueKey(ServiceType.TB_RULE_ENGINE, configuration)).init(configuration); } @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { if (event.getServiceType().equals(getServiceType())) { - var consumer = consumers.get(event.getQueueKey()); - if (consumer != null) { - consumer.subscribe(event); - } else { - log.warn("Received invalid partition change event for {} that is not managed by this service", event.getQueueKey()); - } + event.getPartitionsMap().forEach((queueKey, partitions) -> { + var consumer = consumers.get(queueKey); + if (consumer != null) { + consumer.update(partitions); + } else { + log.warn("Received invalid partition change event for {} that is not managed by this service", queueKey); + } + }); } } + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void onApplicationEvent(ApplicationReadyEvent event) { + super.onApplicationEvent(event); + ctx.setReady(true); + } + @Override protected void launchMainConsumers() { consumers.values().forEach(TbRuleEngineQueueConsumerManager::launchMainConsumer); } @Override - protected void stopMainConsumers() { + protected void stopConsumers() { consumers.values().forEach(TbRuleEngineQueueConsumerManager::stop); + ctx.stop(); } @Override @@ -164,8 +178,15 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< QueueId queueId = new QueueId(new UUID(queueUpdateMsg.getQueueIdMSB(), queueUpdateMsg.getQueueIdLSB())); String queueName = queueUpdateMsg.getQueueName(); QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueName, tenantId); - Queue queue = ctx.getQueueService().findQueueById(tenantId, queueId); - consumers.computeIfAbsent(queueKey, key -> new TbRuleEngineQueueConsumerManager(ctx, key)).update(queue); + Queue queue = queueService.findQueueById(tenantId, queueId); + + TbRuleEngineQueueConsumerManager consumerManager = getOrCreateConsumer(queueKey); + Queue oldQueue = consumerManager.getQueue(); + consumerManager.update(queue); + + if (oldQueue != null && queue.getPartitions() == oldQueue.getPartitions()) { + return; + } } partitionService.updateQueue(queueUpdateMsg); @@ -177,15 +198,19 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< log.info("Received queue delete msg: [{}]", queueDeleteMsg); TenantId tenantId = new TenantId(new UUID(queueDeleteMsg.getTenantIdMSB(), queueDeleteMsg.getTenantIdLSB())); QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueDeleteMsg.getQueueName(), tenantId); + var consumerManager = consumers.remove(queueKey); + if (consumerManager != null) { + consumerManager.delete(); + } partitionService.removeQueue(queueDeleteMsg); - var manager = consumers.remove(queueKey); - if (manager != null) { - manager.delete(); - } partitionService.recalculatePartitions(ctx.getServiceInfoProvider().getServiceInfo(), new ArrayList<>(partitionService.getOtherServices(ServiceType.TB_RULE_ENGINE))); } + private TbRuleEngineQueueConsumerManager getOrCreateConsumer(QueueKey queueKey) { + return consumers.computeIfAbsent(queueKey, key -> new TbRuleEngineQueueConsumerManager(ctx, key)); + } + @Scheduled(fixedDelayString = "${queue.rule-engine.stats.print-interval-ms}") public void printStats() { if (ctx.isStatsEnabled()) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index 45ccbcf32a..842b627c22 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -67,9 +67,9 @@ public class TbRuleEngineConsumerStats { private final String queueName; private final TenantId tenantId; - public TbRuleEngineConsumerStats(QueueKey queue, StatsFactory statsFactory) { - this.queueName = queue.getQueueName(); - this.tenantId = queue.getTenantId(); + public TbRuleEngineConsumerStats(QueueKey queueKey, StatsFactory statsFactory) { + this.queueName = queueKey.getQueueName(); + this.tenantId = queueKey.getTenantId(); this.statsFactory = statsFactory; String statsKey = StatsType.RULE_ENGINE.getName() + "." + queueName; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index b59086a350..d5be1243b7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -65,7 +65,6 @@ import java.util.stream.Collectors; @Slf4j public abstract class AbstractConsumerService extends TbApplicationEventListener { - protected volatile ExecutorService consumersExecutor; protected volatile ExecutorService notificationsConsumerExecutor; protected volatile boolean stopped = false; protected volatile boolean isReady = false; @@ -99,8 +98,7 @@ public abstract class AbstractConsumerService findAllQueues() { - return queueService.findAllQueues(); + @PostConstruct + public void init() { + this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer")); + this.mgmtExecutor = Executors.newFixedThreadPool(mgmtThreadPoolSize, ThingsBoardThreadFactory.forName("tb-rule-engine-mgmt")); + this.scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-scheduler")); } - @PreDestroy public void stop() { - mgmtExecutor.shutdownNow(); - consumersExecutor.shutdownNow(); // TODO: shutdown or shutdownNow? - submitExecutor.shutdownNow(); scheduler.shutdownNow(); + consumersExecutor.shutdown(); + mgmtExecutor.shutdown(); + try { + mgmtExecutor.awaitTermination(15, TimeUnit.SECONDS); + } catch (InterruptedException e) { + log.warn("Failed to await mgmtExecutor termination"); + } } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 9fbfd2fc63..6f51413ccd 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1303,7 +1303,9 @@ queue: pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}" # Time in seconds to wait in consumer thread before retries; max-pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}" # Max allowed time in seconds for pause between retries. # After a queue is deleted (or profile's isolation option was disabled), Rule Engine will continue reading related topics during this period, before deleting the actual topics - topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:30}" + topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:15}" + # Size of the thread pool that handles management operations such as subscribe, unsubscribe, queue delete, etc. + management-thread-pool-size: "${TB_QUEUE_RULE_ENGINE_MGMT_THREAD_POOL_SIZE:12}" transport: # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java index 845ac49f15..8418ba67d9 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java @@ -37,12 +37,14 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.queue.SubmitStrategy; import org.thingsboard.server.common.data.queue.SubmitStrategyType; import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.timeseries.TimeseriesDao; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; import org.thingsboard.server.service.stats.DefaultRuleEngineStatisticsService; @@ -163,7 +165,7 @@ public class BaseQueueControllerTest extends AbstractControllerTest { tenantId, ruleEngineException ))); - TbRuleEngineConsumerStats testStats = new TbRuleEngineConsumerStats(queue, statsFactory); + TbRuleEngineConsumerStats testStats = new TbRuleEngineConsumerStats(new QueueKey(ServiceType.TB_RULE_ENGINE, queue), statsFactory); testStats.log(testProcessingResult, true); int queueStatsTtlDays = 14; @@ -215,7 +217,7 @@ public class BaseQueueControllerTest extends AbstractControllerTest { tenantId, ruleEngineException ))); - TbRuleEngineConsumerStats testStats = new TbRuleEngineConsumerStats(queue, statsFactory); + TbRuleEngineConsumerStats testStats = new TbRuleEngineConsumerStats(new QueueKey(ServiceType.TB_RULE_ENGINE, queue), statsFactory); testStats.log(testProcessingResult, true); ruleEngineStatisticsService.reportQueueStats(System.currentTimeMillis(), testStats); From cf951ee2b22cde58b6fcd39a0f91d13213ce8b25 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 17 Oct 2023 11:23:58 +0300 Subject: [PATCH 10/23] Multiple improvements for Rule Engine consumer services --- .../DefaultTbRuleEngineConsumerService.java | 6 +- .../service/queue/ruleengine/QueueEvent.java | 17 +- .../TbQueueConsumerManagerTask.java | 25 +- .../queue/ruleengine/TbQueueConsumerTask.java | 43 +- .../TbRuleEngineConsumerContext.java | 19 +- .../TbRuleEngineQueueConsumerManager.java | 519 ++++++++++-------- .../server/queue/TbQueueConsumer.java | 6 +- .../AbstractTbQueueConsumerTemplate.java | 39 +- .../queue/kafka/TbKafkaConsumerTemplate.java | 1 - .../queue/memory/InMemoryTbQueueConsumer.java | 17 +- 10 files changed, 384 insertions(+), 308 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 164b48c25f..64139530b1 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -121,9 +121,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } @Override - protected void launchMainConsumers() { - consumers.values().forEach(TbRuleEngineQueueConsumerManager::launchMainConsumer); - } + protected void launchMainConsumers() {} @Override protected void stopConsumers() { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java index 4d01d15d9a..a4a3bcc59a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java @@ -1,9 +1,24 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue.ruleengine; import java.io.Serializable; public enum QueueEvent implements Serializable { - CREATED, LAUNCHED, UPDATED, PARTITION_CHANGE, STOP, DELETED + PARTITION_CHANGE, CONFIG_UPDATE, STOP, DELETE } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java index 5b4f164a10..6f1adbdcf2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java @@ -15,18 +15,33 @@ */ package org.thingsboard.server.service.queue.ruleengine; -import lombok.Data; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import lombok.Getter; +import lombok.ToString; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.Set; -@Data +@Getter +@ToString public class TbQueueConsumerManagerTask { private final QueueEvent event; - private final Queue queue; - private final Set partitions; + private Queue queue; + private Set partitions; + + public TbQueueConsumerManagerTask(QueueEvent event) { + this.event = event; + } + + public TbQueueConsumerManagerTask(QueueEvent event, Queue queue) { + this.event = event; + this.queue = queue; + } + + public TbQueueConsumerManagerTask(QueueEvent event, Set partitions) { + this.event = event; + this.partitions = partitions; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java index ed1f36074f..a0a5579951 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,51 +21,44 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.QueueKey; import java.util.Set; -import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; @Data @Slf4j public class TbQueueConsumerTask { - private final QueueKey key; - private final Object id; + private final Object key; private final TbQueueConsumer> consumer; private volatile Future task; - public void stop() { - this.consumer.stop(); + public void subscribe(Set partitions) { + log.trace("[{}] Subscribing to partitions: {}", key, partitions); + consumer.subscribe(partitions); } - public boolean stopAndAwait() { - this.consumer.stop(); - return await(); + public void initiateStop() { + log.debug("[{}] Initiating stop", key); + consumer.stop(); } - public boolean await() { - if (task != null) { - //TODO: maybe task.cancel() to interrupt the consumer? + public void awaitFinish() { + log.trace("[{}] Awaiting finish", key); + if (isRunning()) { try { - this.task.get(3, TimeUnit.MINUTES); - } catch (ExecutionException | InterruptedException | TimeoutException e) { - log.warn("[{}][{}] Failed to await for consumer to stop", key, id, e); - return false; + task.get(60, TimeUnit.SECONDS); + task = null; + } catch (Exception e) { + log.warn("[{}] Failed to await for consumer to stop", key, e); } } - return true; + log.trace("[{}] Awaited finish", key); } - public void subscribe(Set partitions) { - this.consumer.subscribe(partitions); + public boolean isRunning() { + return task != null && !task.isDone(); } - - public void unsubscribe() { - this.consumer.unsubscribe(); - } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java index c1d17746e1..325a685972 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue.ruleengine; import lombok.Data; @@ -36,8 +51,8 @@ public class TbRuleEngineConsumerContext { @Value("${queue.rule-engine.stats.enabled:true}") private boolean statsEnabled; @Value("${queue.rule-engine.prometheus-stats.enabled:false}") - boolean prometheusStatsEnabled; - @Value("${queue.rule-engine.topic-deletion-delay:30}") + private boolean prometheusStatsEnabled; + @Value("${queue.rule-engine.topic-deletion-delay:15}") private int topicDeletionDelayInSec; @Value("${queue.rule-engine.management-thread-pool-size:12}") private int mgmtThreadPoolSize; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index 2599a214f0..dc3966d0d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,8 +16,7 @@ package org.thingsboard.server.service.queue.ruleengine; import com.google.protobuf.ProtocolStringList; -import lombok.Data; -import lombok.SneakyThrows; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.id.EntityId; @@ -32,11 +31,10 @@ import org.thingsboard.server.common.msg.queue.RuleNodeInfo; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.QueueKey; -import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.service.queue.TbMsgPackCallback; import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; @@ -45,19 +43,20 @@ import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingRes import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; +import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; -@Data @Slf4j public class TbRuleEngineQueueConsumerManager { @@ -65,87 +64,83 @@ public class TbRuleEngineQueueConsumerManager { public static final String FAILED_STATUS = "failed"; private final TbRuleEngineConsumerContext ctx; - private final QueueKey key; - - private final ReentrantLock lock = new ReentrantLock(); //NonfairSync - private final ConcurrentMap consumersPerPartition = new ConcurrentHashMap<>(); + private final QueueKey queueKey; private final TbRuleEngineConsumerStats stats; + private final ReentrantLock lock = new ReentrantLock(); //NonfairSync - private volatile Set partitions = Collections.emptySet(); + @Getter private volatile Queue queue; - private volatile TbQueueConsumerTask mainConsumer; + @Getter + private volatile Set partitions; + private volatile ConsumerWrapper consumerWrapper; + + private volatile boolean stopped; private final java.util.Queue tasks = new ConcurrentLinkedQueue<>(); - public TbRuleEngineQueueConsumerManager(TbRuleEngineConsumerContext ctx, QueueKey key) { + public TbRuleEngineQueueConsumerManager(TbRuleEngineConsumerContext ctx, QueueKey queueKey) { this.ctx = ctx; - this.key = key; - this.stats = new TbRuleEngineConsumerStats(key, ctx.getStatsFactory()); + this.queueKey = queueKey; + this.stats = new TbRuleEngineConsumerStats(queueKey, ctx.getStatsFactory()); } public void init(Queue queue) { - processTask(new TbQueueConsumerManagerTask(QueueEvent.CREATED, queue, null)); + doInit(queue); } public void update(Queue queue) { - processTask(new TbQueueConsumerManagerTask(QueueEvent.UPDATED, queue, null)); + addTask(new TbQueueConsumerManagerTask(QueueEvent.CONFIG_UPDATE, queue)); } - public void subscribe(PartitionChangeEvent event) { - processTask(new TbQueueConsumerManagerTask(QueueEvent.PARTITION_CHANGE, queue, event.getPartitions())); - } - - public void launchMainConsumer() { - processTask(new TbQueueConsumerManagerTask(QueueEvent.LAUNCHED, null, null)); + public void update(Set partitions) { + addTask(new TbQueueConsumerManagerTask(QueueEvent.PARTITION_CHANGE, partitions)); } public void stop() { - processTask(new TbQueueConsumerManagerTask(QueueEvent.STOP, null, null)); + addTask(new TbQueueConsumerManagerTask(QueueEvent.STOP)); } public void delete() { - processTask(new TbQueueConsumerManagerTask(QueueEvent.DELETED, null, null)); + addTask(new TbQueueConsumerManagerTask(QueueEvent.DELETE)); } - private void processTask(TbQueueConsumerManagerTask todo) { + private void addTask(TbQueueConsumerManagerTask todo) { + if (stopped) { + return; + } tasks.add(todo); - log.info("[{}] Adding task: {}", key, todo); + log.trace("[{}] Added task: {}", queueKey, todo); tryProcessTasks(); } private void tryProcessTasks() { + if (!ctx.isReady()) { + log.debug("[{}] TbRuleEngineConsumerContext is not ready yet, will process tasks later", queueKey); + ctx.getScheduler().schedule(this::tryProcessTasks, 1, TimeUnit.SECONDS); + return; + } ctx.getMgmtExecutor().submit(() -> { if (lock.tryLock()) { try { Queue newConfiguration = null; Set newPartitions = null; - while (!tasks.isEmpty()) { + while (!stopped) { TbQueueConsumerManagerTask task = tasks.poll(); - switch (task.getEvent()) { - case CREATED: - doInit(task.getQueue()); - break; - case LAUNCHED: - if (!queue.isConsumerPerPartition()) { - doLaunchMainConsumer(); - } - break; - case UPDATED: - newConfiguration = task.getQueue(); - break; - case PARTITION_CHANGE: - newPartitions = task.getPartitions(); - break; - case STOP: - newConfiguration = null; - newPartitions = null; - doStop(); - break; - case DELETED: - newConfiguration = null; - newPartitions = null; - doDelete(); - break; + if (task == null) { + break; + } + log.trace("[{}] Processing task: {}", queueKey, task); + + if (task.getEvent() == QueueEvent.PARTITION_CHANGE) { + newPartitions = task.getPartitions(); + } else if (task.getEvent() == QueueEvent.CONFIG_UPDATE) { + newConfiguration = task.getQueue(); + } else if (task.getEvent() == QueueEvent.STOP) { + doStop(); + return; + } else if (task.getEvent() == QueueEvent.DELETE) { + doDelete(); + return; } } if (newConfiguration != null) { @@ -158,158 +153,108 @@ public class TbRuleEngineQueueConsumerManager { lock.unlock(); } } else { - log.debug("[{}] Failed to acquire lock.", key); + log.trace("[{}] Failed to acquire lock", queueKey); ctx.getScheduler().schedule(this::tryProcessTasks, 1, TimeUnit.SECONDS); } }); } - public void doInit(Queue queue) { - log.info("[{}] Init consumer with queue: {}", key, queue); + private void doInit(Queue queue) { this.queue = queue; if (queue.isConsumerPerPartition()) { - log.debug("[{}] Ignore init event since isConsumerPerPartition is enabled.", key); + consumerWrapper = new ConsumerPerPartitionWrapper(); } else { - mainConsumer = new TbQueueConsumerTask(key, "main", ctx.getQueueFactory().createToRuleEngineMsgConsumer(queue)); - } - } - - private void doLaunchMainConsumer() { - if (mainConsumer != null) { - launchConsumer(mainConsumer, queue, mainConsumer.getId(), queue.getName()); - } else { - log.warn("[{}] Can't launch main consumer since it is empty!", key); + consumerWrapper = new SingleConsumerWrapper(); } + log.debug("[{}] Initialized consumer for queue: {}", queueKey, queue); } private void doUpdate(Queue newQueue) { - log.info("[{}] Processing queue update: {}", key, newQueue); - var oldQueue = queue; + log.info("[{}] Processing queue update: {}", queueKey, newQueue); + var oldQueue = this.queue; + this.queue = newQueue; if (log.isTraceEnabled()) { - log.trace("[{}] Old queue configuration: {}", key, oldQueue); - log.trace("[{}] New queue configuration: {}", key, newQueue); - } - if (oldQueue != null) { - doStop(oldQueue); - } - doInit(newQueue); - if (!newQueue.isConsumerPerPartition()) { - doLaunchMainConsumer(); + log.trace("[{}] Old queue configuration: {}", queueKey, oldQueue); + log.trace("[{}] New queue configuration: {}", queueKey, newQueue); } - } - - private void doUpdate(Set partitions) { - log.info("[{}] Subscribing to partitions: {}", key, partitions); - if (queue.isConsumerPerPartition()) { - log.debug("[{}] Subscribing consumers per partition separately: {}", key, partitions); - Set addedPartitions = new HashSet<>(partitions); - addedPartitions.removeAll(consumersPerPartition.keySet()); - log.info("calculated addedPartitions {}", addedPartitions); - - Set removedPartitions = new HashSet<>(consumersPerPartition.keySet()); - removedPartitions.removeAll(partitions); - log.info("calculated removedPartitions {}", removedPartitions); - removedPartitions.forEach((tpi) -> { - log.info("[{}] Unsubscribing from topic: {}", queue, tpi); - consumersPerPartition.get(tpi).unsubscribe(); - }); + if (oldQueue == null) { + doInit(queue); + } else if (newQueue.isConsumerPerPartition() != oldQueue.isConsumerPerPartition()) { + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::initiateStop); + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitFinish); - removedPartitions.forEach((tpi) -> { - log.info("[{}] Removing consumer for topic: {}", queue, tpi); - consumersPerPartition.get(tpi).stopAndAwait(); - consumersPerPartition.remove(tpi); - }); - - addedPartitions.forEach((tpi) -> { - log.info("[{}] Adding consumer for topic: {}", key, tpi); - TbQueueConsumerTask consumerTask = new TbQueueConsumerTask(key, tpi, ctx.getQueueFactory().createToRuleEngineMsgConsumer(queue)); - consumersPerPartition.put(tpi, consumerTask); - //TODO: Is it ok to subscribe first? - consumerTask.subscribe(Collections.singleton(tpi)); - launchConsumer(consumerTask, queue, mainConsumer.getId(), key + "-" + tpi.getPartition().orElse(-999999)); - }); + doInit(queue); + if (partitions != null) { + doUpdate(partitions); // even if partitions number was changed, there can be no partition change event + } } else { - mainConsumer.subscribe(partitions); + // do nothing, because partitions change (if they changed) will be handled on PartitionChangeEvent, + // and changes to pollInterval/packProcessingTimeout/submitStrategy/processingStrategy will be picked up by consumer on the fly, + // and queue topic and name are immutable } } - private void doStop() { - doStop(queue); + private void doUpdate(Set partitions) { + this.partitions = partitions; + consumerWrapper.updatePartitions(partitions); } - private void doStop(Queue queue) { - if (queue.isConsumerPerPartition()) { - consumersPerPartition.values().forEach(TbQueueConsumerTask::unsubscribe); - consumersPerPartition.values().forEach(TbQueueConsumerTask::stopAndAwait); - } else if (mainConsumer != null) { - mainConsumer.unsubscribe(); - mainConsumer.stopAndAwait(); - } + private void doStop() { + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::initiateStop); + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitFinish); + log.debug("[{}] Unsubscribed and stopped consumers", queueKey); + stopped = true; } private void doDelete() { - doStop(); - //TODO: repack messages + stopped = true; + log.info("[{}] Handling queue deletion", queueKey); + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitFinish); + + List>> queueConsumers = consumerWrapper.getConsumers().stream() + .map(TbQueueConsumerTask::getConsumer).collect(Collectors.toList()); + ctx.getConsumersExecutor().submit(() -> { + drainQueue(queueConsumers); + + queueConsumers.forEach(consumer -> { + for (String topic : consumer.getFullTopicNames()) { + try { + ctx.getQueueAdmin().deleteTopic(topic); + log.info("Deleted topic {}", topic); + } catch (Exception e) { + log.error("Failed to delete topic {}", topic, e); + } + } + try { + consumer.unsubscribe(); + } catch (Exception e) { + log.error("[{}] Failed to unsubscribe consumer", queueKey, e); + } + }); + }); } - @SneakyThrows - void launchConsumer(TbQueueConsumerTask consumerTask, Queue configuration, Object consumerKey, String threadSuffix) { - log.info("[{}] Launching consumer: [{}]", key, consumerKey); - while (!ctx.isReady) { - //TODO: Remember this task. Cancel previous task if needed. - log.debug("[{}][{}] Waiting for consumer to get ready..", key, consumerKey); - Thread.sleep(1000); - } - consumerTask.setTask(ctx.getConsumersExecutor().submit(() -> consumerLoop(consumerTask.getConsumer(), configuration, threadSuffix))); + private void launchConsumer(TbQueueConsumerTask consumerTask) { + log.info("[{}] Launching consumer", consumerTask.getKey()); + Future consumerLoop = ctx.getConsumersExecutor().submit(() -> { + ThingsBoardThreadFactory.updateCurrentThreadName(consumerTask.getKey().toString()); + consumerLoop(consumerTask.getConsumer()); + }); + consumerTask.setTask(consumerLoop); } - void consumerLoop(TbQueueConsumer> consumer, Queue configuration, String threadSuffix) { - ThingsBoardThreadFactory.updateCurrentThreadName(threadSuffix); - while (!ctx.stopped && !consumer.isStopped() - //TODO: remove this. - && !consumer.isQueueDeleted()) { + private void consumerLoop(TbQueueConsumer> consumer) { + while (!stopped && !consumer.isStopped()) { try { - List> msgs = consumer.poll(queue.getPollInterval()); + List> msgs = consumer.poll(queue.getPollInterval()); if (msgs.isEmpty()) { continue; } - final TbRuleEngineSubmitStrategy submitStrategy = getSubmitStrategy(queue); - final TbRuleEngineProcessingStrategy ackStrategy = getProcessingStrategy(queue); - submitStrategy.init(msgs); - while (!ctx.isStopped() && !consumer.isStopped()) { - TbMsgPackProcessingContext packCtx = new TbMsgPackProcessingContext(queue.getName(), submitStrategy, ackStrategy.isSkipTimeoutMsgs()); - submitStrategy.submitAttempt((id, msg) -> ctx.getSubmitExecutor().submit(() -> submitMessage(configuration, stats, packCtx, id, msg))); - - final boolean timeout = !packCtx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS); - - TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(configuration.getName(), timeout, packCtx); - if (timeout) { - printFirstOrAll(configuration, packCtx, packCtx.getPendingMap(), "Timeout"); - } - if (!packCtx.getFailedMap().isEmpty()) { - printFirstOrAll(configuration, packCtx, packCtx.getFailedMap(), "Failed"); - } - packCtx.printProfilerStats(); - - TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result); - if (ctx.isStatsEnabled()) { - stats.log(result, decision.isCommit()); - } - - packCtx.cleanup(); - - if (decision.isCommit()) { - submitStrategy.stop(); - break; - } else { - submitStrategy.update(decision.getReprocessMap()); - } - } - consumer.commit(); + processMsgs(msgs, consumer, queue); } catch (Exception e) { - if (!ctx.stopped) { - log.warn("Failed to process messages from queue.", e); + if (!stopped) { + log.warn("Failed to process messages from queue", e); try { Thread.sleep(ctx.getPollDuration()); } catch (InterruptedException e2) { @@ -318,72 +263,68 @@ public class TbRuleEngineQueueConsumerManager { } } } - //TODO: refactor and move to the "doDelete" method. Use separate consumer if needed (it is still synchronous). - if (consumer.isQueueDeleted()) { - processQueueDeletion(configuration, consumer); + if (consumer.isStopped()) { + consumer.unsubscribe(); } - log.info("TB Rule Engine Consumer stopped."); + log.info("Rule Engine consumer stopped"); } - private void processQueueDeletion(Queue queue, TbQueueConsumer> consumer) { -// long finishTs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(topicDeletionDelayInSec); -// try { -// int n = 0; -// while (System.currentTimeMillis() <= finishTs) { -// List> msgs = consumer.poll(queue.getPollInterval()); -// if (msgs.isEmpty()) { -// continue; -// } -// for (TbProtoQueueMsg msg : msgs) { -// try { -// MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray()); -// EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB())); -// -// TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue.getName(), TenantId.SYS_TENANT_ID, originator); -// producerProvider.getRuleEngineMsgProducer().send(tpi, msg, null); -// n++; -// } catch (Throwable e) { -// log.debug("Failed to move message to system {}: {}", consumer.getTopic(), msg, e); -// } -// } -// consumer.commit(); -// } -// if (n > 0) { -// log.info("Moved {} messages from {} to system {}", n, consumer.getFullTopicNames(), consumer.getTopic()); -// } -// -// consumer.unsubscribe(); -// for (String topic : consumer.getFullTopicNames()) { -// try { -// queueAdmin.deleteTopic(topic); -// log.info("Deleted topic {}", topic); -// } catch (Exception e) { -// log.error("Failed to delete topic {} after unsubscribing", topic, e); -// } -// } -// } catch (Exception e) { -// log.error("Failed to process deletion of {} ({})", consumer.getTopic(), queue.getTenantId(), e); -// } + private void processMsgs(List> msgs, + TbQueueConsumer> consumer, + Queue queue) throws InterruptedException { + TbRuleEngineSubmitStrategy submitStrategy = getSubmitStrategy(queue); + TbRuleEngineProcessingStrategy ackStrategy = getProcessingStrategy(queue); + submitStrategy.init(msgs); + while (!stopped && !consumer.isStopped()) { + TbMsgPackProcessingContext packCtx = new TbMsgPackProcessingContext(queue.getName(), submitStrategy, ackStrategy.isSkipTimeoutMsgs()); + submitStrategy.submitAttempt((id, msg) -> submitMessage(packCtx, id, msg)); + + final boolean timeout = !packCtx.await(queue.getPackProcessingTimeout(), TimeUnit.MILLISECONDS); + + TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(queue.getName(), timeout, packCtx); + if (timeout) { + printFirstOrAll(packCtx, packCtx.getPendingMap(), "Timeout"); + } + if (!packCtx.getFailedMap().isEmpty()) { + printFirstOrAll(packCtx, packCtx.getFailedMap(), "Failed"); + } + packCtx.printProfilerStats(); + + TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result); + if (ctx.isStatsEnabled()) { + stats.log(result, decision.isCommit()); + } + + packCtx.cleanup(); + + if (decision.isCommit()) { + submitStrategy.stop(); + consumer.commit(); + break; + } else { + submitStrategy.update(decision.getReprocessMap()); + } + } } - TbRuleEngineSubmitStrategy getSubmitStrategy(Queue configuration) { - return ctx.getSubmitStrategyFactory().newInstance(configuration.getName(), configuration.getSubmitStrategy()); + private TbRuleEngineSubmitStrategy getSubmitStrategy(Queue queue) { + return ctx.getSubmitStrategyFactory().newInstance(queue.getName(), queue.getSubmitStrategy()); } - TbRuleEngineProcessingStrategy getProcessingStrategy(Queue configuration) { - return ctx.getProcessingStrategyFactory().newInstance(configuration.getName(), configuration.getProcessingStrategy()); + private TbRuleEngineProcessingStrategy getProcessingStrategy(Queue queue) { + return ctx.getProcessingStrategyFactory().newInstance(queue.getName(), queue.getProcessingStrategy()); } - void submitMessage(Queue configuration, TbRuleEngineConsumerStats stats, TbMsgPackProcessingContext packCtx, UUID id, TbProtoQueueMsg msg) { - log.trace("[{}] Creating callback for topic {} message: {}", id, configuration.getName(), msg.getValue()); - TransportProtos.ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); + private void submitMessage(TbMsgPackProcessingContext packCtx, UUID id, TbProtoQueueMsg msg) { + log.trace("[{}] Creating callback for topic {} message: {}", id, queue.getName(), msg.getValue()); + ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); TenantId tenantId = TenantId.fromUUID(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); - TbMsgCallback callback = ctx.prometheusStatsEnabled ? + TbMsgCallback callback = ctx.isPrometheusStatsEnabled() ? new TbMsgPackCallback(id, tenantId, packCtx, stats.getTimer(tenantId, SUCCESSFUL_STATUS), stats.getTimer(tenantId, FAILED_STATUS)) : new TbMsgPackCallback(id, tenantId, packCtx); try { if (!toRuleEngineMsg.getTbMsg().isEmpty()) { - forwardToRuleEngineActor(configuration.getName(), tenantId, toRuleEngineMsg, callback); + forwardToRuleEngineActor(queue.getName(), tenantId, toRuleEngineMsg, callback); } else { callback.onSuccess(); } @@ -392,7 +333,7 @@ public class TbRuleEngineQueueConsumerManager { } } - private void forwardToRuleEngineActor(String queueName, TenantId tenantId, TransportProtos.ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { + private void forwardToRuleEngineActor(String queueName, TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { TbMsg tbMsg = TbMsg.fromBytes(queueName, toRuleEngineMsg.getTbMsg().toByteArray(), callback); QueueToRuleEngineMsg msg; ProtocolStringList relationTypesList = toRuleEngineMsg.getRelationTypesList(); @@ -406,16 +347,15 @@ public class TbRuleEngineQueueConsumerManager { ctx.getActorContext().tell(msg); } - - private void printFirstOrAll(Queue configuration, TbMsgPackProcessingContext ctx, Map> map, String prefix) { + private void printFirstOrAll(TbMsgPackProcessingContext ctx, Map> map, String prefix) { boolean printAll = log.isTraceEnabled(); - log.info("{} to process [{}] messages", prefix, map.size()); - for (Map.Entry> pending : map.entrySet()) { - TransportProtos.ToRuleEngineMsg tmp = pending.getValue().getValue(); - TbMsg tmpMsg = TbMsg.fromBytes(configuration.getName(), tmp.getTbMsg().toByteArray(), TbMsgCallback.EMPTY); + log.info("[{}] {} to process [{}] messages", queueKey, prefix, map.size()); + for (Map.Entry> pending : map.entrySet()) { + ToRuleEngineMsg tmp = pending.getValue().getValue(); + TbMsg tmpMsg = TbMsg.fromBytes(queue.getName(), tmp.getTbMsg().toByteArray(), TbMsgCallback.EMPTY); RuleNodeInfo ruleNodeInfo = ctx.getLastVisitedRuleNode(pending.getKey()); if (printAll) { - log.trace("[{}] {} to process message: {}, Last Rule Node: {}", TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); + log.trace("[{}][{}] {} to process message: {}, Last Rule Node: {}", queueKey, TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); } else { log.info("[{}] {} to process message: {}, Last Rule Node: {}", TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); break; @@ -428,4 +368,117 @@ public class TbRuleEngineQueueConsumerManager { ctx.getStatisticsService().reportQueueStats(ts, stats); stats.reset(); } + + private void drainQueue(List>> consumers) { + long finishTs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(ctx.getTopicDeletionDelayInSec()); + try { + int n = 0; + while (System.currentTimeMillis() <= finishTs) { + for (TbQueueConsumer> consumer : consumers) { + List> msgs = consumer.poll(queue.getPollInterval()); + if (msgs.isEmpty()) { + continue; + } + for (TbProtoQueueMsg msg : msgs) { + try { + MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray()); + EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB())); + + TopicPartitionInfo tpi = ctx.getPartitionService().resolve(ServiceType.TB_RULE_ENGINE, queue.getName(), TenantId.SYS_TENANT_ID, originator); + ctx.getProducerProvider().getRuleEngineMsgProducer().send(tpi, msg, null); + n++; + } catch (Throwable e) { + log.warn("Failed to move message to system {}: {}", consumer.getTopic(), msg, e); + } + } + consumer.commit(); + } + } + if (n > 0) { + log.info("Moved {} messages from {} to system {}", n, queueKey, queue.getName()); + } + } catch (Exception e) { + log.error("[{}] Failed to drain queue", queueKey, e); + } + } + + private static String partitionsToString(Collection partitions) { + return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.joining(", ", "[", "]")); + } + + interface ConsumerWrapper { + + void updatePartitions(Set partitions); + + Collection getConsumers(); + + } + + class ConsumerPerPartitionWrapper implements ConsumerWrapper { + private final Map consumers = new HashMap<>(); + + @Override + public void updatePartitions(Set partitions) { + Set addedPartitions = new HashSet<>(partitions); + addedPartitions.removeAll(consumers.keySet()); + + Set removedPartitions = new HashSet<>(consumers.keySet()); + removedPartitions.removeAll(partitions); + log.info("[{}] Added partitions: {}, removed partitions: {}", queueKey, partitionsToString(addedPartitions), partitionsToString(removedPartitions)); + + removedPartitions.forEach((tpi) -> { + consumers.get(tpi).initiateStop(); + }); + removedPartitions.forEach((tpi) -> { + consumers.remove(tpi).awaitFinish(); + }); + + addedPartitions.forEach((tpi) -> { + String key = queueKey + "-" + tpi.getPartition().orElse(-999999); + TbQueueConsumerTask consumer = new TbQueueConsumerTask(key, ctx.getQueueFactory().createToRuleEngineMsgConsumer(queue)); + consumers.put(tpi, consumer); + consumer.subscribe(Set.of(tpi)); + launchConsumer(consumer); + }); + } + + @Override + public Collection getConsumers() { + return consumers.values(); + } + } + + class SingleConsumerWrapper implements ConsumerWrapper { + private TbQueueConsumerTask consumer; + + @Override + public void updatePartitions(Set partitions) { + log.info("[{}] New partitions: {}", queueKey, partitionsToString(partitions)); + if (partitions.isEmpty()) { + if (consumer != null && consumer.isRunning()) { + consumer.initiateStop(); + consumer.awaitFinish(); + } + consumer = null; + return; + } + + if (consumer == null) { + consumer = new TbQueueConsumerTask(queueKey, ctx.getQueueFactory().createToRuleEngineMsgConsumer(queue)); + } + consumer.subscribe(partitions); + if (!consumer.isRunning()) { + launchConsumer(consumer); + } + } + + @Override + public Collection getConsumers() { + if (consumer == null) { + return Collections.emptyList(); + } + return List.of(consumer); + } + } + } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java index 9c41f9d342..21216e164e 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java @@ -28,6 +28,8 @@ public interface TbQueueConsumer { void subscribe(Set partitions); + void stop(); + void unsubscribe(); List poll(long durationInMillis); @@ -36,10 +38,6 @@ public interface TbQueueConsumer { boolean isStopped(); - void onQueueDelete(); - - boolean isQueueDeleted(); - List getFullTopicNames(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 2ebe41850d..073371ebb5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -44,7 +44,6 @@ public abstract class AbstractTbQueueConsumerTemplate i protected volatile Set partitions; protected final ReentrantLock consumerLock = new ReentrantLock(); //NonfairSync final Queue> subscribeQueue = new ConcurrentLinkedQueue<>(); - protected volatile boolean queueDeleted = false; @Getter private final String topic; @@ -55,7 +54,7 @@ public abstract class AbstractTbQueueConsumerTemplate i @Override public void subscribe() { - log.info("enqueue topic subscribe {} ", topic); + log.debug("enqueue topic subscribe {} ", topic); if (stopped) { log.error("trying subscribe, but consumer stopped for topic {}", topic); return; @@ -65,7 +64,7 @@ public abstract class AbstractTbQueueConsumerTemplate i @Override public void subscribe(Set partitions) { - log.info("enqueue topics subscribe {} ", partitions); + log.debug("enqueue topics subscribe {} ", partitions); if (stopped) { log.error("trying subscribe, but consumer stopped for topic {}", topic); return; @@ -78,7 +77,8 @@ public abstract class AbstractTbQueueConsumerTemplate i List records; long startNanos = System.nanoTime(); if (stopped) { - return errorAndReturnEmpty(); + log.error("poll invoked but consumer stopped for topic " + topic, new RuntimeException("stacktrace")); + return emptyList(); } if (!subscribed && partitions == null && subscribeQueue.isEmpty()) { return sleepAndReturnEmpty(startNanos, durationInMillis); @@ -96,6 +96,7 @@ public abstract class AbstractTbQueueConsumerTemplate i } if (!subscribed) { List topicNames = getFullTopicNames(); + log.info("Subscribing to topics {}", topicNames); doSubscribe(topicNames); subscribed = true; } @@ -127,11 +128,6 @@ public abstract class AbstractTbQueueConsumerTemplate i return result; } - List errorAndReturnEmpty() { - log.error("poll invoked but consumer stopped for topic" + topic, new RuntimeException("stacktrace")); - return emptyList(); - } - List sleepAndReturnEmpty(final long startNanos, final long durationInMillis) { long durationNanos = TimeUnit.MILLISECONDS.toNanos(durationInMillis); long spentNanos = System.nanoTime() - startNanos; @@ -163,15 +159,20 @@ public abstract class AbstractTbQueueConsumerTemplate i } } + @Override + public void stop() { + stopped = true; + } + @Override public void unsubscribe() { - log.info("Unsubscribing from topics and stopping consumer for topics {}", partitions.stream() - .map(TopicPartitionInfo::getFullTopicName) - .collect(Collectors.joining(", "))); + log.info("Unsubscribing and stopping consumer for topics {}", getFullTopicNames()); stopped = true; consumerLock.lock(); try { - doUnsubscribe(); + if (subscribed) { + doUnsubscribe(); + } } finally { consumerLock.unlock(); } @@ -192,17 +193,11 @@ public abstract class AbstractTbQueueConsumerTemplate i abstract protected void doUnsubscribe(); - @Override - public void onQueueDelete() { - queueDeleted = true; - } - - public boolean isQueueDeleted() { - return queueDeleted; - } - @Override public List getFullTopicNames() { + if (partitions == null) { + return Collections.emptyList(); + } return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 9f58446966..ff3f0cc9b3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -73,7 +73,6 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue protected void doSubscribe(List topicNames) { if (!topicNames.isEmpty()) { topicNames.forEach(admin::createTopicIfNotExists); - log.info("subscribe topics {}", topicNames); consumer.subscribe(topicNames); } else { log.info("unsubscribe due to empty topic list"); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index 8711cbbcf1..a7f8cadd0d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -31,7 +31,6 @@ public class InMemoryTbQueueConsumer implements TbQueueCon private volatile Set partitions; private volatile boolean stopped; private volatile boolean subscribed; - private volatile boolean queueDeleted; public InMemoryTbQueueConsumer(InMemoryStorage storage, String topic) { this.storage = storage; @@ -58,9 +57,15 @@ public class InMemoryTbQueueConsumer implements TbQueueCon subscribed = true; } + @Override + public void stop() { + stopped = true; + } + @Override public void unsubscribe() { stopped = true; + subscribed = false; } @Override @@ -104,16 +109,6 @@ public class InMemoryTbQueueConsumer implements TbQueueCon return stopped; } - @Override - public void onQueueDelete() { - queueDeleted = true; - } - - @Override - public boolean isQueueDeleted() { - return queueDeleted; - } - @Override public List getFullTopicNames() { return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); From 921b2ae4b7d41b58c616b79d3d62bce70ea6182b Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 17 Oct 2023 11:24:33 +0300 Subject: [PATCH 11/23] Tests for Rule Engine consumer manager --- .../controller/TenantControllerTest.java | 40 +- .../TbRuleEngineQueueConsumerManagerTest.java | 780 ++++++++++++++++++ 2 files changed, 801 insertions(+), 19 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java index 4b594d55f9..d1aefc097f 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java @@ -42,6 +42,7 @@ import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; @@ -54,12 +55,12 @@ import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; -import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.discovery.PartitionService; @@ -67,9 +68,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; -import java.util.Deque; import java.util.HashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Random; @@ -680,26 +679,29 @@ public class TenantControllerTest extends AbstractControllerTest { .until(() -> partitionService.resolve(ServiceType.TB_RULE_ENGINE, MAIN_QUEUE_NAME, tenantId, tenantId) .getTenantId().get().isSysTenantId()); - Deque submittedMsgs = new LinkedList<>(); - await().atLeast(8, TimeUnit.SECONDS) // due to topic-deletion-delay - .atMost(20, TimeUnit.SECONDS) - .pollInterval(1, TimeUnit.SECONDS) - .untilAsserted(() -> { - TbMsg tbMsg = publishTbMsg(tenantId, tpi); - submittedMsgs.add(tbMsg.getId()); - - verify(queueAdmin, times(1)).deleteTopic(eq(isolatedTopic)); - }); - submittedMsgs.removeLast(); - for (UUID msgId : submittedMsgs) { - verify(actorContext, timeout(2000)).tell(argThat(msg -> { - return msg instanceof QueueToRuleEngineMsg && ((QueueToRuleEngineMsg) msg).getMsg().getId().equals(msgId); - })); + List submittedMsgs = new ArrayList<>(); + long timeLeft = TimeUnit.SECONDS.toMillis(7); // based on topic-deletion-delay + int msgs = 100; + for (int i = 1; i <= msgs; i++) { + TbMsg tbMsg = publishTbMsg(tenantId, tpi); + submittedMsgs.add(tbMsg.getId()); + Thread.sleep(timeLeft / msgs); } + await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> { + verify(queueAdmin, times(1)).deleteTopic(eq(isolatedTopic)); + }); + + await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> { + for (UUID msgId : submittedMsgs) { + verify(actorContext).tell(argThat(msg -> { + return msg instanceof QueueToRuleEngineMsg && ((QueueToRuleEngineMsg) msg).getMsg().getId().equals(msgId); + })); + } + }); } private TbMsg publishTbMsg(TenantId tenantId, TopicPartitionInfo tpi) { - TbMsg tbMsg = TbMsg.newMsg("POST_TELEMETRY_REQUEST", tenantId, TbMsgMetaData.EMPTY, "{\"test\":1}"); + TbMsg tbMsg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, tenantId, TbMsgMetaData.EMPTY, "{\"test\":1}"); TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java new file mode 100644 index 0000000000..276e65ec39 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java @@ -0,0 +1,780 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.ruleengine; + +import lombok.Getter; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomUtils; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.QueueId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.data.queue.ProcessingStrategy; +import org.thingsboard.server.common.data.queue.ProcessingStrategyType; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.data.queue.SubmitStrategy; +import org.thingsboard.server.common.data.queue.SubmitStrategyType; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; +import org.thingsboard.server.service.stats.RuleEngineStatisticsService; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.after; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +@Slf4j +@RunWith(MockitoJUnitRunner.class) +public class TbRuleEngineQueueConsumerManagerTest { + + @Mock + private ActorSystemContext actorContext; + @Mock + private StatsFactory statsFactory; + @Mock + private TbRuleEngineQueueFactory queueFactory; + @Mock + private RuleEngineStatisticsService statisticsService; + @Mock + private TbServiceInfoProvider serviceInfoProvider; + @Mock + private PartitionService partitionService; + @Mock + private TbQueueProducerProvider producerProvider; + private TbQueueProducer> ruleEngineMsgProducer; + @Mock + private TbQueueAdmin queueAdmin; + private TbRuleEngineConsumerContext ruleEngineConsumerContext; + + private TbRuleEngineQueueConsumerManager consumerManager; + private Queue queue; + + private Set consumers; + private boolean generateQueueMsgs; + private AtomicInteger totalConsumedMsgs; + private AtomicInteger totalProcessedMsgs; + + @Before + public void beforeEach() { + ruleEngineConsumerContext = new TbRuleEngineConsumerContext( + actorContext, statsFactory, spy(new TbRuleEngineSubmitStrategyFactory()), + spy(new TbRuleEngineProcessingStrategyFactory()), queueFactory, statisticsService, + serviceInfoProvider, partitionService, producerProvider, queueAdmin + ); + consumers = ConcurrentHashMap.newKeySet(); + generateQueueMsgs = true; + totalConsumedMsgs = new AtomicInteger(); + totalProcessedMsgs = new AtomicInteger(); + doAnswer(inv -> { + QueueToRuleEngineMsg msg = inv.getArgument(0); + msg.getMsg().getCallback().onSuccess(); + totalProcessedMsgs.incrementAndGet(); + log.trace("totalProcessedMsgs = {}", totalProcessedMsgs); + return null; + }).when(actorContext).tell(any()); + ruleEngineMsgProducer = mock(TbQueueProducer.class); + when(producerProvider.getRuleEngineMsgProducer()).thenReturn(ruleEngineMsgProducer); + ruleEngineConsumerContext.setMgmtThreadPoolSize(2); + ruleEngineConsumerContext.setTopicDeletionDelayInSec(5); + ruleEngineConsumerContext.init(); + ruleEngineConsumerContext.setReady(false); + + queue = new Queue(); + queue.setName("Test"); + queue.setTenantId(TenantId.SYS_TENANT_ID); + queue.setId(new QueueId(UUID.randomUUID())); + queue.setTopic("tb_test"); + queue.setPartitions(10); + queue.setConsumerPerPartition(true); + queue.setPollInterval(250); + queue.setPackProcessingTimeout(2000); + SubmitStrategy submitStrategy = new SubmitStrategy(); + submitStrategy.setType(SubmitStrategyType.BURST); + submitStrategy.setBatchSize(200); + queue.setSubmitStrategy(submitStrategy); + ProcessingStrategy processingStrategy = new ProcessingStrategy(); + processingStrategy.setType(ProcessingStrategyType.SKIP_ALL_FAILURES_AND_TIMED_OUT); + processingStrategy.setRetries(0); + queue.setProcessingStrategy(processingStrategy); + + doAnswer(i -> { + TestConsumer consumer = spy(new TestConsumer(queue.getTopic())); + if (generateQueueMsgs) { + consumer.setUpTestMsg(); + } + consumers.add(consumer); + return consumer; + }).when(queueFactory).createToRuleEngineMsgConsumer(any()); + + QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queue); + consumerManager = new TbRuleEngineQueueConsumerManager(ruleEngineConsumerContext, queueKey); + } + + @After + public void afterEach() { + consumerManager.stop(); + ruleEngineConsumerContext.stop(); + + if (generateQueueMsgs) { + await().atMost(1, TimeUnit.SECONDS) + .until(() -> totalProcessedMsgs.get() == totalConsumedMsgs.get()); + } + log.debug("totalConsumedMsgs = {}, totalProcessedMsgs = {}", totalConsumedMsgs.get(), totalProcessedMsgs.get()); + } + + @Test + public void testInit_consumerPerPartition() { + queue.setConsumerPerPartition(true); + consumerManager.init(queue); + + Set partitions = createTpis(2, 3, 4); + consumerManager.update(partitions); + partitions = createTpis(3, 4, 5); + consumerManager.update(partitions); + partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + // simulated multiple partition change events before consumer is ready; only latest partitions should be processed + verifyNoInteractions(queueFactory); + + ruleEngineConsumerContext.setReady(true); + await().atMost(2, TimeUnit.SECONDS) + .until(() -> consumers.size() == 3); + for (TopicPartitionInfo partition : partitions) { + TestConsumer consumer = getConsumer(partition); + verifySubscribedAndLaunched(consumer, Set.of(partition)); + } + } + + @Test + public void testInit_singleConsumer() { + queue.setConsumerPerPartition(false); + consumerManager.init(queue); + + Set partitions = createTpis(2, 3, 4); + consumerManager.update(partitions); + partitions = createTpis(3, 4, 5); + consumerManager.update(partitions); + partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + + verifyNoInteractions(queueFactory); + + ruleEngineConsumerContext.setReady(true); + await().atMost(2, TimeUnit.SECONDS) + .until(() -> consumers.size() == 1); + TestConsumer consumer = getConsumer(); + verifySubscribedAndLaunched(consumer, partitions); + } + + @Test + public void testPartitionsUpdate_singleConsumer() { + queue.setConsumerPerPartition(false); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + + Set partitions = Collections.emptySet(); + consumerManager.update(partitions); + verify(queueFactory, after(1000).never()).createToRuleEngineMsgConsumer(any()); + + partitions = createTpis(1); + consumerManager.update(partitions); + TestConsumer consumer = getConsumer(); + verifySubscribedAndLaunched(consumer, partitions); + + partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + verifySubscribedAndLaunched(consumer, partitions); + + partitions = createTpis(4, 5, 6); + consumerManager.update(partitions); + verifySubscribedAndLaunched(consumer, partitions); + + partitions = Collections.emptySet(); + consumerManager.update(partitions); + verifyUnsubscribedAndStopped(consumer); + + partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + consumer = getConsumer(); + verifySubscribedAndLaunched(consumer, partitions); + } + + @Test + public void testPartitionsUpdate_consumerPerPartition() { + queue.setConsumerPerPartition(true); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + + consumerManager.update(Collections.emptySet()); + verify(queueFactory, after(1000).never()).createToRuleEngineMsgConsumer(any()); + + consumerManager.update(createTpis(1)); + TestConsumer consumer1 = getConsumer(1); + verifySubscribedAndLaunched(consumer1, 1); + + consumerManager.update(createTpis(1, 2, 3)); + TestConsumer consumer2 = getConsumer(2); + TestConsumer consumer3 = getConsumer(3); + verifySubscribedAndLaunched(consumer2, 2); + verifySubscribedAndLaunched(consumer3, 3); + verifyNotTouched(consumer1); + + consumerManager.update(createTpis(3, 4, 5)); + TestConsumer consumer4 = getConsumer(4); + TestConsumer consumer5 = getConsumer(5); + verifySubscribedAndLaunched(consumer4, 4); + verifySubscribedAndLaunched(consumer5, 5); + verifyUnsubscribedAndStopped(consumer1); + verifyUnsubscribedAndStopped(consumer2); + verifyNotTouched(consumer3); + + consumerManager.update(Collections.emptySet()); + verifyUnsubscribedAndStopped(consumer3); + verifyUnsubscribedAndStopped(consumer4); + verifyUnsubscribedAndStopped(consumer5); + + consumerManager.update(createTpis(1, 2, 3)); + consumer1 = getConsumer(1); + consumer2 = getConsumer(2); + consumer3 = getConsumer(3); + verifySubscribedAndLaunched(consumer1, 1); + verifySubscribedAndLaunched(consumer2, 2); + verifySubscribedAndLaunched(consumer3, 3); + } + + @Test + public void testConfigUpdate_singleConsumer() { + queue.setConsumerPerPartition(false); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + Set partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + TestConsumer consumer = getConsumer(); + verifySubscribedAndLaunched(consumer, partitions); + + Queue newConfig = JacksonUtil.clone(queue); + newConfig.setPollInterval(queue.getPollInterval() / 2); + newConfig.setPartitions(queue.getPartitions() / 2); + newConfig.setPackProcessingTimeout(queue.getPackProcessingTimeout() * 2); + newConfig.getSubmitStrategy().setType(SubmitStrategyType.SEQUENTIAL_BY_ORIGINATOR); + newConfig.getProcessingStrategy().setType(ProcessingStrategyType.RETRY_ALL); + consumerManager.update(newConfig); + + await().atMost(2, TimeUnit.SECONDS) + .untilAsserted(() -> { + verify(consumer, atLeastOnce()).poll(eq((long) newConfig.getPollInterval())); + verify(ruleEngineConsumerContext.getSubmitStrategyFactory(), atLeastOnce()).newInstance(any(), eq(newConfig.getSubmitStrategy())); + verify(ruleEngineConsumerContext.getProcessingStrategyFactory(), atLeastOnce()).newInstance(any(), eq(newConfig.getProcessingStrategy())); + }); + } + + @Test + public void testConfigUpdate_consumerPerPartition() { + queue.setConsumerPerPartition(true); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + Set partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + TestConsumer consumer1 = getConsumer(1); + TestConsumer consumer2 = getConsumer(2); + TestConsumer consumer3 = getConsumer(3); + verifySubscribedAndLaunched(consumer1, 1); + verifySubscribedAndLaunched(consumer2, 2); + verifySubscribedAndLaunched(consumer3, 3); + + Queue newConfig = JacksonUtil.clone(queue); + newConfig.setPollInterval(queue.getPollInterval() / 2); + newConfig.setPartitions(queue.getPartitions() / 2); + newConfig.setPackProcessingTimeout(queue.getPackProcessingTimeout() * 2); + newConfig.getSubmitStrategy().setType(SubmitStrategyType.SEQUENTIAL_BY_ORIGINATOR); + newConfig.getProcessingStrategy().setType(ProcessingStrategyType.RETRY_ALL); + consumerManager.update(newConfig); + + await().atMost(2, TimeUnit.SECONDS) + .untilAsserted(() -> { + verify(consumer1, atLeastOnce()).poll(eq((long) newConfig.getPollInterval())); + verify(consumer2, atLeastOnce()).poll(eq((long) newConfig.getPollInterval())); + verify(consumer3, atLeastOnce()).poll(eq((long) newConfig.getPollInterval())); + }); + verifyNotTouched(consumer1); + verifyNotTouched(consumer2); + verifyNotTouched(consumer3); + } + + @Test + public void testConfigUpdate_fromSingleToConsumerPerPartition() { + queue.setConsumerPerPartition(false); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + Set partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + TestConsumer consumer = getConsumer(); + verifySubscribedAndLaunched(consumer, partitions); + + Queue newConfig = JacksonUtil.clone(queue); + newConfig.setConsumerPerPartition(true); + consumerManager.update(newConfig); + + verifyUnsubscribedAndStopped(consumer); + verifySubscribedAndLaunched(getConsumer(1), 1); + verifySubscribedAndLaunched(getConsumer(2), 2); + verifySubscribedAndLaunched(getConsumer(3), 3); + } + + @Test + public void testConfigUpdate_fromConsumerPerPartitionToSingle() { + queue.setConsumerPerPartition(true); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + Set partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + TestConsumer consumer1 = getConsumer(1); + TestConsumer consumer2 = getConsumer(2); + TestConsumer consumer3 = getConsumer(3); + verifySubscribedAndLaunched(consumer1, 1); + verifySubscribedAndLaunched(consumer2, 2); + verifySubscribedAndLaunched(consumer3, 3); + + Queue newConfig = JacksonUtil.clone(queue); + newConfig.setConsumerPerPartition(false); + consumerManager.update(newConfig); + + verifyUnsubscribedAndStopped(consumer1); + verifyUnsubscribedAndStopped(consumer2); + verifyUnsubscribedAndStopped(consumer3); + verifySubscribedAndLaunched(getConsumer(), partitions); + } + + @Test + public void testStop() { + queue.setConsumerPerPartition(true); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + consumerManager.update(createTpis(1)); + TestConsumer consumer = getConsumer(1); + verifySubscribedAndLaunched(consumer, 1); + verify(queueFactory, times(1)).createToRuleEngineMsgConsumer(any()); + + consumerManager.stop(); + consumerManager.update(createTpis(1, 2, 3, 4)); // to check that no new tasks after stop are processed + consumerManager.update(createTpis(5, 6, 7)); + + verifyUnsubscribedAndStopped(consumer); + verifyNoMoreInteractions(queueFactory); + } + + @Test + public void testDelete_consumerPerPartition() { + queue.setConsumerPerPartition(true); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + Set partitions = createTpis(1, 2); + consumerManager.update(partitions); + TestConsumer consumer1 = getConsumer(1); + TestConsumer consumer2 = getConsumer(2); + verifySubscribedAndLaunched(consumer1, 1); + verifySubscribedAndLaunched(consumer2, 2); + verifyMsgProcessed(consumer1.testMsg); + verifyMsgProcessed(consumer2.testMsg); + + consumerManager.delete(); + + await().atMost(2, TimeUnit.SECONDS) + .untilAsserted(() -> { + verify(ruleEngineMsgProducer).send(any(), any(), any()); + }); + clearInvocations(actorContext); + verify(consumer1, never()).unsubscribe(); + verify(consumer2, never()).unsubscribe(); + int msgCount1 = consumer1.msgCount; + int msgCount2 = consumer2.msgCount; + + await().atLeast(4, TimeUnit.SECONDS) // based on topicDeletionDelayInSec + .atMost(7, TimeUnit.SECONDS) + .untilAsserted(() -> { + partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .forEach(topic -> { + verify(queueAdmin).deleteTopic(eq(topic)); + }); + }); + verify(consumer1).unsubscribe(); + verify(consumer2).unsubscribe(); + + int movedMsgs1 = consumer1.msgCount - msgCount1; + int movedMsgs2 = consumer2.msgCount - msgCount2; + int totalMovedMsgs = movedMsgs1 + movedMsgs2; + assertThat(totalMovedMsgs).isGreaterThan(10); + verify(ruleEngineMsgProducer, atLeast(totalMovedMsgs)).send(any(), any(), any()); + verify(actorContext, never()).tell(any()); + generateQueueMsgs = false; + } + + @Test + public void testDelete_singleConsumer() { + queue.setConsumerPerPartition(false); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + Set partitions = createTpis(1, 2); + consumerManager.update(partitions); + TestConsumer consumer = getConsumer(); + verifySubscribedAndLaunched(consumer, partitions); + verifyMsgProcessed(consumer.testMsg); + + consumerManager.delete(); + + await().atMost(2, TimeUnit.SECONDS) + .untilAsserted(() -> { + verify(ruleEngineMsgProducer).send(any(), any(), any()); + }); + clearInvocations(actorContext); + verify(consumer, never()).unsubscribe(); + int msgCount = consumer.msgCount; + + await().atLeast(4, TimeUnit.SECONDS) + .atMost(7, TimeUnit.SECONDS) + .untilAsserted(() -> { + partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .forEach(topic -> { + verify(queueAdmin).deleteTopic(eq(topic)); + }); + }); + verify(consumer).unsubscribe(); + + int movedMsgs = consumer.msgCount - msgCount; + assertThat(movedMsgs).isGreaterThan(10); + verify(ruleEngineMsgProducer, atLeast(movedMsgs)).send(any(), any(), any()); + verify(actorContext, never()).tell(any()); + generateQueueMsgs = false; + } + + @Test + public void testManyDifferentUpdates() throws Exception { + queue.setConsumerPerPartition(RandomUtils.nextBoolean()); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + + Supplier queueConfigUpdater = () -> { + Queue oldConfig = consumerManager.getQueue(); + Queue newConfig = JacksonUtil.clone(oldConfig); + newConfig.setConsumerPerPartition(RandomUtils.nextBoolean()); + newConfig.setPollInterval(RandomUtils.nextInt(100, 501)); + newConfig.setPartitions(RandomUtils.nextInt(1, 10)); + newConfig.setPackProcessingTimeout(RandomUtils.nextLong(100, 5001)); + newConfig.getSubmitStrategy().setType(SubmitStrategyType.values()[RandomUtils.nextInt(0, SubmitStrategyType.values().length)]); + newConfig.getProcessingStrategy().setType(ProcessingStrategyType.values()[RandomUtils.nextInt(0, ProcessingStrategyType.values().length)]); + log.info("Generated new config: consumerPerPartition={}, pollInterval={}, processingStrategy={}", + newConfig.isConsumerPerPartition(), newConfig.getPollInterval(), newConfig.getProcessingStrategy().getType()); + return newConfig; + }; + Supplier> partitionsUpdater = () -> { + int partitionsCount = RandomUtils.nextInt(0, 20); + int[] partitions = IntStream.generate(() -> RandomUtils.nextInt(0, 20)) + .distinct().limit(partitionsCount) + .sorted().toArray(); + log.info("Generated new partitions: {}", Arrays.toString(partitions)); + return createTpis(partitions); + }; + + int iterations = 100; + Queue latestConfig = queue; + Set latestPartitions = Collections.emptySet(); + for (int i = 1; i <= iterations; i++) { + boolean updateQueueConfig = RandomUtils.nextBoolean(); + boolean updatePartitions = !updateQueueConfig; + if (updateQueueConfig) { + latestConfig = queueConfigUpdater.get(); + consumerManager.update(latestConfig); + } + if (updatePartitions) { + latestPartitions = partitionsUpdater.get(); + consumerManager.update(latestPartitions); + } + Thread.sleep(RandomUtils.nextLong(0, 200)); + } + if (latestPartitions.isEmpty()) { + do { + latestPartitions = partitionsUpdater.get(); + } while (latestPartitions.isEmpty()); + consumerManager.update(latestPartitions); + } + + Queue expectedConfig = latestConfig; + Set expectedPartitions = latestPartitions; + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + assertThat(consumerManager.getQueue()).isEqualTo(expectedConfig); + assertThat(consumerManager.getPartitions()).isEqualTo(expectedPartitions); + }); + + if (expectedConfig.isConsumerPerPartition()) { + await().atMost(5, TimeUnit.SECONDS).until(() -> { + for (TopicPartitionInfo partition : expectedPartitions) { + if (consumers.stream().noneMatch(consumer -> consumer.subscribed && + consumer.pollingStarted && Set.of(partition).equals(consumer.getPartitions()))) { + return false; + } + } + return consumers.size() == expectedPartitions.size(); + }); + } else { + await().atMost(5, TimeUnit.SECONDS).until(() -> { + return consumers.size() == 1 && consumers.stream() + .anyMatch(consumer -> consumer.subscribed && consumer.pollingStarted && + expectedPartitions.equals(consumer.getPartitions())); + }); + } + Mockito.reset(ruleEngineConsumerContext.getSubmitStrategyFactory()); + Mockito.reset(ruleEngineConsumerContext.getProcessingStrategyFactory()); + consumers.forEach(Mockito::clearInvocations); + + await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> { + for (TestConsumer consumer : consumers) { + verify(consumer, atLeastOnce().description("consumer " + consumer.topics)).poll(expectedConfig.getPollInterval()); + } + verify(ruleEngineConsumerContext.getSubmitStrategyFactory(), atLeastOnce()).newInstance(any(), eq(expectedConfig.getSubmitStrategy())); + verify(ruleEngineConsumerContext.getProcessingStrategyFactory(), atLeastOnce()).newInstance(any(), eq(expectedConfig.getProcessingStrategy())); + }); + } + + /* + * 2023-10-15 18:34:06,090 [main] INFO o.t.s.s.q.r.TbRuleEngineQueueConsumerManagerTest - Generated new partitions: [0, 1, 2, 3, 4, 5, 6, 8, 9, 11, 12, 13, 15, 16, 17, 18, 19] +2023-10-15 18:34:06,090 [main] INFO o.t.s.s.q.r.TbRuleEngineQueueConsumerManagerTest - Generated new config: consumerPerPartition=false, pollInterval=299, processingStrategy=RETRY_FAILED + * */ + + private void verifySubscribedAndLaunched(TestConsumer consumer, Set expectedPartitions) { + await().atMost(2, TimeUnit.SECONDS) + .until(() -> consumer.subscribed && consumer.getPartitions().equals(expectedPartitions) && consumer.pollingStarted); + verify(consumer, times(1)).subscribe(any()); + verify(consumer).subscribe(eq(expectedPartitions)); + verify(consumer).doSubscribe(argThat(topics -> topics.containsAll(expectedPartitions.stream() + .map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList())))); + verify(consumer, atLeastOnce()).poll(eq((long) queue.getPollInterval())); + verify(consumer, atLeastOnce()).doPoll(eq((long) queue.getPollInterval())); + verify(consumer, never()).unsubscribe(); + Mockito.reset(consumer); + } + + private void verifySubscribedAndLaunched(TestConsumer consumer, int... expectedPartitions) { + verifySubscribedAndLaunched(consumer, createTpis(expectedPartitions)); + } + + private void verifyUnsubscribedAndStopped(TestConsumer consumer) { + await().atMost(2, TimeUnit.SECONDS) + .until(() -> !consumer.subscribed && !consumer.topics.isEmpty()); + verify(consumer, never()).subscribe(any()); + verify(consumer, never()).doSubscribe(any()); + assertThat(consumers).doesNotContain(consumer); + Mockito.reset(consumer); + } + + private void verifyNotTouched(TestConsumer consumer) { + verify(consumer, never()).subscribe(any()); + verify(consumer, never()).subscribe(); + verify(consumer, never()).doSubscribe(any()); + verify(consumer, never()).unsubscribe(); + verify(consumer, never()).doUnsubscribe(); + } + + private void verifyMsgProcessed(TbMsg tbMsg) { + await().atMost(2, TimeUnit.SECONDS).untilAsserted(() -> { + verify(actorContext, atLeastOnce()).tell(argThat(msg -> { + return ((QueueToRuleEngineMsg) msg).getMsg().getId().equals(tbMsg.getId()); + })); + }); + } + + // for consumer-per-partition + private TestConsumer getConsumer(TopicPartitionInfo tpi) { + return await().atMost(5, TimeUnit.SECONDS) + .until(() -> consumers.stream() + .filter(consumer -> consumer.getPartitions() != null && + consumer.getPartitions().size() == 1 && + consumer.getPartitions().contains(tpi)) + .findFirst().orElse(null), Objects::nonNull); + } + + private TestConsumer getConsumer(int partition) { + return await().atMost(5, TimeUnit.SECONDS) + .until(() -> consumers.stream() + .filter(consumer -> consumer.getPartitions() != null && + consumer.getPartitions().size() == 1 && + consumer.getPartitions().stream() + .anyMatch(tpi -> tpi.getPartition().get().equals(partition))) + .findFirst().orElse(null), Objects::nonNull); + } + + // for single consumer + private TestConsumer getConsumer() { + return await().atMost(5, TimeUnit.SECONDS) + .until(() -> consumers.size() == 1 ? consumers.iterator().next() : null, Objects::nonNull); + } + + private Set createTpis(int... partitions) { + return Arrays.stream(partitions) + .mapToObj(n -> TopicPartitionInfo.builder() + .tenantId(queue.getTenantId()) + .topic(queue.getTopic()) + .partition(n) + .myPartition(true) + .build()) + .collect(Collectors.toSet()); + } + + + class TestConsumer extends AbstractTbQueueConsumerTemplate> { + + @Getter + private List topics; + + private boolean subscribed; + private boolean pollingStarted; + + private TbMsg testMsg; + private int msgCount; + + public TestConsumer(String topic) { + super(topic); + } + + @SneakyThrows + @Override + protected List doPoll(long durationInMillis) { + log.debug("doPoll({} ms)", durationInMillis); + if (!subscribed) { + throw new IllegalStateException("Cannot poll because not subscribed"); + } + pollingStarted = true; + if (testMsg != null && RandomUtils.nextBoolean()) { + Thread.sleep(100); + return List.of(testMsg); + } + return Collections.emptyList(); + } + + @Override + protected TbProtoQueueMsg decode(TbMsg tbMsg) throws IOException { + log.debug("decode()"); + UUID tenantId = UUID.randomUUID(); + return new TbProtoQueueMsg<>(UUID.randomUUID(), ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(tenantId.getMostSignificantBits()) + .setTenantIdLSB(tenantId.getLeastSignificantBits()) + .addRelationTypes("Success") + .setTbMsg(TbMsg.toByteString(tbMsg)) + .build()); + } + + @Override + protected void doSubscribe(List topicNames) { + log.debug("doSubscribe({})", topicNames); + this.topics = topicNames; + subscribed = true; + } + + @Override + protected void doCommit() { + if (!subscribed) { + throw new IllegalStateException("Cannot commit because not subscribed"); + } + msgCount++; + log.debug("doCommit() totalConsumedMsgs = {}", totalConsumedMsgs.incrementAndGet()); + } + + @Override + public void unsubscribe() { + super.unsubscribe(); + consumers.remove(this); + } + + @Override + protected void doUnsubscribe() { + log.debug("doUnsubscribe()"); + if (!subscribed) { + throw new IllegalStateException("Already unsubscribed!"); + } + subscribed = false; + } + + @Override + protected boolean isLongPollingSupported() { + return false; + } + + public Set getPartitions() { + return partitions; + } + + public void setUpTestMsg() { + testMsg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, new DeviceId(UUID.randomUUID()), new TbMsgMetaData(), "{}"); + } + } + +} From ea4eadb47f9184326f04f05e5461744376368d06 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 18 Oct 2023 14:20:41 +0300 Subject: [PATCH 12/23] Cancelling consumer loop task on stop --- .../DefaultTbRuleEngineConsumerService.java | 1 + .../service/queue/ruleengine/QueueEvent.java | 2 +- .../queue/ruleengine/TbQueueConsumerTask.java | 34 +++++++-- .../TbRuleEngineConsumerContext.java | 10 +-- .../TbRuleEngineQueueConsumerManager.java | 70 +++++++++++-------- .../TbRuleEngineQueueConsumerManagerTest.java | 10 +-- 6 files changed, 78 insertions(+), 49 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 64139530b1..3f05a5b923 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -126,6 +126,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @Override protected void stopConsumers() { consumers.values().forEach(TbRuleEngineQueueConsumerManager::stop); + consumers.values().forEach(TbRuleEngineQueueConsumerManager::awaitStop); ctx.stop(); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java index a4a3bcc59a..86955be7e7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java @@ -19,6 +19,6 @@ import java.io.Serializable; public enum QueueEvent implements Serializable { - PARTITION_CHANGE, CONFIG_UPDATE, STOP, DELETE + PARTITION_CHANGE, CONFIG_UPDATE, DELETE } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java index a0a5579951..2e3a4676e8 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java @@ -15,7 +15,8 @@ */ package org.thingsboard.server.service.queue.ruleengine; -import lombok.Data; +import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; @@ -23,16 +24,26 @@ import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import java.util.Set; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -@Data +@RequiredArgsConstructor @Slf4j public class TbQueueConsumerTask { + @Getter private final Object key; + @Getter private final TbQueueConsumer> consumer; - private volatile Future task; + + private Future task; + private CountDownLatch completionLatch; + + public void setTask(Future task) { + this.task = task; + this.completionLatch = new CountDownLatch(1); + } public void subscribe(Set partitions) { log.trace("[{}] Subscribing to partitions: {}", key, partitions); @@ -42,23 +53,32 @@ public class TbQueueConsumerTask { public void initiateStop() { log.debug("[{}] Initiating stop", key); consumer.stop(); + if (isRunning()) { + task.cancel(true); + } } - public void awaitFinish() { + public void awaitCompletion() { log.trace("[{}] Awaiting finish", key); if (isRunning()) { try { - task.get(60, TimeUnit.SECONDS); - task = null; + if (!completionLatch.await(30, TimeUnit.SECONDS)) { + throw new IllegalStateException("timeout of 30 seconds expired"); + } } catch (Exception e) { log.warn("[{}] Failed to await for consumer to stop", key, e); } + task = null; } log.trace("[{}] Awaited finish", key); } public boolean isRunning() { - return task != null && !task.isDone(); + return task != null; + } + + public void finished() { + completionLatch.countDown(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java index 325a685972..f4c8e4c2ae 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java @@ -36,7 +36,6 @@ import javax.annotation.PostConstruct; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; @Component @TbRuleEngineComponent @@ -83,12 +82,7 @@ public class TbRuleEngineConsumerContext { public void stop() { scheduler.shutdownNow(); - consumersExecutor.shutdown(); - mgmtExecutor.shutdown(); - try { - mgmtExecutor.awaitTermination(15, TimeUnit.SECONDS); - } catch (InterruptedException e) { - log.warn("Failed to await mgmtExecutor termination"); - } + consumersExecutor.shutdownNow(); + mgmtExecutor.shutdownNow(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index dc3966d0d2..3a7b2f15d8 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -85,7 +85,13 @@ public class TbRuleEngineQueueConsumerManager { } public void init(Queue queue) { - doInit(queue); + this.queue = queue; + if (queue.isConsumerPerPartition()) { + this.consumerWrapper = new ConsumerPerPartitionWrapper(); + } else { + this.consumerWrapper = new SingleConsumerWrapper(); + } + log.debug("[{}] Initialized consumer for queue: {}", queueKey, queue); } public void update(Queue queue) { @@ -96,10 +102,6 @@ public class TbRuleEngineQueueConsumerManager { addTask(new TbQueueConsumerManagerTask(QueueEvent.PARTITION_CHANGE, partitions)); } - public void stop() { - addTask(new TbQueueConsumerManagerTask(QueueEvent.STOP)); - } - public void delete() { addTask(new TbQueueConsumerManagerTask(QueueEvent.DELETE)); } @@ -135,20 +137,22 @@ public class TbRuleEngineQueueConsumerManager { newPartitions = task.getPartitions(); } else if (task.getEvent() == QueueEvent.CONFIG_UPDATE) { newConfiguration = task.getQueue(); - } else if (task.getEvent() == QueueEvent.STOP) { - doStop(); - return; } else if (task.getEvent() == QueueEvent.DELETE) { doDelete(); return; } } + if (stopped) { + return; + } if (newConfiguration != null) { doUpdate(newConfiguration); } if (newPartitions != null) { doUpdate(newPartitions); } + } catch (Exception e) { + log.error("[{}] Failed to process tasks", queueKey, e); } finally { lock.unlock(); } @@ -159,16 +163,6 @@ public class TbRuleEngineQueueConsumerManager { }); } - private void doInit(Queue queue) { - this.queue = queue; - if (queue.isConsumerPerPartition()) { - consumerWrapper = new ConsumerPerPartitionWrapper(); - } else { - consumerWrapper = new SingleConsumerWrapper(); - } - log.debug("[{}] Initialized consumer for queue: {}", queueKey, queue); - } - private void doUpdate(Queue newQueue) { log.info("[{}] Processing queue update: {}", queueKey, newQueue); var oldQueue = this.queue; @@ -179,12 +173,12 @@ public class TbRuleEngineQueueConsumerManager { } if (oldQueue == null) { - doInit(queue); + init(queue); } else if (newQueue.isConsumerPerPartition() != oldQueue.isConsumerPerPartition()) { consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::initiateStop); - consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitFinish); + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitCompletion); - doInit(queue); + init(queue); if (partitions != null) { doUpdate(partitions); // even if partitions number was changed, there can be no partition change event } @@ -200,17 +194,21 @@ public class TbRuleEngineQueueConsumerManager { consumerWrapper.updatePartitions(partitions); } - private void doStop() { + public void stop() { + log.debug("[{}] Stopping consumers", queueKey); consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::initiateStop); - consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitFinish); - log.debug("[{}] Unsubscribed and stopped consumers", queueKey); stopped = true; } + public void awaitStop() { + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitCompletion); + log.debug("[{}] Unsubscribed and stopped consumers", queueKey); + } + private void doDelete() { stopped = true; log.info("[{}] Handling queue deletion", queueKey); - consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitFinish); + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitCompletion); List>> queueConsumers = consumerWrapper.getConsumers().stream() .map(TbQueueConsumerTask::getConsumer).collect(Collectors.toList()); @@ -240,6 +238,7 @@ public class TbRuleEngineQueueConsumerManager { Future consumerLoop = ctx.getConsumersExecutor().submit(() -> { ThingsBoardThreadFactory.updateCurrentThreadName(consumerTask.getKey().toString()); consumerLoop(consumerTask.getConsumer()); + consumerTask.finished(); }); consumerTask.setTask(consumerLoop); } @@ -253,7 +252,7 @@ public class TbRuleEngineQueueConsumerManager { } processMsgs(msgs, consumer, queue); } catch (Exception e) { - if (!stopped) { + if (!consumer.isStopped()) { log.warn("Failed to process messages from queue", e); try { Thread.sleep(ctx.getPollDuration()); @@ -279,7 +278,7 @@ public class TbRuleEngineQueueConsumerManager { TbMsgPackProcessingContext packCtx = new TbMsgPackProcessingContext(queue.getName(), submitStrategy, ackStrategy.isSkipTimeoutMsgs()); submitStrategy.submitAttempt((id, msg) -> submitMessage(packCtx, id, msg)); - final boolean timeout = !packCtx.await(queue.getPackProcessingTimeout(), TimeUnit.MILLISECONDS); + final boolean timeout = !awaitPackProcessing(packCtx, queue.getPackProcessingTimeout(), true); TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(queue.getName(), timeout, packCtx); if (timeout) { @@ -307,6 +306,19 @@ public class TbRuleEngineQueueConsumerManager { } } + private boolean awaitPackProcessing(TbMsgPackProcessingContext packCtx, long processingTimeout, boolean ignoreInterrupt) throws InterruptedException { + try { + return packCtx.await(processingTimeout, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + if (ignoreInterrupt) { + log.debug("Interrupt happened while waiting for pack processing, trying to await one more time"); + return awaitPackProcessing(packCtx, processingTimeout, false); + } else { + throw new RuntimeException("Failed to await pack processing due to thread interrupt", e); + } + } + } + private TbRuleEngineSubmitStrategy getSubmitStrategy(Queue queue) { return ctx.getSubmitStrategyFactory().newInstance(queue.getName(), queue.getSubmitStrategy()); } @@ -430,7 +442,7 @@ public class TbRuleEngineQueueConsumerManager { consumers.get(tpi).initiateStop(); }); removedPartitions.forEach((tpi) -> { - consumers.remove(tpi).awaitFinish(); + consumers.remove(tpi).awaitCompletion(); }); addedPartitions.forEach((tpi) -> { @@ -457,7 +469,7 @@ public class TbRuleEngineQueueConsumerManager { if (partitions.isEmpty()) { if (consumer != null && consumer.isRunning()) { consumer.initiateStop(); - consumer.awaitFinish(); + consumer.awaitCompletion(); } consumer = null; return; diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java index 276e65ec39..20b765da6d 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java @@ -180,13 +180,16 @@ public class TbRuleEngineQueueConsumerManagerTest { @After public void afterEach() { consumerManager.stop(); + consumerManager.awaitStop(); ruleEngineConsumerContext.stop(); if (generateQueueMsgs) { - await().atMost(1, TimeUnit.SECONDS) - .until(() -> totalProcessedMsgs.get() == totalConsumedMsgs.get()); + await().atMost(2, TimeUnit.SECONDS) + .untilAsserted(() -> { + log.debug("totalConsumedMsgs = {}, totalProcessedMsgs = {}", totalConsumedMsgs.get(), totalProcessedMsgs.get()); + assertThat(totalProcessedMsgs.get()).isEqualTo(totalConsumedMsgs.get()); + }); } - log.debug("totalConsumedMsgs = {}, totalProcessedMsgs = {}", totalConsumedMsgs.get(), totalProcessedMsgs.get()); } @Test @@ -510,7 +513,6 @@ public class TbRuleEngineQueueConsumerManagerTest { verify(consumer).unsubscribe(); int movedMsgs = consumer.msgCount - msgCount; - assertThat(movedMsgs).isGreaterThan(10); verify(ruleEngineMsgProducer, atLeast(movedMsgs)).send(any(), any(), any()); verify(actorContext, never()).tell(any()); generateQueueMsgs = false; From 5108e98f38631ff4f828cb014965b4836ef475a2 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 19 Oct 2023 11:03:53 +0300 Subject: [PATCH 13/23] Minor refactoring --- .../service/queue/ruleengine/TbRuleEngineConsumerContext.java | 2 +- application/src/main/resources/thingsboard.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java index f4c8e4c2ae..b18df6b62b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java @@ -74,7 +74,7 @@ public class TbRuleEngineConsumerContext { private volatile boolean isReady = false; @PostConstruct - public void init() { + void init() { this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer")); this.mgmtExecutor = Executors.newFixedThreadPool(mgmtThreadPoolSize, ThingsBoardThreadFactory.forName("tb-rule-engine-mgmt")); this.scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-scheduler")); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 6f51413ccd..be01c95fad 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1304,7 +1304,7 @@ queue: max-pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}" # Max allowed time in seconds for pause between retries. # After a queue is deleted (or profile's isolation option was disabled), Rule Engine will continue reading related topics during this period, before deleting the actual topics topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:15}" - # Size of the thread pool that handles management operations such as subscribe, unsubscribe, queue delete, etc. + # Size of the thread pool that handles such operations as partition changes, config updates, queue deletion management-thread-pool-size: "${TB_QUEUE_RULE_ENGINE_MGMT_THREAD_POOL_SIZE:12}" transport: # For high priority notifications that require minimum latency and processing time From 049f40a62e4cba6ffcd863eaada4a0ad6a7f360d Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 19 Oct 2023 14:00:04 +0300 Subject: [PATCH 14/23] Clear interruption status after exiting consumer while loop --- .../service/queue/ruleengine/TbQueueConsumerTask.java | 7 ++++--- .../ruleengine/TbRuleEngineQueueConsumerManager.java | 8 ++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java index 2e3a4676e8..9c0df63ad5 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java @@ -41,8 +41,8 @@ public class TbQueueConsumerTask { private CountDownLatch completionLatch; public void setTask(Future task) { - this.task = task; this.completionLatch = new CountDownLatch(1); + this.task = task; } public void subscribe(Set partitions) { @@ -63,14 +63,14 @@ public class TbQueueConsumerTask { if (isRunning()) { try { if (!completionLatch.await(30, TimeUnit.SECONDS)) { + task = null; throw new IllegalStateException("timeout of 30 seconds expired"); } + log.trace("[{}] Awaited finish", key); } catch (Exception e) { log.warn("[{}] Failed to await for consumer to stop", key, e); } - task = null; } - log.trace("[{}] Awaited finish", key); } public boolean isRunning() { @@ -79,6 +79,7 @@ public class TbQueueConsumerTask { public void finished() { completionLatch.countDown(); + task = null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index 3a7b2f15d8..71a7c45d45 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -237,7 +237,11 @@ public class TbRuleEngineQueueConsumerManager { log.info("[{}] Launching consumer", consumerTask.getKey()); Future consumerLoop = ctx.getConsumersExecutor().submit(() -> { ThingsBoardThreadFactory.updateCurrentThreadName(consumerTask.getKey().toString()); - consumerLoop(consumerTask.getConsumer()); + try { + consumerLoop(consumerTask.getConsumer()); + } catch (Throwable e) { + log.error("Failure in consumer loop", e); + } consumerTask.finished(); }); consumerTask.setTask(consumerLoop); @@ -262,7 +266,7 @@ public class TbRuleEngineQueueConsumerManager { } } } - if (consumer.isStopped()) { + if (Thread.interrupted() || consumer.isStopped()) { consumer.unsubscribe(); } log.info("Rule Engine consumer stopped"); From 2d1640a0025798cafb3a967562c3773978b936d5 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 19 Oct 2023 15:29:46 +0300 Subject: [PATCH 15/23] Don't interrupt consumer loop task on stop --- .../queue/ruleengine/TbQueueConsumerTask.java | 23 ++++--------------- .../TbRuleEngineQueueConsumerManager.java | 18 ++------------- 2 files changed, 6 insertions(+), 35 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java index 9c0df63ad5..59d55285f0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.queue.ruleengine; import lombok.Getter; import lombok.RequiredArgsConstructor; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; @@ -24,7 +25,6 @@ import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import java.util.Set; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -37,13 +37,8 @@ public class TbQueueConsumerTask { @Getter private final TbQueueConsumer> consumer; + @Setter private Future task; - private CountDownLatch completionLatch; - - public void setTask(Future task) { - this.completionLatch = new CountDownLatch(1); - this.task = task; - } public void subscribe(Set partitions) { log.trace("[{}] Subscribing to partitions: {}", key, partitions); @@ -53,23 +48,18 @@ public class TbQueueConsumerTask { public void initiateStop() { log.debug("[{}] Initiating stop", key); consumer.stop(); - if (isRunning()) { - task.cancel(true); - } } public void awaitCompletion() { log.trace("[{}] Awaiting finish", key); if (isRunning()) { try { - if (!completionLatch.await(30, TimeUnit.SECONDS)) { - task = null; - throw new IllegalStateException("timeout of 30 seconds expired"); - } + task.get(30, TimeUnit.SECONDS); log.trace("[{}] Awaited finish", key); } catch (Exception e) { log.warn("[{}] Failed to await for consumer to stop", key, e); } + task = null; } } @@ -77,9 +67,4 @@ public class TbQueueConsumerTask { return task != null; } - public void finished() { - completionLatch.countDown(); - task = null; - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index 71a7c45d45..5a59cb124e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -242,7 +242,6 @@ public class TbRuleEngineQueueConsumerManager { } catch (Throwable e) { log.error("Failure in consumer loop", e); } - consumerTask.finished(); }); consumerTask.setTask(consumerLoop); } @@ -266,7 +265,7 @@ public class TbRuleEngineQueueConsumerManager { } } } - if (Thread.interrupted() || consumer.isStopped()) { + if (consumer.isStopped()) { consumer.unsubscribe(); } log.info("Rule Engine consumer stopped"); @@ -282,7 +281,7 @@ public class TbRuleEngineQueueConsumerManager { TbMsgPackProcessingContext packCtx = new TbMsgPackProcessingContext(queue.getName(), submitStrategy, ackStrategy.isSkipTimeoutMsgs()); submitStrategy.submitAttempt((id, msg) -> submitMessage(packCtx, id, msg)); - final boolean timeout = !awaitPackProcessing(packCtx, queue.getPackProcessingTimeout(), true); + final boolean timeout = !packCtx.await(queue.getPackProcessingTimeout(), TimeUnit.MILLISECONDS); TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(queue.getName(), timeout, packCtx); if (timeout) { @@ -310,19 +309,6 @@ public class TbRuleEngineQueueConsumerManager { } } - private boolean awaitPackProcessing(TbMsgPackProcessingContext packCtx, long processingTimeout, boolean ignoreInterrupt) throws InterruptedException { - try { - return packCtx.await(processingTimeout, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - if (ignoreInterrupt) { - log.debug("Interrupt happened while waiting for pack processing, trying to await one more time"); - return awaitPackProcessing(packCtx, processingTimeout, false); - } else { - throw new RuntimeException("Failed to await pack processing due to thread interrupt", e); - } - } - } - private TbRuleEngineSubmitStrategy getSubmitStrategy(Queue queue) { return ctx.getSubmitStrategyFactory().newInstance(queue.getName(), queue.getSubmitStrategy()); } From be7854fb5baee31af0a9455c749755c80134b17e Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 19 Oct 2023 18:58:41 +0300 Subject: [PATCH 16/23] Add 'isolated' to topic and consumer group name for isolated queues --- .../thingsboard/server/common/msg/queue/TopicPartitionInfo.java | 2 +- .../server/queue/provider/KafkaMonolithQueueFactory.java | 2 +- .../server/queue/provider/KafkaTbRuleEngineQueueFactory.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java index 7122d5ef2a..d2144e8111 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -42,7 +42,7 @@ public class TopicPartitionInfo { this.myPartition = myPartition; String tmp = topic; if (tenantId != null && !tenantId.isNullUid()) { - tmp += "." + tenantId.getId().toString(); + tmp += ".isolated." + tenantId.getId().toString(); } if (partition != null) { tmp += "." + partition; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 22a16de64f..55050ce08c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -187,7 +187,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(configuration.getTopic()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId("re-" + queueName + (configuration.getTenantId().isSysTenantId() ? "" : ("-" + configuration.getTenantId())) + "-consumer"); + consumerBuilder.groupId("re-" + queueName + (configuration.getTenantId().isSysTenantId() ? "" : ("-isolated-" + configuration.getTenantId())) + "-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(ruleEngineAdmin); consumerBuilder.statsService(consumerStatsService); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 2e3bf784d7..cb063b6a7f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -166,7 +166,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(configuration.getTopic()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId("re-" + queueName + (configuration.getTenantId().isSysTenantId() ? "" : ("-" + configuration.getTenantId())) + "-consumer"); + consumerBuilder.groupId("re-" + queueName + (configuration.getTenantId().isSysTenantId() ? "" : ("-isolated-" + configuration.getTenantId())) + "-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(ruleEngineAdmin); consumerBuilder.statsService(consumerStatsService); From 23cd7f08654fddf66b6d3fa7f882f0e0ce50e336 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 20 Oct 2023 15:03:24 +0300 Subject: [PATCH 17/23] Fix testNotificationRuleProcessing_exceededRateLimits --- .../notification/NotificationRuleApiTest.java | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java index 0a2854a28d..33aaf2af35 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java @@ -420,11 +420,11 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { notificationRulesCache.evict(TenantId.SYS_TENANT_ID); int n = 10; - updateDefaultTenantProfile(profileConfiguration -> { - profileConfiguration.getProfileConfiguration().get().setTenantEntityExportRateLimit(n + ":600"); - profileConfiguration.getProfileConfiguration().get().setCustomerServerRestLimitsConfiguration(n + ":600"); - profileConfiguration.getProfileConfiguration().get().setTenantNotificationRequestsPerRuleRateLimit(n + ":600"); - profileConfiguration.getProfileConfiguration().get().setTransportDeviceTelemetryMsgRateLimit(n + ":600"); + updateDefaultTenantProfileConfig(profileConfiguration -> { + profileConfiguration.setTenantEntityExportRateLimit(n + ":600"); + profileConfiguration.setCustomerServerRestLimitsConfiguration(n + ":600"); + profileConfiguration.setTenantNotificationRequestsPerRuleRateLimit(n + ":600"); + profileConfiguration.setTransportDeviceTelemetryMsgRateLimit(n + ":600"); }); loginTenantAdmin(); NotificationRule rule = createNotificationRule(AlarmCommentNotificationRuleTriggerConfig.builder() @@ -434,11 +434,14 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { rateLimitService.checkRateLimit(LimitedApi.ENTITY_EXPORT, tenantId); rateLimitService.checkRateLimit(LimitedApi.REST_REQUESTS_PER_CUSTOMER, tenantId, customerId); rateLimitService.checkRateLimit(LimitedApi.NOTIFICATION_REQUESTS_PER_RULE, tenantId, rule.getId()); + Thread.sleep(100); } loginTenantAdmin(); - List notifications = await().atMost(30, TimeUnit.SECONDS) - .until(() -> getMyNotifications(true, 10), list -> list.size() == 3); + List notifications = await().atMost(15, TimeUnit.SECONDS) + .until(() -> getMyNotifications(true, 10).stream() + .filter(notification -> notification.getType() == NotificationType.RATE_LIMITS) + .collect(Collectors.toList()), list -> list.size() == 3); assertThat(notifications).allSatisfy(notification -> { assertThat(notification.getSubject()).isEqualTo("Rate limits exceeded"); }); @@ -455,12 +458,14 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { }); loginSysAdmin(); - notifications = await().atMost(30, TimeUnit.SECONDS) - .until(() -> getMyNotifications(true, 10), list -> list.size() == 1); - assertThat(notifications).allSatisfy(notification -> { + notifications = await().atMost(15, TimeUnit.SECONDS) + .until(() -> getMyNotifications(true, 10).stream() + .filter(notification -> notification.getType() == NotificationType.RATE_LIMITS) + .collect(Collectors.toList()), list -> list.size() == 1); + assertThat(notifications).singleElement().satisfies(notification -> { assertThat(notification.getSubject()).isEqualTo("Rate limits exceeded for tenant " + TEST_TENANT_NAME); + assertThat(notification.getText()).isEqualTo("Rate limits for entity version creation exceeded"); }); - assertThat(notifications.get(0).getText()).isEqualTo("Rate limits for entity version creation exceeded"); } @Test From aea4d267ffebaa8dca0ffe4b11a1336a4cb35df3 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 20 Oct 2023 15:26:25 +0300 Subject: [PATCH 18/23] Minor improvements --- .../queue/ruleengine/TbRuleEngineConsumerContext.java | 3 ++- .../home/components/profile/tenant-profile.component.ts | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java index b18df6b62b..da2a5d0db4 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java @@ -19,6 +19,7 @@ import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.stats.StatsFactory; @@ -76,7 +77,7 @@ public class TbRuleEngineConsumerContext { @PostConstruct void init() { this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer")); - this.mgmtExecutor = Executors.newFixedThreadPool(mgmtThreadPoolSize, ThingsBoardThreadFactory.forName("tb-rule-engine-mgmt")); + this.mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(mgmtThreadPoolSize, "tb-rule-engine-mgmt"); this.scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-scheduler")); } diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts index 26d6611a3c..3f048ab2e3 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts @@ -56,7 +56,7 @@ export class TenantProfileComponent extends EntityComponent { const mainQueue = [ { id: guid(), - consumerPerPartition: true, + consumerPerPartition: false, name: 'Main', packProcessingTimeout: 10000, partitions: 1, @@ -84,7 +84,7 @@ export class TenantProfileComponent extends EntityComponent { topic: 'tb_rule_engine.hp', pollInterval: 2000, partitions: 1, - consumerPerPartition: true, + consumerPerPartition: false, packProcessingTimeout: 10000, submitStrategy: { type: 'BURST', @@ -108,7 +108,7 @@ export class TenantProfileComponent extends EntityComponent { topic: 'tb_rule_engine.sq', pollInterval: 2000, partitions: 1, - consumerPerPartition: true, + consumerPerPartition: false, packProcessingTimeout: 10000, submitStrategy: { type: 'SEQUENTIAL_BY_ORIGINATOR', From 6d7cb95492c1a90fc55dab13a5f3d95a563bf690 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 20 Oct 2023 15:46:53 +0300 Subject: [PATCH 19/23] Bigger timeout in tests for msg processing --- .../queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java index 20b765da6d..38582d0fce 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java @@ -184,7 +184,7 @@ public class TbRuleEngineQueueConsumerManagerTest { ruleEngineConsumerContext.stop(); if (generateQueueMsgs) { - await().atMost(2, TimeUnit.SECONDS) + await().atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> { log.debug("totalConsumedMsgs = {}, totalProcessedMsgs = {}", totalConsumedMsgs.get(), totalProcessedMsgs.get()); assertThat(totalProcessedMsgs.get()).isEqualTo(totalConsumedMsgs.get()); From 802890cdc0b99703f1431dad5c4e5ec643dc130c Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Mon, 23 Oct 2023 19:37:38 +0300 Subject: [PATCH 20/23] Fixed testIsolatedTenantProfile - correct expected number of downlink msgs --- .../server/edge/TenantProfileEdgeTest.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java index 7ec32f8aff..6f665ce462 100644 --- a/application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java @@ -32,6 +32,7 @@ import org.thingsboard.server.gen.edge.v1.TenantUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import java.util.List; +import java.util.Optional; import static org.assertj.core.api.Assertions.assertThat; @@ -75,17 +76,21 @@ public class TenantProfileEdgeTest extends AbstractEdgeTest { TenantProfileQueueConfiguration mainQueueConfiguration = createQueueConfig(DataConstants.MAIN_QUEUE_NAME, DataConstants.MAIN_QUEUE_TOPIC); TenantProfileQueueConfiguration isolatedQueueConfiguration = createQueueConfig("IsolatedHighPriority", "tb_rule_engine.isolated_hp"); edgeTenantProfile.getProfileData().setQueueConfiguration(List.of(mainQueueConfiguration, isolatedQueueConfiguration)); - edgeImitator.expectMessageAmount(1); + edgeImitator.expectMessageAmount(3); edgeTenantProfile = doPost("/api/tenantProfile", edgeTenantProfile, TenantProfile.class); Assert.assertTrue(edgeImitator.waitForMessages()); - AbstractMessage latestMessage = edgeImitator.getLatestMessage(); - Assert.assertTrue(latestMessage instanceof TenantProfileUpdateMsg); - TenantProfileUpdateMsg tenantProfileUpdateMsg = (TenantProfileUpdateMsg) latestMessage; + + Optional tenantProfileUpdateMsgOpt = edgeImitator.findMessageByType(TenantProfileUpdateMsg.class); + Assert.assertTrue(tenantProfileUpdateMsgOpt.isPresent()); + TenantProfileUpdateMsg tenantProfileUpdateMsg = tenantProfileUpdateMsgOpt.get(); Assert.assertEquals(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE, tenantProfileUpdateMsg.getMsgType()); Assert.assertEquals(edgeTenantProfile.getUuidId().getMostSignificantBits(), tenantProfileUpdateMsg.getIdMSB()); Assert.assertEquals(edgeTenantProfile.getUuidId().getLeastSignificantBits(), tenantProfileUpdateMsg.getIdLSB()); Assert.assertEquals(edgeTenantProfile.getDescription(), tenantProfileUpdateMsg.getDescription()); + List queueUpdateMsgs = edgeImitator.findAllMessagesByType(QueueUpdateMsg.class); + Assert.assertEquals(2, queueUpdateMsgs.size()); + loginTenantAdmin(); edgeImitator.expectMessageAmount(21); @@ -95,7 +100,7 @@ public class TenantProfileEdgeTest extends AbstractEdgeTest { Assert.assertTrue(edgeImitator.getDownlinkMsgs().get(0) instanceof TenantUpdateMsg); Assert.assertTrue(edgeImitator.getDownlinkMsgs().get(1) instanceof TenantProfileUpdateMsg); - List queueUpdateMsgs = edgeImitator.findAllMessagesByType(QueueUpdateMsg.class); + queueUpdateMsgs = edgeImitator.findAllMessagesByType(QueueUpdateMsg.class); Assert.assertEquals(2, queueUpdateMsgs.size()); for (QueueUpdateMsg queueUpdateMsg : queueUpdateMsgs) { Assert.assertEquals(tenantId.getId().getMostSignificantBits(), queueUpdateMsg.getTenantIdMSB()); From 0571f7e6ddbe62605bc703cfbecfa4c177e2411a Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 24 Oct 2023 12:24:01 +0300 Subject: [PATCH 21/23] Fix compatibility between CE and PE --- .../server/service/queue/ProtoUtils.java | 22 ++++++- .../server/service/queue/ProtoUtilsTest.java | 40 +++--------- common/cluster-api/src/main/proto/queue.proto | 60 +++++++++--------- .../server/common/data/EntityType.java | 63 ++++++++++--------- .../msg/plugin/ComponentLifecycleMsg.java | 2 + 5 files changed, 98 insertions(+), 89 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java b/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java index 3406265747..e39949e891 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java @@ -15,33 +15,51 @@ */ package org.thingsboard.server.service.queue; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.gen.transport.TransportProtos; +import java.util.Arrays; import java.util.UUID; public class ProtoUtils { + private static final EntityType[] entityTypeByProtoNumber; + + static { + int arraySize = Arrays.stream(EntityType.values()).mapToInt(EntityType::getProtoNumber).max().orElse(0); + entityTypeByProtoNumber = new EntityType[arraySize + 1]; + Arrays.stream(EntityType.values()).forEach(entityType -> entityTypeByProtoNumber[entityType.getProtoNumber()] = entityType); + } + public static TransportProtos.ComponentLifecycleMsgProto toProto(ComponentLifecycleMsg msg) { return TransportProtos.ComponentLifecycleMsgProto.newBuilder() .setTenantIdMSB(msg.getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(msg.getTenantId().getId().getLeastSignificantBits()) - .setEntityType(TransportProtos.EntityType.forNumber(msg.getEntityId().getEntityType().ordinal())) + .setEntityType(toProto(msg.getEntityId().getEntityType())) .setEntityIdMSB(msg.getEntityId().getId().getMostSignificantBits()) .setEntityIdLSB(msg.getEntityId().getId().getLeastSignificantBits()) .setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(msg.getEvent().ordinal())) .build(); } + public static TransportProtos.EntityTypeProto toProto(EntityType entityType) { + return TransportProtos.EntityTypeProto.forNumber(entityType.getProtoNumber()); + } + public static ComponentLifecycleMsg fromProto(TransportProtos.ComponentLifecycleMsgProto proto) { return new ComponentLifecycleMsg( TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), - EntityIdFactory.getByTypeAndUuid(proto.getEntityTypeValue(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())), + EntityIdFactory.getByTypeAndUuid(fromProto(proto.getEntityType()), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())), ComponentLifecycleEvent.values()[proto.getEventValue()] ); } + public static EntityType fromProto(TransportProtos.EntityTypeProto entityType) { + return entityTypeByProtoNumber[entityType.getNumber()]; + } + } diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ProtoUtilsTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ProtoUtilsTest.java index c63055a6f1..e235076635 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ProtoUtilsTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ProtoUtilsTest.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.queue; import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; @@ -31,41 +32,18 @@ class ProtoUtilsTest { TenantId tenantId = TenantId.fromUUID(UUID.fromString("35e10f77-16e7-424d-ae46-ee780f87ac4f")); EntityId entityId = new RuleChainId(UUID.fromString("c640b635-4f0f-41e6-b10b-25a86003094e")); @Test - void toProtoComponentLifecycleMsg() { + void protoComponentLifecycleSerialization() { ComponentLifecycleMsg msg = new ComponentLifecycleMsg(tenantId, entityId, ComponentLifecycleEvent.UPDATED); - - TransportProtos.ComponentLifecycleMsgProto proto = ProtoUtils.toProto(msg); - - assertThat(proto).as("to proto").isEqualTo(TransportProtos.ComponentLifecycleMsgProto.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setEntityType(TransportProtos.EntityType.forNumber(entityId.getEntityType().ordinal())) - .setEntityIdMSB(entityId.getId().getMostSignificantBits()) - .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) - .setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(ComponentLifecycleEvent.UPDATED.ordinal())) - .build() - ); - - assertThat(ProtoUtils.fromProto(proto)).as("from proto").isEqualTo(msg); + assertThat(ProtoUtils.fromProto(ProtoUtils.toProto(msg))).as("deserialized").isEqualTo(msg); + msg = new ComponentLifecycleMsg(tenantId, entityId, ComponentLifecycleEvent.STARTED); + assertThat(ProtoUtils.fromProto(ProtoUtils.toProto(msg))).as("deserialized").isEqualTo(msg); } @Test - void fromProtoComponentLifecycleMsg() { - TransportProtos.ComponentLifecycleMsgProto proto = TransportProtos.ComponentLifecycleMsgProto.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setEntityType(TransportProtos.EntityType.forNumber(entityId.getEntityType().ordinal())) - .setEntityIdMSB(entityId.getId().getMostSignificantBits()) - .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) - .setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(ComponentLifecycleEvent.STARTED.ordinal())) - .build(); - - ComponentLifecycleMsg msg = ProtoUtils.fromProto(proto); - - assertThat(msg).as("from proto").isEqualTo( - new ComponentLifecycleMsg(tenantId, entityId, ComponentLifecycleEvent.STARTED)); - - assertThat(ProtoUtils.toProto(msg)).as("to proto").isEqualTo(proto); + void protoEntityTypeSerialization() { + for(EntityType entityType : EntityType.values()){ + assertThat(ProtoUtils.fromProto(ProtoUtils.toProto(entityType))).as(entityType.getNormalName()).isEqualTo(entityType); + } } } \ No newline at end of file diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index d8439d0cd0..e560dfccf5 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -23,33 +23,37 @@ option java_outer_classname = "TransportProtos"; /** * Common data structures */ -enum EntityType { - TENANT = 0; - CUSTOMER = 1; - USER = 2; - DASHBOARD = 3; - ASSET = 4; - DEVICE = 5; - ALARM = 6; - RULE_CHAIN = 7; - RULE_NODE = 8; - ENTITY_VIEW = 9; - WIDGETS_BUNDLE = 10; - WIDGET_TYPE = 11; - TENANT_PROFILE = 12; - DEVICE_PROFILE = 13; - ASSET_PROFILE = 14; - API_USAGE_STATE = 15; - TB_RESOURCE = 16; - OTA_PACKAGE = 17; - EDGE = 18; - RPC = 19; - QUEUE = 20; - NOTIFICATION_TARGET = 21; - NOTIFICATION_TEMPLATE = 22; - NOTIFICATION_REQUEST = 23; - NOTIFICATION = 24; - NOTIFICATION_RULE = 25; +enum EntityTypeProto { + UNSPECIFIED = 0; + TENANT = 1; + CUSTOMER = 2; + USER = 3; + DASHBOARD = 4; + ASSET = 5; + DEVICE = 6; + ALARM = 7; + // next 2 reserved for PE; + RULE_CHAIN = 10; + RULE_NODE = 11; + // next 2 reserved for PE; + ENTITY_VIEW = 14; + WIDGETS_BUNDLE = 15; + WIDGET_TYPE = 16; + // next 2 reserved for PE; + TENANT_PROFILE = 19; + DEVICE_PROFILE = 20; + ASSET_PROFILE = 21; + API_USAGE_STATE = 22; + TB_RESOURCE = 23; + OTA_PACKAGE = 24; + EDGE = 25; + RPC = 26; + QUEUE = 27; + NOTIFICATION_TARGET = 28; + NOTIFICATION_TEMPLATE = 29; + NOTIFICATION_REQUEST = 30; + NOTIFICATION = 31; + NOTIFICATION_RULE = 32; } /** @@ -776,7 +780,7 @@ enum ComponentLifecycleEvent { message ComponentLifecycleMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; - EntityType entityType = 3; + EntityTypeProto entityType = 3; int64 entityIdMSB = 4; int64 entityIdLSB = 5; ComponentLifecycleEvent event = 6; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index c4344e8789..c755659966 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -26,39 +26,46 @@ import java.util.stream.Collectors; * @author Andrew Shvayka */ public enum EntityType { - // In sync with EntityType proto - TENANT, - CUSTOMER, - USER, - DASHBOARD, - ASSET, - DEVICE, - ALARM, - RULE_CHAIN, - RULE_NODE, - ENTITY_VIEW { + TENANT(1), + CUSTOMER(2), + USER(3), + DASHBOARD(4), + ASSET(5), + DEVICE(6), + ALARM (7), + RULE_CHAIN (10), + RULE_NODE (11), + + ENTITY_VIEW (14) { // backward compatibility for TbOriginatorTypeSwitchNode to return correct rule node connection. @Override - public String getNormalName() { + public String getNormalName () { return "Entity View"; } }, - WIDGETS_BUNDLE, - WIDGET_TYPE, - TENANT_PROFILE, - DEVICE_PROFILE, - ASSET_PROFILE, - API_USAGE_STATE, - TB_RESOURCE, - OTA_PACKAGE, - EDGE, - RPC, - QUEUE, - NOTIFICATION_TARGET, - NOTIFICATION_TEMPLATE, - NOTIFICATION_REQUEST, - NOTIFICATION, - NOTIFICATION_RULE; + WIDGETS_BUNDLE (15), + WIDGET_TYPE (16), + TENANT_PROFILE (19), + DEVICE_PROFILE (20), + ASSET_PROFILE (21), + API_USAGE_STATE (22), + TB_RESOURCE (23), + OTA_PACKAGE (24), + EDGE (25), + RPC (26), + QUEUE (27), + NOTIFICATION_TARGET (28), + NOTIFICATION_TEMPLATE (29), + NOTIFICATION_REQUEST (30), + NOTIFICATION (31), + NOTIFICATION_RULE (32); + + @Getter + private final int protoNumber; // Corresponds to EntityTypeProto + + private EntityType(int protoNumber) { + this.protoNumber = protoNumber; + } public static final List NORMAL_NAMES = EnumSet.allOf(EntityType.class).stream() .map(EntityType::getNormalName).collect(Collectors.toUnmodifiableList()); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index f110a6f209..f293b5dc24 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -32,6 +32,8 @@ import java.util.Optional; */ @Data public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { + private static final long serialVersionUID = -5303421482781273062L; + private final TenantId tenantId; private final EntityId entityId; private final ComponentLifecycleEvent event; From b2098fc11258fd37af36057b2f308cb8eadc0345 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 24 Oct 2023 12:30:17 +0300 Subject: [PATCH 22/23] Fix compatibility between CE and PE --- common/cluster-api/src/main/proto/queue.proto | 40 +++++++++---------- .../server/common/data/EntityType.java | 38 +++++++++--------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index e560dfccf5..8202f1c749 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -32,28 +32,28 @@ enum EntityTypeProto { ASSET = 5; DEVICE = 6; ALARM = 7; + // next 3 reserved for PE; + RULE_CHAIN = 11; + RULE_NODE = 12; // next 2 reserved for PE; - RULE_CHAIN = 10; - RULE_NODE = 11; + ENTITY_VIEW = 15; + WIDGETS_BUNDLE = 16; + WIDGET_TYPE = 17; // next 2 reserved for PE; - ENTITY_VIEW = 14; - WIDGETS_BUNDLE = 15; - WIDGET_TYPE = 16; - // next 2 reserved for PE; - TENANT_PROFILE = 19; - DEVICE_PROFILE = 20; - ASSET_PROFILE = 21; - API_USAGE_STATE = 22; - TB_RESOURCE = 23; - OTA_PACKAGE = 24; - EDGE = 25; - RPC = 26; - QUEUE = 27; - NOTIFICATION_TARGET = 28; - NOTIFICATION_TEMPLATE = 29; - NOTIFICATION_REQUEST = 30; - NOTIFICATION = 31; - NOTIFICATION_RULE = 32; + TENANT_PROFILE = 20; + DEVICE_PROFILE = 21; + ASSET_PROFILE = 22; + API_USAGE_STATE = 23; + TB_RESOURCE = 24; + OTA_PACKAGE = 25; + EDGE = 26; + RPC = 27; + QUEUE = 28; + NOTIFICATION_TARGET = 29; + NOTIFICATION_TEMPLATE = 30; + NOTIFICATION_REQUEST = 31; + NOTIFICATION = 32; + NOTIFICATION_RULE = 33; } /** diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index c755659966..ce87edf43a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -33,32 +33,32 @@ public enum EntityType { ASSET(5), DEVICE(6), ALARM (7), - RULE_CHAIN (10), - RULE_NODE (11), + RULE_CHAIN (11), + RULE_NODE (12), - ENTITY_VIEW (14) { + ENTITY_VIEW (15) { // backward compatibility for TbOriginatorTypeSwitchNode to return correct rule node connection. @Override public String getNormalName () { return "Entity View"; } }, - WIDGETS_BUNDLE (15), - WIDGET_TYPE (16), - TENANT_PROFILE (19), - DEVICE_PROFILE (20), - ASSET_PROFILE (21), - API_USAGE_STATE (22), - TB_RESOURCE (23), - OTA_PACKAGE (24), - EDGE (25), - RPC (26), - QUEUE (27), - NOTIFICATION_TARGET (28), - NOTIFICATION_TEMPLATE (29), - NOTIFICATION_REQUEST (30), - NOTIFICATION (31), - NOTIFICATION_RULE (32); + WIDGETS_BUNDLE (16), + WIDGET_TYPE (17), + TENANT_PROFILE (20), + DEVICE_PROFILE (21), + ASSET_PROFILE (22), + API_USAGE_STATE (23), + TB_RESOURCE (24), + OTA_PACKAGE (25), + EDGE (26), + RPC (27), + QUEUE (28), + NOTIFICATION_TARGET (29), + NOTIFICATION_TEMPLATE (30), + NOTIFICATION_REQUEST (31), + NOTIFICATION (32), + NOTIFICATION_RULE (33); @Getter private final int protoNumber; // Corresponds to EntityTypeProto From 9ca568d0700daab0b77ee1727b33abbdca670589 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 24 Oct 2023 13:23:14 +0300 Subject: [PATCH 23/23] Improvements for queue-related tests --- .../controller/TenantControllerTest.java | 2 +- .../TbRuleEngineQueueConsumerManagerTest.java | 21 ++++++------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java index d1aefc097f..27a9872ba5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java @@ -687,7 +687,7 @@ public class TenantControllerTest extends AbstractControllerTest { submittedMsgs.add(tbMsg.getId()); Thread.sleep(timeLeft / msgs); } - await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> { + await().atMost(15, TimeUnit.SECONDS).untilAsserted(() -> { verify(queueAdmin, times(1)).deleteTopic(eq(isolatedTopic)); }); diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java index 38582d0fce..05eacc68e3 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java @@ -456,8 +456,7 @@ public class TbRuleEngineQueueConsumerManagerTest { clearInvocations(actorContext); verify(consumer1, never()).unsubscribe(); verify(consumer2, never()).unsubscribe(); - int msgCount1 = consumer1.msgCount; - int msgCount2 = consumer2.msgCount; + int msgCount = totalConsumedMsgs.get(); await().atLeast(4, TimeUnit.SECONDS) // based on topicDeletionDelayInSec .atMost(7, TimeUnit.SECONDS) @@ -471,10 +470,8 @@ public class TbRuleEngineQueueConsumerManagerTest { verify(consumer1).unsubscribe(); verify(consumer2).unsubscribe(); - int movedMsgs1 = consumer1.msgCount - msgCount1; - int movedMsgs2 = consumer2.msgCount - msgCount2; - int totalMovedMsgs = movedMsgs1 + movedMsgs2; - assertThat(totalMovedMsgs).isGreaterThan(10); + int totalMovedMsgs = totalConsumedMsgs.get() - msgCount; + assertThat(totalMovedMsgs).isNotZero(); verify(ruleEngineMsgProducer, atLeast(totalMovedMsgs)).send(any(), any(), any()); verify(actorContext, never()).tell(any()); generateQueueMsgs = false; @@ -499,7 +496,7 @@ public class TbRuleEngineQueueConsumerManagerTest { }); clearInvocations(actorContext); verify(consumer, never()).unsubscribe(); - int msgCount = consumer.msgCount; + int msgCount = totalConsumedMsgs.get(); await().atLeast(4, TimeUnit.SECONDS) .atMost(7, TimeUnit.SECONDS) @@ -512,7 +509,8 @@ public class TbRuleEngineQueueConsumerManagerTest { }); verify(consumer).unsubscribe(); - int movedMsgs = consumer.msgCount - msgCount; + int movedMsgs = totalConsumedMsgs.get() - msgCount; + assertThat(movedMsgs).isNotZero(); verify(ruleEngineMsgProducer, atLeast(movedMsgs)).send(any(), any(), any()); verify(actorContext, never()).tell(any()); generateQueueMsgs = false; @@ -607,11 +605,6 @@ public class TbRuleEngineQueueConsumerManagerTest { }); } - /* - * 2023-10-15 18:34:06,090 [main] INFO o.t.s.s.q.r.TbRuleEngineQueueConsumerManagerTest - Generated new partitions: [0, 1, 2, 3, 4, 5, 6, 8, 9, 11, 12, 13, 15, 16, 17, 18, 19] -2023-10-15 18:34:06,090 [main] INFO o.t.s.s.q.r.TbRuleEngineQueueConsumerManagerTest - Generated new config: consumerPerPartition=false, pollInterval=299, processingStrategy=RETRY_FAILED - * */ - private void verifySubscribedAndLaunched(TestConsumer consumer, Set expectedPartitions) { await().atMost(2, TimeUnit.SECONDS) .until(() -> consumer.subscribed && consumer.getPartitions().equals(expectedPartitions) && consumer.pollingStarted); @@ -701,7 +694,6 @@ public class TbRuleEngineQueueConsumerManagerTest { private boolean pollingStarted; private TbMsg testMsg; - private int msgCount; public TestConsumer(String topic) { super(topic); @@ -746,7 +738,6 @@ public class TbRuleEngineQueueConsumerManagerTest { if (!subscribed) { throw new IllegalStateException("Cannot commit because not subscribed"); } - msgCount++; log.debug("doCommit() totalConsumedMsgs = {}", totalConsumedMsgs.incrementAndGet()); }