diff --git a/application/src/main/data/upgrade/3.5.1/schema_update.sql b/application/src/main/data/upgrade/3.5.1/schema_update.sql index f41e370ab5..fab91a4e67 100644 --- a/application/src/main/data/upgrade/3.5.1/schema_update.sql +++ b/application/src/main/data/upgrade/3.5.1/schema_update.sql @@ -189,7 +189,7 @@ DO $$ BEGIN IF EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = 'widget_type' and column_name='bundle_alias') THEN - INSERT INTO widgets_bundle_widget SELECT wb.id as widgets_bundle_id, wt.id as widget_type_id from widget_type wt left join widgets_bundle wb ON wt.bundle_alias = wb.alias ON CONFLICT (widgets_bundle_id, widget_type_id) DO NOTHING; + INSERT INTO widgets_bundle_widget SELECT wb.id as widgets_bundle_id, wt.id as widget_type_id from widget_type wt left join widgets_bundle wb ON wt.bundle_alias = wb.alias AND wt.tenant_id = wb.tenant_id ON CONFLICT (widgets_bundle_id, widget_type_id) DO NOTHING; ALTER TABLE widget_type DROP COLUMN IF EXISTS bundle_alias; END IF; END; diff --git a/application/src/main/data/upgrade/3.6.0/schema_update.sql b/application/src/main/data/upgrade/3.6.0/schema_update.sql index a610cddd05..0361a5db65 100644 --- a/application/src/main/data/upgrade/3.6.0/schema_update.sql +++ b/application/src/main/data/upgrade/3.6.0/schema_update.sql @@ -25,3 +25,9 @@ ALTER TABLE notification DROP CONSTRAINT IF EXISTS fk_notification_recipient_id; CREATE INDEX IF NOT EXISTS idx_notification_notification_request_id ON notification(request_id); CREATE INDEX IF NOT EXISTS idx_notification_request_tenant_id ON notification_request(tenant_id); +-- DELETE invalid records from M:N widgets_bundle_widget table caused by the bug in previous upgrade script; +DELETE +FROM widgets_bundle_widget wbw +WHERE (SELECT tenant_id FROM widgets_bundle wb WHERE wb.id = wbw.widgets_bundle_id) != + (SELECT tenant_id FROM widget_type wt WHERE wt.id = wbw.widget_type_id); + diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java index 43b05094a4..006af577c9 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.service.edge; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @@ -23,6 +25,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmApiCallResult; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; @@ -36,6 +39,7 @@ import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.RelationActionEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; +import org.thingsboard.server.dao.user.UserServiceImpl; import javax.annotation.PostConstruct; @@ -75,7 +79,7 @@ public class EdgeEventSourcingListener { return; } try { - if (!isValidEdgeEventEntity(event.getEntity())) { + if (!isValidSaveEntityEventForEdgeProcessing(event.getEntity(), event.getOldEntity())) { return; } log.trace("[{}] SaveEntityEvent called: {}", event.getTenantId(), event); @@ -83,7 +87,7 @@ public class EdgeEventSourcingListener { tbClusterService.sendNotificationMsgToEdge(event.getTenantId(), null, event.getEntityId(), null, null, action); } catch (Exception e) { - log.error("[{}] failed to process SaveEntityEvent: {}", event.getTenantId(), event); + log.error("[{}] failed to process SaveEntityEvent: {}", event.getTenantId(), event, e); } } @@ -97,7 +101,7 @@ public class EdgeEventSourcingListener { tbClusterService.sendNotificationMsgToEdge(event.getTenantId(), event.getEdgeId(), event.getEntityId(), JacksonUtil.toString(event.getEntity()), null, EdgeEventActionType.DELETED); } catch (Exception e) { - log.error("[{}] failed to process DeleteEntityEvent: {}", event.getTenantId(), event); + log.error("[{}] failed to process DeleteEntityEvent: {}", event.getTenantId(), event, e); } } @@ -111,7 +115,7 @@ public class EdgeEventSourcingListener { tbClusterService.sendNotificationMsgToEdge(event.getTenantId(), event.getEdgeId(), event.getEntityId(), event.getBody(), null, edgeTypeByActionType(event.getActionType())); } catch (Exception e) { - log.error("[{}] failed to process ActionEntityEvent: {}", event.getTenantId(), event); + log.error("[{}] failed to process ActionEntityEvent: {}", event.getTenantId(), event, e); } } @@ -134,11 +138,11 @@ public class EdgeEventSourcingListener { tbClusterService.sendNotificationMsgToEdge(event.getTenantId(), null, null, JacksonUtil.toString(relation), EdgeEventType.RELATION, edgeTypeByActionType(event.getActionType())); } catch (Exception e) { - log.error("[{}] failed to process RelationActionEvent: {}", event.getTenantId(), event); + log.error("[{}] failed to process RelationActionEvent: {}", event.getTenantId(), event, e); } } - private boolean isValidEdgeEventEntity(Object entity) { + private boolean isValidSaveEntityEventForEdgeProcessing(Object entity, Object oldEntity) { if (entity instanceof OtaPackageInfo) { OtaPackageInfo otaPackageInfo = (OtaPackageInfo) entity; return otaPackageInfo.hasUrl() || otaPackageInfo.isHasData(); @@ -147,12 +151,36 @@ public class EdgeEventSourcingListener { return RuleChainType.EDGE.equals(ruleChain.getType()); } else if (entity instanceof User) { User user = (User) entity; - return !Authority.SYS_ADMIN.equals(user.getAuthority()); - } else if (entity instanceof AlarmApiCallResult) { - AlarmApiCallResult alarmApiCallResult = (AlarmApiCallResult) entity; - return alarmApiCallResult.isModified(); + if (Authority.SYS_ADMIN.equals(user.getAuthority())) { + return false; + } + if (oldEntity != null) { + User oldUser = (User) oldEntity; + cleanUpUserAdditionalInfo(oldUser); + cleanUpUserAdditionalInfo(user); + return !user.equals(oldUser); + } + } else if (entity instanceof AlarmApiCallResult || entity instanceof Alarm) { + return false; } // Default: If the entity doesn't match any of the conditions, consider it as valid. return true; } + + private void cleanUpUserAdditionalInfo(User user) { + // reset FAILED_LOGIN_ATTEMPTS and LAST_LOGIN_TS - edge is not interested in this information + if (user.getAdditionalInfo() instanceof NullNode) { + user.setAdditionalInfo(null); + } + if (user.getAdditionalInfo() instanceof ObjectNode) { + ObjectNode additionalInfo = ((ObjectNode) user.getAdditionalInfo()); + additionalInfo.remove(UserServiceImpl.FAILED_LOGIN_ATTEMPTS); + additionalInfo.remove(UserServiceImpl.LAST_LOGIN_TS); + if (additionalInfo.isEmpty()) { + user.setAdditionalInfo(null); + } else { + user.setAdditionalInfo(additionalInfo); + } + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index 328a64efe2..a888ddc27b 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -24,6 +24,7 @@ import org.springframework.context.annotation.Lazy; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EdgeUtils; @@ -46,6 +47,7 @@ import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -110,11 +112,13 @@ import org.thingsboard.server.service.entitiy.TbNotificationEntityService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import org.thingsboard.server.service.state.DefaultDeviceStateService; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -305,6 +309,61 @@ public abstract class BaseEdgeProcessor { EdgeEventActionType action, EntityId entityId, JsonNode body) { + ListenableFuture> future = + attributesService.find(tenantId, edgeId, DataConstants.SERVER_SCOPE, DefaultDeviceStateService.ACTIVITY_STATE); + return Futures.transformAsync(future, activeOpt -> { + if (activeOpt.isEmpty()) { + log.trace("Edge is not activated. Skipping event. tenantId [{}], edgeId [{}], type[{}], " + + "action [{}], entityId [{}], body [{}]", + tenantId, edgeId, type, action, entityId, body); + return Futures.immediateFuture(null); + } + if (activeOpt.get().getBooleanValue().isPresent() && activeOpt.get().getBooleanValue().get()) { + return doSaveEdgeEvent(tenantId, edgeId, type, action, entityId, body); + } else { + if (doSaveIfEdgeIsOffline(type, action)) { + return doSaveEdgeEvent(tenantId, edgeId, type, action, entityId, body); + } else { + log.trace("Edge is not active at the moment. Skipping event. tenantId [{}], edgeId [{}], type[{}], " + + "action [{}], entityId [{}], body [{}]", + tenantId, edgeId, type, action, entityId, body); + return Futures.immediateFuture(null); + } + } + }, dbCallbackExecutorService); + } + + private boolean doSaveIfEdgeIsOffline(EdgeEventType type, + EdgeEventActionType action) { + switch (action) { + case TIMESERIES_UPDATED: + case ALARM_ACK: + case ALARM_CLEAR: + case ALARM_ASSIGNED: + case ALARM_UNASSIGNED: + case CREDENTIALS_REQUEST: + return true; + } + switch (type) { + case ALARM: + case RULE_CHAIN: + case RULE_CHAIN_METADATA: + case USER: + case CUSTOMER: + case TENANT: + case TENANT_PROFILE: + case WIDGETS_BUNDLE: + case WIDGET_TYPE: + case ADMIN_SETTINGS: + case OTA_PACKAGE: + case QUEUE: + case RELATION: + return true; + } + return false; + } + + private ListenableFuture doSaveEdgeEvent(TenantId tenantId, EdgeId edgeId, EdgeEventType type, EdgeEventActionType action, EntityId entityId, JsonNode body) { log.debug("Pushing event to edge queue. tenantId [{}], edgeId [{}], type[{}], " + "action [{}], entityId [{}], body [{}]", tenantId, edgeId, type, action, entityId, body); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 53ef38d96a..937ee12fab 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -395,7 +395,7 @@ public class DefaultTbClusterService implements TbClusterService { } private void broadcast(ComponentLifecycleMsg msg) { - byte[] msgBytes = encodingService.encode(msg); + TransportProtos.ComponentLifecycleMsgProto componentLifecycleMsgProto = ProtoUtils.toProto(msg); TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); Set tbRuleEngineServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); EntityType entityType = msg.getEntityId().getEntityType(); @@ -413,7 +413,7 @@ public class DefaultTbClusterService implements TbClusterService { Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); for (String serviceId : tbCoreServices) { TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); - ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); + ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setComponentLifecycle(componentLifecycleMsgProto).build(); toCoreNfProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCoreMsg), null); toCoreNfs.incrementAndGet(); } @@ -422,7 +422,7 @@ public class DefaultTbClusterService implements TbClusterService { } for (String serviceId : tbRuleEngineServices) { TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); - ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build(); + ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycle(componentLifecycleMsgProto).build(); toRuleEngineProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toRuleEngineMsg), null); toRuleEngineNfs.incrementAndGet(); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index fd5a252cc5..7415ee6717 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -142,8 +142,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> usageStatsConsumer; private final TbQueueConsumer> firmwareStatesConsumer; + protected volatile ExecutorService consumersExecutor; protected volatile ExecutorService usageStatsExecutor; - private volatile ExecutorService firmwareStatesExecutor; public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, @@ -186,7 +186,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService implements TbRuleEngineConsumerService { - public static final String SUCCESSFUL_STATUS = "successful"; - public static final String FAILED_STATUS = "failed"; - public static final String THREAD_TOPIC_SEPARATOR = " | "; - @Value("${queue.rule-engine.poll-interval}") - private long pollDuration; - @Value("${queue.rule-engine.pack-processing-timeout}") - private long packProcessingTimeout; - @Value("${queue.rule-engine.stats.enabled:true}") - private boolean statsEnabled; - @Value("${queue.rule-engine.prometheus-stats.enabled:false}") - boolean prometheusStatsEnabled; - @Value("${queue.rule-engine.topic-deletion-delay:30}") - private int topicDeletionDelayInSec; - - private final StatsFactory statsFactory; - private final TbRuleEngineSubmitStrategyFactory submitStrategyFactory; - private final TbRuleEngineProcessingStrategyFactory processingStrategyFactory; - private final TbRuleEngineQueueFactory tbRuleEngineQueueFactory; - private final RuleEngineStatisticsService statisticsService; - private final TbRuleEngineDeviceRpcService tbDeviceRpcService; - private final TbServiceInfoProvider serviceInfoProvider; + private final TbRuleEngineConsumerContext ctx; private final QueueService queueService; - private final TbQueueProducerProvider producerProvider; - private final TbQueueAdmin queueAdmin; - private final ConcurrentMap>> consumers = new ConcurrentHashMap<>(); - private final ConcurrentMap consumerConfigurations = new ConcurrentHashMap<>(); - private final ConcurrentMap consumerStats = new ConcurrentHashMap<>(); - private final ConcurrentMap topicsConsumerPerPartition = new ConcurrentHashMap<>(); - final ExecutorService submitExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-submit")); - final ScheduledExecutorService repartitionExecutor = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-repartition")); + private final TbRuleEngineDeviceRpcService tbDeviceRpcService; - public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory, - TbRuleEngineSubmitStrategyFactory submitStrategyFactory, + private final ConcurrentMap consumers = new ConcurrentHashMap<>(); + + public DefaultTbRuleEngineConsumerService(TbRuleEngineConsumerContext ctx, TbRuleEngineQueueFactory tbRuleEngineQueueFactory, - RuleEngineStatisticsService statisticsService, ActorSystemContext actorContext, DataDecodingEncodingService encodingService, TbRuleEngineDeviceRpcService tbDeviceRpcService, - StatsFactory statsFactory, + QueueService queueService, TbDeviceProfileCache deviceProfileCache, TbAssetProfileCache assetProfileCache, TbTenantProfileCache tenantProfileCache, TbApiUsageStateService apiUsageStateService, - PartitionService partitionService, ApplicationEventPublisher eventPublisher, - TbServiceInfoProvider serviceInfoProvider, QueueService queueService, - TbQueueProducerProvider producerProvider, TbQueueAdmin queueAdmin) { - super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, eventPublisher, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer(), Optional.empty()); - this.statisticsService = statisticsService; - this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; - this.submitStrategyFactory = submitStrategyFactory; - this.processingStrategyFactory = processingStrategyFactory; + PartitionService partitionService, ApplicationEventPublisher eventPublisher) { + super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, + eventPublisher, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer(), Optional.empty()); + this.ctx = ctx; this.tbDeviceRpcService = tbDeviceRpcService; - this.statsFactory = statsFactory; - this.serviceInfoProvider = serviceInfoProvider; this.queueService = queueService; - this.producerProvider = producerProvider; - this.queueAdmin = queueAdmin; } @PostConstruct public void init() { - super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer"); + super.init("tb-rule-engine-notifications-consumer"); List queues = queueService.findAllQueues(); for (Queue configuration : queues) { if (partitionService.isManagedByCurrentService(configuration.getTenantId())) { @@ -163,246 +97,37 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } private void initConsumer(Queue configuration) { - QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, configuration); - consumerConfigurations.putIfAbsent(queueKey, configuration); - consumerStats.putIfAbsent(queueKey, new TbRuleEngineConsumerStats(configuration, statsFactory)); - if (!configuration.isConsumerPerPartition()) { - consumers.computeIfAbsent(queueKey, queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration)); - } else { - topicsConsumerPerPartition.computeIfAbsent(queueKey, k -> new TbTopicWithConsumerPerPartition(k.getQueueName())); - } - } - - @PreDestroy - public void stop() { - super.destroy(); - submitExecutor.shutdownNow(); - repartitionExecutor.shutdownNow(); + getOrCreateConsumer(new QueueKey(ServiceType.TB_RULE_ENGINE, configuration)).init(configuration); } @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { if (event.getServiceType().equals(getServiceType())) { event.getPartitionsMap().forEach((queueKey, partitions) -> { - String serviceQueue = queueKey.getQueueName(); - log.info("[{}] Subscribing to partitions: {}", serviceQueue, partitions); - Queue configuration = consumerConfigurations.get(queueKey); - if (configuration == null) { - return; - } - if (!configuration.isConsumerPerPartition()) { - consumers.get(queueKey).subscribe(partitions); + var consumer = consumers.get(queueKey); + if (consumer != null) { + consumer.update(partitions); } else { - log.info("[{}] Subscribing consumer per partition: {}", serviceQueue, partitions); - subscribeConsumerPerPartition(queueKey, partitions); + log.warn("Received invalid partition change event for {} that is not managed by this service", queueKey); } }); } } - void subscribeConsumerPerPartition(QueueKey queue, Set partitions) { - topicsConsumerPerPartition.get(queue).getSubscribeQueue().add(partitions); - scheduleTopicRepartition(queue); - } - - private void scheduleTopicRepartition(QueueKey queue) { - repartitionExecutor.schedule(() -> repartitionTopicWithConsumerPerPartition(queue), 1, TimeUnit.SECONDS); - } - - void repartitionTopicWithConsumerPerPartition(final QueueKey queueKey) { - if (stopped) { - return; - } - TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.get(queueKey); - java.util.Queue> subscribeQueue = tbTopicWithConsumerPerPartition.getSubscribeQueue(); - if (subscribeQueue.isEmpty()) { - return; - } - if (tbTopicWithConsumerPerPartition.getLock().tryLock()) { - try { - Set partitions = null; - while (!subscribeQueue.isEmpty()) { - partitions = subscribeQueue.poll(); - } - if (partitions == null) { - return; - } - - Set addedPartitions = new HashSet<>(partitions); - ConcurrentMap>> consumers = tbTopicWithConsumerPerPartition.getConsumers(); - addedPartitions.removeAll(consumers.keySet()); - log.info("calculated addedPartitions {}", addedPartitions); - - Set removedPartitions = new HashSet<>(consumers.keySet()); - removedPartitions.removeAll(partitions); - log.info("calculated removedPartitions {}", removedPartitions); - - removedPartitions.forEach((tpi) -> { - removeConsumerForTopicByTpi(queueKey.getQueueName(), consumers, tpi); - }); - - addedPartitions.forEach((tpi) -> { - log.info("[{}] Adding consumer for topic: {}", queueKey, tpi); - Queue configuration = consumerConfigurations.get(queueKey); - TbQueueConsumer> consumer = tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration); - consumers.put(tpi, consumer); - launchConsumer(consumer, consumerConfigurations.get(queueKey), consumerStats.get(queueKey), "" + queueKey + "-" + tpi.getPartition().orElse(-999999)); - consumer.subscribe(Collections.singleton(tpi)); - }); - } finally { - tbTopicWithConsumerPerPartition.getLock().unlock(); - } - } else { - scheduleTopicRepartition(queueKey); //reschedule later - } - - } - - void removeConsumerForTopicByTpi(String queue, ConcurrentMap>> consumers, TopicPartitionInfo tpi) { - log.info("[{}] Removing consumer for topic: {}", queue, tpi); - consumers.get(tpi).unsubscribe(); - consumers.remove(tpi); + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void onApplicationEvent(ApplicationReadyEvent event) { + super.onApplicationEvent(event); + ctx.setReady(true); } @Override - protected void launchMainConsumers() { - consumers.forEach((queue, consumer) -> launchConsumer(consumer, consumerConfigurations.get(queue), consumerStats.get(queue), queue.getQueueName())); - } + protected void launchMainConsumers() {} @Override - protected void stopMainConsumers() { - consumers.values().forEach(TbQueueConsumer::unsubscribe); - topicsConsumerPerPartition.values().forEach(tbTopicWithConsumerPerPartition -> tbTopicWithConsumerPerPartition.getConsumers().keySet() - .forEach((tpi) -> removeConsumerForTopicByTpi(tbTopicWithConsumerPerPartition.getTopic(), tbTopicWithConsumerPerPartition.getConsumers(), tpi))); - } - - void launchConsumer(TbQueueConsumer> consumer, Queue configuration, TbRuleEngineConsumerStats stats, String threadSuffix) { - if (isReady) { - consumersExecutor.execute(() -> consumerLoop(consumer, configuration, stats, threadSuffix)); - } else { - scheduleLaunchConsumer(consumer, configuration, stats, threadSuffix); - } - } - - private void scheduleLaunchConsumer(TbQueueConsumer> consumer, Queue configuration, TbRuleEngineConsumerStats stats, String threadSuffix) { - repartitionExecutor.schedule(() -> { - if (isReady) { - consumersExecutor.execute(() -> consumerLoop(consumer, configuration, stats, threadSuffix)); - } else { - scheduleLaunchConsumer(consumer, configuration, stats, threadSuffix); - } - }, 10, TimeUnit.SECONDS); - } - - void consumerLoop(TbQueueConsumer> consumer, org.thingsboard.server.common.data.queue.Queue configuration, TbRuleEngineConsumerStats stats, String threadSuffix) { - updateCurrentThreadName(threadSuffix); - while (!stopped && !consumer.isStopped() && !consumer.isQueueDeleted()) { - try { - List> msgs = consumer.poll(configuration.getPollInterval()); - if (msgs.isEmpty()) { - continue; - } - final TbRuleEngineSubmitStrategy submitStrategy = getSubmitStrategy(configuration); - final TbRuleEngineProcessingStrategy ackStrategy = getAckStrategy(configuration); - submitStrategy.init(msgs); - while (!stopped && !consumer.isStopped()) { - TbMsgPackProcessingContext ctx = new TbMsgPackProcessingContext(configuration.getName(), submitStrategy, ackStrategy.isSkipTimeoutMsgs()); - submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> submitMessage(configuration, stats, ctx, id, msg))); - - final boolean timeout = !ctx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS); - - TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(configuration.getName(), timeout, ctx); - if (timeout) { - printFirstOrAll(configuration, ctx, ctx.getPendingMap(), "Timeout"); - } - if (!ctx.getFailedMap().isEmpty()) { - printFirstOrAll(configuration, ctx, ctx.getFailedMap(), "Failed"); - } - ctx.printProfilerStats(); - - TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result); - if (statsEnabled) { - stats.log(result, decision.isCommit()); - } - - ctx.cleanup(); - - if (decision.isCommit()) { - submitStrategy.stop(); - break; - } else { - submitStrategy.update(decision.getReprocessMap()); - } - } - consumer.commit(); - } catch (Exception e) { - if (!stopped) { - log.warn("Failed to process messages from queue.", e); - try { - Thread.sleep(pollDuration); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); - } - } - } - } - - if (consumer.isQueueDeleted()) { - processQueueDeletion(configuration, consumer); - } - log.info("TB Rule Engine Consumer stopped."); - } - - void updateCurrentThreadName(String threadSuffix) { - String name = Thread.currentThread().getName(); - int spliteratorIndex = name.indexOf(THREAD_TOPIC_SEPARATOR); - if (spliteratorIndex > 0) { - name = name.substring(0, spliteratorIndex); - } - name = name + THREAD_TOPIC_SEPARATOR + threadSuffix; - Thread.currentThread().setName(name); - } - - TbRuleEngineProcessingStrategy getAckStrategy(Queue configuration) { - return processingStrategyFactory.newInstance(configuration.getName(), configuration.getProcessingStrategy()); - } - - TbRuleEngineSubmitStrategy getSubmitStrategy(Queue configuration) { - return submitStrategyFactory.newInstance(configuration.getName(), configuration.getSubmitStrategy()); - } - - void submitMessage(Queue configuration, TbRuleEngineConsumerStats stats, TbMsgPackProcessingContext ctx, UUID id, TbProtoQueueMsg msg) { - log.trace("[{}] Creating callback for topic {} message: {}", id, configuration.getName(), msg.getValue()); - ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); - TenantId tenantId = TenantId.fromUUID(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); - TbMsgCallback callback = prometheusStatsEnabled ? - new TbMsgPackCallback(id, tenantId, ctx, stats.getTimer(tenantId, SUCCESSFUL_STATUS), stats.getTimer(tenantId, FAILED_STATUS)) : - new TbMsgPackCallback(id, tenantId, ctx); - try { - if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) { - forwardToRuleEngineActor(configuration.getName(), tenantId, toRuleEngineMsg, callback); - } else { - callback.onSuccess(); - } - } catch (Exception e) { - callback.onFailure(new RuleEngineException(e.getMessage(), e)); - } - } - - private void printFirstOrAll(Queue configuration, TbMsgPackProcessingContext ctx, Map> map, String prefix) { - boolean printAll = log.isTraceEnabled(); - log.info("{} to process [{}] messages", prefix, map.size()); - for (Map.Entry> pending : map.entrySet()) { - ToRuleEngineMsg tmp = pending.getValue().getValue(); - TbMsg tmpMsg = TbMsg.fromBytes(configuration.getName(), tmp.getTbMsg().toByteArray(), TbMsgCallback.EMPTY); - RuleNodeInfo ruleNodeInfo = ctx.getLastVisitedRuleNode(pending.getKey()); - if (printAll) { - log.trace("[{}] {} to process message: {}, Last Rule Node: {}", TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); - } else { - log.info("[{}] {} to process message: {}, Last Rule Node: {}", TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); - break; - } - } + protected void stopConsumers() { + consumers.values().forEach(TbRuleEngineQueueConsumerManager::stop); + consumers.values().forEach(TbRuleEngineQueueConsumerManager::awaitStop); + ctx.stop(); } @Override @@ -412,18 +137,22 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @Override protected long getNotificationPollDuration() { - return pollDuration; + return ctx.getPollDuration(); } @Override protected long getNotificationPackProcessingTimeout() { - return packProcessingTimeout; + return ctx.getPackProcessingTimeout(); } @Override protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) throws Exception { ToRuleEngineNotificationMsg nfMsg = msg.getValue(); - if (nfMsg.getComponentLifecycleMsg() != null && !nfMsg.getComponentLifecycleMsg().isEmpty()) { + if (nfMsg.hasComponentLifecycle()) { + handleComponentLifecycleMsg(id, ProtoUtils.fromProto(nfMsg.getComponentLifecycle())); + callback.onSuccess(); + } else if (!nfMsg.getComponentLifecycleMsg().isEmpty()) { + //will be removed in 3.6.1 in favour of hasComponentLifecycle() handleComponentLifecycleMsg(id, nfMsg.getComponentLifecycleMsg()); callback.onSuccess(); } else if (nfMsg.hasFromDeviceRpcResponse()) { @@ -434,10 +163,10 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< tbDeviceRpcService.processRpcResponseFromDevice(response); callback.onSuccess(); } else if (nfMsg.hasQueueUpdateMsg()) { - repartitionExecutor.execute(() -> updateQueue(nfMsg.getQueueUpdateMsg())); + ctx.getScheduler().execute(() -> updateQueue(nfMsg.getQueueUpdateMsg())); callback.onSuccess(); } else if (nfMsg.hasQueueDeleteMsg()) { - repartitionExecutor.execute(() -> deleteQueue(nfMsg.getQueueDeleteMsg())); + ctx.getScheduler().execute(() -> deleteQueue(nfMsg.getQueueDeleteMsg())); callback.onSuccess(); } else { log.trace("Received notification with missing handler"); @@ -453,123 +182,43 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< String queueName = queueUpdateMsg.getQueueName(); QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueName, tenantId); Queue queue = queueService.findQueueById(tenantId, queueId); - Queue oldQueue = consumerConfigurations.remove(queueKey); - if (oldQueue != null) { - if (oldQueue.isConsumerPerPartition()) { - TbTopicWithConsumerPerPartition consumerPerPartition = topicsConsumerPerPartition.remove(queueKey); - ReentrantLock lock = consumerPerPartition.getLock(); - try { - lock.lock(); - consumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::unsubscribe); - } finally { - lock.unlock(); - } - } else { - TbQueueConsumer> consumer = consumers.remove(queueKey); - consumer.unsubscribe(); - } - } - initConsumer(queue); + TbRuleEngineQueueConsumerManager consumerManager = getOrCreateConsumer(queueKey); + Queue oldQueue = consumerManager.getQueue(); + consumerManager.update(queue); - if (!queue.isConsumerPerPartition()) { - launchConsumer(consumers.get(queueKey), consumerConfigurations.get(queueKey), consumerStats.get(queueKey), queueName); + if (oldQueue != null && queue.getPartitions() == oldQueue.getPartitions()) { + return; } } partitionService.updateQueue(queueUpdateMsg); - partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), new ArrayList<>(partitionService.getOtherServices(ServiceType.TB_RULE_ENGINE))); + partitionService.recalculatePartitions(ctx.getServiceInfoProvider().getServiceInfo(), + new ArrayList<>(partitionService.getOtherServices(ServiceType.TB_RULE_ENGINE))); } private void deleteQueue(TransportProtos.QueueDeleteMsg queueDeleteMsg) { log.info("Received queue delete msg: [{}]", queueDeleteMsg); TenantId tenantId = new TenantId(new UUID(queueDeleteMsg.getTenantIdMSB(), queueDeleteMsg.getTenantIdLSB())); QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueDeleteMsg.getQueueName(), tenantId); - - partitionService.removeQueue(queueDeleteMsg); - Queue queue = consumerConfigurations.remove(queueKey); - if (queue != null) { - if (queue.isConsumerPerPartition()) { - TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.remove(queueKey); - if (tbTopicWithConsumerPerPartition != null) { - tbTopicWithConsumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::onQueueDelete); - tbTopicWithConsumerPerPartition.getConsumers().clear(); - } - } else { - TbQueueConsumer> consumer = consumers.remove(queueKey); - if (consumer != null) { - consumer.onQueueDelete(); - } - } + var consumerManager = consumers.remove(queueKey); + if (consumerManager != null) { + consumerManager.delete(); } - } - private void forwardToRuleEngineActor(String queueName, TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { - TbMsg tbMsg = TbMsg.fromBytes(queueName, toRuleEngineMsg.getTbMsg().toByteArray(), callback); - QueueToRuleEngineMsg msg; - ProtocolStringList relationTypesList = toRuleEngineMsg.getRelationTypesList(); - Set relationTypes = null; - if (relationTypesList != null) { - if (relationTypesList.size() == 1) { - relationTypes = Collections.singleton(relationTypesList.get(0)); - } else { - relationTypes = new HashSet<>(relationTypesList); - } - } - msg = new QueueToRuleEngineMsg(tenantId, tbMsg, relationTypes, toRuleEngineMsg.getFailureMessage()); - actorContext.tell(msg); + partitionService.removeQueue(queueDeleteMsg); + partitionService.recalculatePartitions(ctx.getServiceInfoProvider().getServiceInfo(), new ArrayList<>(partitionService.getOtherServices(ServiceType.TB_RULE_ENGINE))); } - private void processQueueDeletion(Queue queue, TbQueueConsumer> consumer) { - long finishTs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(topicDeletionDelayInSec); - try { - int n = 0; - while (System.currentTimeMillis() <= finishTs) { - List> msgs = consumer.poll(queue.getPollInterval()); - if (msgs.isEmpty()) { - continue; - } - for (TbProtoQueueMsg msg : msgs) { - try { - MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray()); - EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB())); - - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue.getName(), TenantId.SYS_TENANT_ID, originator); - producerProvider.getRuleEngineMsgProducer().send(tpi, msg, null); - n++; - } catch (Throwable e) { - log.debug("Failed to move message to system {}: {}", consumer.getTopic(), msg, e); - } - } - consumer.commit(); - } - if (n > 0) { - log.info("Moved {} messages from {} to system {}", n, consumer.getFullTopicNames(), consumer.getTopic()); - } - - consumer.unsubscribe(); - for (String topic : consumer.getFullTopicNames()) { - try { - queueAdmin.deleteTopic(topic); - log.info("Deleted topic {}", topic); - } catch (Exception e) { - log.error("Failed to delete topic {} after unsubscribing", topic, e); - } - } - } catch (Exception e) { - log.error("Failed to process deletion of {} ({})", consumer.getTopic(), queue.getTenantId(), e); - } + private TbRuleEngineQueueConsumerManager getOrCreateConsumer(QueueKey queueKey) { + return consumers.computeIfAbsent(queueKey, key -> new TbRuleEngineQueueConsumerManager(ctx, key)); } @Scheduled(fixedDelayString = "${queue.rule-engine.stats.print-interval-ms}") public void printStats() { - if (statsEnabled) { + if (ctx.isStatsEnabled()) { long ts = System.currentTimeMillis(); - consumerStats.forEach((queue, stats) -> { - stats.printStats(); - statisticsService.reportQueueStats(ts, stats); - stats.reset(); - }); + consumers.values().forEach(manager -> manager.printStats(ts)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java b/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java new file mode 100644 index 0000000000..e39949e891 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.Arrays; +import java.util.UUID; + +public class ProtoUtils { + + private static final EntityType[] entityTypeByProtoNumber; + + static { + int arraySize = Arrays.stream(EntityType.values()).mapToInt(EntityType::getProtoNumber).max().orElse(0); + entityTypeByProtoNumber = new EntityType[arraySize + 1]; + Arrays.stream(EntityType.values()).forEach(entityType -> entityTypeByProtoNumber[entityType.getProtoNumber()] = entityType); + } + + public static TransportProtos.ComponentLifecycleMsgProto toProto(ComponentLifecycleMsg msg) { + return TransportProtos.ComponentLifecycleMsgProto.newBuilder() + .setTenantIdMSB(msg.getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(msg.getTenantId().getId().getLeastSignificantBits()) + .setEntityType(toProto(msg.getEntityId().getEntityType())) + .setEntityIdMSB(msg.getEntityId().getId().getMostSignificantBits()) + .setEntityIdLSB(msg.getEntityId().getId().getLeastSignificantBits()) + .setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(msg.getEvent().ordinal())) + .build(); + } + + public static TransportProtos.EntityTypeProto toProto(EntityType entityType) { + return TransportProtos.EntityTypeProto.forNumber(entityType.getProtoNumber()); + } + + public static ComponentLifecycleMsg fromProto(TransportProtos.ComponentLifecycleMsgProto proto) { + return new ComponentLifecycleMsg( + TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), + EntityIdFactory.getByTypeAndUuid(fromProto(proto.getEntityType()), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())), + ComponentLifecycleEvent.values()[proto.getEventValue()] + ); + } + + public static EntityType fromProto(TransportProtos.EntityTypeProto entityType) { + return entityTypeByProtoNumber[entityType.getNumber()]; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java index 4d195a0dec..9107159615 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java @@ -168,6 +168,8 @@ public class TbCoreConsumerStats { toCoreNfSubscriptionServiceCounter.increment(); } else if (msg.hasFromDeviceRpcResponse()) { toCoreNfDeviceRpcResponseCounter.increment(); + } else if (msg.hasComponentLifecycle()) { + toCoreNfComponentLifecycleCounter.increment(); } else if (!msg.getComponentLifecycleMsg().isEmpty()) { toCoreNfComponentLifecycleCounter.increment(); } else if (!msg.getEdgeEventUpdateMsg().isEmpty()) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index 2904c299ce..842b627c22 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsType; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; import java.util.ArrayList; @@ -66,9 +67,9 @@ public class TbRuleEngineConsumerStats { private final String queueName; private final TenantId tenantId; - public TbRuleEngineConsumerStats(Queue queue, StatsFactory statsFactory) { - this.queueName = queue.getName(); - this.tenantId = queue.getTenantId(); + public TbRuleEngineConsumerStats(QueueKey queueKey, StatsFactory statsFactory) { + this.queueName = queueKey.getQueueName(); + this.tenantId = queueKey.getTenantId(); this.statsFactory = statsFactory; String statsKey = StatsType.RULE_ENGINE.getName() + "." + queueName; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index b59086a350..7cd4ec52eb 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -65,7 +65,6 @@ import java.util.stream.Collectors; @Slf4j public abstract class AbstractConsumerService extends TbApplicationEventListener { - protected volatile ExecutorService consumersExecutor; protected volatile ExecutorService notificationsConsumerExecutor; protected volatile boolean stopped = false; protected volatile boolean isReady = false; @@ -99,8 +98,7 @@ public abstract class AbstractConsumerService actorMsgOpt = encodingService.decode(nfMsg.toByteArray()); - if (actorMsgOpt.isPresent()) { - TbActorMsg actorMsg = actorMsgOpt.get(); - if (actorMsg instanceof ComponentLifecycleMsg) { - ComponentLifecycleMsg componentLifecycleMsg = (ComponentLifecycleMsg) actorMsg; - log.debug("[{}][{}][{}] Received Lifecycle event: {}", componentLifecycleMsg.getTenantId(), componentLifecycleMsg.getEntityId().getEntityType(), - componentLifecycleMsg.getEntityId(), componentLifecycleMsg.getEvent()); - if (EntityType.TENANT_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { - TenantProfileId tenantProfileId = new TenantProfileId(componentLifecycleMsg.getEntityId().getId()); - tenantProfileCache.evict(tenantProfileId); + actorMsgOpt.ifPresent(tbActorMsg -> handleComponentLifecycleMsg(id, tbActorMsg)); + } + + protected void handleComponentLifecycleMsg(UUID id, TbActorMsg actorMsg) { + if (actorMsg instanceof ComponentLifecycleMsg) { + ComponentLifecycleMsg componentLifecycleMsg = (ComponentLifecycleMsg) actorMsg; + log.debug("[{}][{}][{}] Received Lifecycle event: {}", componentLifecycleMsg.getTenantId(), componentLifecycleMsg.getEntityId().getEntityType(), + componentLifecycleMsg.getEntityId(), componentLifecycleMsg.getEvent()); + if (EntityType.TENANT_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { + TenantProfileId tenantProfileId = new TenantProfileId(componentLifecycleMsg.getEntityId().getId()); + tenantProfileCache.evict(tenantProfileId); + if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.UPDATED)) { + apiUsageStateService.onTenantProfileUpdate(tenantProfileId); + } + } else if (EntityType.TENANT.equals(componentLifecycleMsg.getEntityId().getEntityType())) { + if (TenantId.SYS_TENANT_ID.equals(componentLifecycleMsg.getTenantId())) { + jwtSettingsService.ifPresent(JwtSettingsService::reloadJwtSettings); + return; + } else { + tenantProfileCache.evict(componentLifecycleMsg.getTenantId()); + partitionService.removeTenant(componentLifecycleMsg.getTenantId()); if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.UPDATED)) { - apiUsageStateService.onTenantProfileUpdate(tenantProfileId); - } - } else if (EntityType.TENANT.equals(componentLifecycleMsg.getEntityId().getEntityType())) { - if (TenantId.SYS_TENANT_ID.equals(componentLifecycleMsg.getTenantId())) { - jwtSettingsService.ifPresent(JwtSettingsService::reloadJwtSettings); - return; - } else { - tenantProfileCache.evict(componentLifecycleMsg.getTenantId()); - partitionService.removeTenant(componentLifecycleMsg.getTenantId()); - if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.UPDATED)) { - apiUsageStateService.onTenantUpdate(componentLifecycleMsg.getTenantId()); - } else if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.DELETED)) { - apiUsageStateService.onTenantDelete((TenantId) componentLifecycleMsg.getEntityId()); - } - } - } else if (EntityType.DEVICE_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { - deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceProfileId(componentLifecycleMsg.getEntityId().getId())); - } else if (EntityType.DEVICE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { - deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceId(componentLifecycleMsg.getEntityId().getId())); - } else if (EntityType.ASSET_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { - assetProfileCache.evict(componentLifecycleMsg.getTenantId(), new AssetProfileId(componentLifecycleMsg.getEntityId().getId())); - } else if (EntityType.ASSET.equals(componentLifecycleMsg.getEntityId().getEntityType())) { - assetProfileCache.evict(componentLifecycleMsg.getTenantId(), new AssetId(componentLifecycleMsg.getEntityId().getId())); - } else if (EntityType.ENTITY_VIEW.equals(componentLifecycleMsg.getEntityId().getEntityType())) { - actorContext.getTbEntityViewService().onComponentLifecycleMsg(componentLifecycleMsg); - } else if (EntityType.API_USAGE_STATE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { - apiUsageStateService.onApiUsageStateUpdate(componentLifecycleMsg.getTenantId()); - } else if (EntityType.CUSTOMER.equals(componentLifecycleMsg.getEntityId().getEntityType())) { - if (componentLifecycleMsg.getEvent() == ComponentLifecycleEvent.DELETED) { - apiUsageStateService.onCustomerDelete((CustomerId) componentLifecycleMsg.getEntityId()); + apiUsageStateService.onTenantUpdate(componentLifecycleMsg.getTenantId()); + } else if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.DELETED)) { + apiUsageStateService.onTenantDelete((TenantId) componentLifecycleMsg.getEntityId()); } } - eventPublisher.publishEvent(componentLifecycleMsg); + } else if (EntityType.DEVICE_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { + deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceProfileId(componentLifecycleMsg.getEntityId().getId())); + } else if (EntityType.DEVICE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { + deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceId(componentLifecycleMsg.getEntityId().getId())); + } else if (EntityType.ASSET_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { + assetProfileCache.evict(componentLifecycleMsg.getTenantId(), new AssetProfileId(componentLifecycleMsg.getEntityId().getId())); + } else if (EntityType.ASSET.equals(componentLifecycleMsg.getEntityId().getEntityType())) { + assetProfileCache.evict(componentLifecycleMsg.getTenantId(), new AssetId(componentLifecycleMsg.getEntityId().getId())); + } else if (EntityType.ENTITY_VIEW.equals(componentLifecycleMsg.getEntityId().getEntityType())) { + actorContext.getTbEntityViewService().onComponentLifecycleMsg(componentLifecycleMsg); + } else if (EntityType.API_USAGE_STATE.equals(componentLifecycleMsg.getEntityId().getEntityType())) { + apiUsageStateService.onApiUsageStateUpdate(componentLifecycleMsg.getTenantId()); + } else if (EntityType.CUSTOMER.equals(componentLifecycleMsg.getEntityId().getEntityType())) { + if (componentLifecycleMsg.getEvent() == ComponentLifecycleEvent.DELETED) { + apiUsageStateService.onCustomerDelete((CustomerId) componentLifecycleMsg.getEntityId()); + } } - log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg); - actorContext.tellWithHighPriority(actorMsg); + eventPublisher.publishEvent(componentLifecycleMsg); } + log.trace("[{}] Forwarding component lifecycle message to App Actor {}", id, actorMsg); + actorContext.tellWithHighPriority(actorMsg); } protected abstract void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) throws Exception; @@ -222,13 +222,10 @@ public abstract class AbstractConsumerService partitions; + + public TbQueueConsumerManagerTask(QueueEvent event) { + this.event = event; + } + + public TbQueueConsumerManagerTask(QueueEvent event, Queue queue) { + this.event = event; + this.queue = queue; + } + + public TbQueueConsumerManagerTask(QueueEvent event, Set partitions) { + this.event = event; + this.partitions = partitions; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java new file mode 100644 index 0000000000..59d55285f0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java @@ -0,0 +1,70 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.ruleengine; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import java.util.Set; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +@RequiredArgsConstructor +@Slf4j +public class TbQueueConsumerTask { + + @Getter + private final Object key; + @Getter + private final TbQueueConsumer> consumer; + + @Setter + private Future task; + + public void subscribe(Set partitions) { + log.trace("[{}] Subscribing to partitions: {}", key, partitions); + consumer.subscribe(partitions); + } + + public void initiateStop() { + log.debug("[{}] Initiating stop", key); + consumer.stop(); + } + + public void awaitCompletion() { + log.trace("[{}] Awaiting finish", key); + if (isRunning()) { + try { + task.get(30, TimeUnit.SECONDS); + log.trace("[{}] Awaited finish", key); + } catch (Exception e) { + log.warn("[{}] Failed to await for consumer to stop", key, e); + } + task = null; + } + } + + public boolean isRunning() { + return task != null; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java new file mode 100644 index 0000000000..da2a5d0db4 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineConsumerContext.java @@ -0,0 +1,89 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.ruleengine; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; +import org.thingsboard.server.service.stats.RuleEngineStatisticsService; + +import javax.annotation.PostConstruct; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +@Component +@TbRuleEngineComponent +@Slf4j +@Data +public class TbRuleEngineConsumerContext { + + @Value("${queue.rule-engine.poll-interval}") + private long pollDuration; + @Value("${queue.rule-engine.pack-processing-timeout}") + private long packProcessingTimeout; + @Value("${queue.rule-engine.stats.enabled:true}") + private boolean statsEnabled; + @Value("${queue.rule-engine.prometheus-stats.enabled:false}") + private boolean prometheusStatsEnabled; + @Value("${queue.rule-engine.topic-deletion-delay:15}") + private int topicDeletionDelayInSec; + @Value("${queue.rule-engine.management-thread-pool-size:12}") + private int mgmtThreadPoolSize; + + private final ActorSystemContext actorContext; + private final StatsFactory statsFactory; + private final TbRuleEngineSubmitStrategyFactory submitStrategyFactory; + private final TbRuleEngineProcessingStrategyFactory processingStrategyFactory; + private final TbRuleEngineQueueFactory queueFactory; + private final RuleEngineStatisticsService statisticsService; + private final TbServiceInfoProvider serviceInfoProvider; + private final PartitionService partitionService; + private final TbQueueProducerProvider producerProvider; + private final TbQueueAdmin queueAdmin; + + private ExecutorService consumersExecutor; + private ExecutorService mgmtExecutor; + private ScheduledExecutorService scheduler; + + private volatile boolean isReady = false; + + @PostConstruct + void init() { + this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer")); + this.mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(mgmtThreadPoolSize, "tb-rule-engine-mgmt"); + this.scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-scheduler")); + } + + public void stop() { + scheduler.shutdownNow(); + consumersExecutor.shutdownNow(); + mgmtExecutor.shutdownNow(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java new file mode 100644 index 0000000000..5a59cb124e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -0,0 +1,486 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.ruleengine; + +import com.google.protobuf.ProtocolStringList; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.gen.MsgProtos; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.RuleNodeInfo; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.service.queue.TbMsgPackCallback; +import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; +import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +@Slf4j +public class TbRuleEngineQueueConsumerManager { + + public static final String SUCCESSFUL_STATUS = "successful"; + public static final String FAILED_STATUS = "failed"; + + private final TbRuleEngineConsumerContext ctx; + private final QueueKey queueKey; + private final TbRuleEngineConsumerStats stats; + private final ReentrantLock lock = new ReentrantLock(); //NonfairSync + + @Getter + private volatile Queue queue; + @Getter + private volatile Set partitions; + private volatile ConsumerWrapper consumerWrapper; + + private volatile boolean stopped; + + private final java.util.Queue tasks = new ConcurrentLinkedQueue<>(); + + public TbRuleEngineQueueConsumerManager(TbRuleEngineConsumerContext ctx, QueueKey queueKey) { + this.ctx = ctx; + this.queueKey = queueKey; + this.stats = new TbRuleEngineConsumerStats(queueKey, ctx.getStatsFactory()); + } + + public void init(Queue queue) { + this.queue = queue; + if (queue.isConsumerPerPartition()) { + this.consumerWrapper = new ConsumerPerPartitionWrapper(); + } else { + this.consumerWrapper = new SingleConsumerWrapper(); + } + log.debug("[{}] Initialized consumer for queue: {}", queueKey, queue); + } + + public void update(Queue queue) { + addTask(new TbQueueConsumerManagerTask(QueueEvent.CONFIG_UPDATE, queue)); + } + + public void update(Set partitions) { + addTask(new TbQueueConsumerManagerTask(QueueEvent.PARTITION_CHANGE, partitions)); + } + + public void delete() { + addTask(new TbQueueConsumerManagerTask(QueueEvent.DELETE)); + } + + private void addTask(TbQueueConsumerManagerTask todo) { + if (stopped) { + return; + } + tasks.add(todo); + log.trace("[{}] Added task: {}", queueKey, todo); + tryProcessTasks(); + } + + private void tryProcessTasks() { + if (!ctx.isReady()) { + log.debug("[{}] TbRuleEngineConsumerContext is not ready yet, will process tasks later", queueKey); + ctx.getScheduler().schedule(this::tryProcessTasks, 1, TimeUnit.SECONDS); + return; + } + ctx.getMgmtExecutor().submit(() -> { + if (lock.tryLock()) { + try { + Queue newConfiguration = null; + Set newPartitions = null; + while (!stopped) { + TbQueueConsumerManagerTask task = tasks.poll(); + if (task == null) { + break; + } + log.trace("[{}] Processing task: {}", queueKey, task); + + if (task.getEvent() == QueueEvent.PARTITION_CHANGE) { + newPartitions = task.getPartitions(); + } else if (task.getEvent() == QueueEvent.CONFIG_UPDATE) { + newConfiguration = task.getQueue(); + } else if (task.getEvent() == QueueEvent.DELETE) { + doDelete(); + return; + } + } + if (stopped) { + return; + } + if (newConfiguration != null) { + doUpdate(newConfiguration); + } + if (newPartitions != null) { + doUpdate(newPartitions); + } + } catch (Exception e) { + log.error("[{}] Failed to process tasks", queueKey, e); + } finally { + lock.unlock(); + } + } else { + log.trace("[{}] Failed to acquire lock", queueKey); + ctx.getScheduler().schedule(this::tryProcessTasks, 1, TimeUnit.SECONDS); + } + }); + } + + private void doUpdate(Queue newQueue) { + log.info("[{}] Processing queue update: {}", queueKey, newQueue); + var oldQueue = this.queue; + this.queue = newQueue; + if (log.isTraceEnabled()) { + log.trace("[{}] Old queue configuration: {}", queueKey, oldQueue); + log.trace("[{}] New queue configuration: {}", queueKey, newQueue); + } + + if (oldQueue == null) { + init(queue); + } else if (newQueue.isConsumerPerPartition() != oldQueue.isConsumerPerPartition()) { + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::initiateStop); + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitCompletion); + + init(queue); + if (partitions != null) { + doUpdate(partitions); // even if partitions number was changed, there can be no partition change event + } + } else { + // do nothing, because partitions change (if they changed) will be handled on PartitionChangeEvent, + // and changes to pollInterval/packProcessingTimeout/submitStrategy/processingStrategy will be picked up by consumer on the fly, + // and queue topic and name are immutable + } + } + + private void doUpdate(Set partitions) { + this.partitions = partitions; + consumerWrapper.updatePartitions(partitions); + } + + public void stop() { + log.debug("[{}] Stopping consumers", queueKey); + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::initiateStop); + stopped = true; + } + + public void awaitStop() { + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitCompletion); + log.debug("[{}] Unsubscribed and stopped consumers", queueKey); + } + + private void doDelete() { + stopped = true; + log.info("[{}] Handling queue deletion", queueKey); + consumerWrapper.getConsumers().forEach(TbQueueConsumerTask::awaitCompletion); + + List>> queueConsumers = consumerWrapper.getConsumers().stream() + .map(TbQueueConsumerTask::getConsumer).collect(Collectors.toList()); + ctx.getConsumersExecutor().submit(() -> { + drainQueue(queueConsumers); + + queueConsumers.forEach(consumer -> { + for (String topic : consumer.getFullTopicNames()) { + try { + ctx.getQueueAdmin().deleteTopic(topic); + log.info("Deleted topic {}", topic); + } catch (Exception e) { + log.error("Failed to delete topic {}", topic, e); + } + } + try { + consumer.unsubscribe(); + } catch (Exception e) { + log.error("[{}] Failed to unsubscribe consumer", queueKey, e); + } + }); + }); + } + + private void launchConsumer(TbQueueConsumerTask consumerTask) { + log.info("[{}] Launching consumer", consumerTask.getKey()); + Future consumerLoop = ctx.getConsumersExecutor().submit(() -> { + ThingsBoardThreadFactory.updateCurrentThreadName(consumerTask.getKey().toString()); + try { + consumerLoop(consumerTask.getConsumer()); + } catch (Throwable e) { + log.error("Failure in consumer loop", e); + } + }); + consumerTask.setTask(consumerLoop); + } + + private void consumerLoop(TbQueueConsumer> consumer) { + while (!stopped && !consumer.isStopped()) { + try { + List> msgs = consumer.poll(queue.getPollInterval()); + if (msgs.isEmpty()) { + continue; + } + processMsgs(msgs, consumer, queue); + } catch (Exception e) { + if (!consumer.isStopped()) { + log.warn("Failed to process messages from queue", e); + try { + Thread.sleep(ctx.getPollDuration()); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } + } + } + } + if (consumer.isStopped()) { + consumer.unsubscribe(); + } + log.info("Rule Engine consumer stopped"); + } + + private void processMsgs(List> msgs, + TbQueueConsumer> consumer, + Queue queue) throws InterruptedException { + TbRuleEngineSubmitStrategy submitStrategy = getSubmitStrategy(queue); + TbRuleEngineProcessingStrategy ackStrategy = getProcessingStrategy(queue); + submitStrategy.init(msgs); + while (!stopped && !consumer.isStopped()) { + TbMsgPackProcessingContext packCtx = new TbMsgPackProcessingContext(queue.getName(), submitStrategy, ackStrategy.isSkipTimeoutMsgs()); + submitStrategy.submitAttempt((id, msg) -> submitMessage(packCtx, id, msg)); + + final boolean timeout = !packCtx.await(queue.getPackProcessingTimeout(), TimeUnit.MILLISECONDS); + + TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(queue.getName(), timeout, packCtx); + if (timeout) { + printFirstOrAll(packCtx, packCtx.getPendingMap(), "Timeout"); + } + if (!packCtx.getFailedMap().isEmpty()) { + printFirstOrAll(packCtx, packCtx.getFailedMap(), "Failed"); + } + packCtx.printProfilerStats(); + + TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result); + if (ctx.isStatsEnabled()) { + stats.log(result, decision.isCommit()); + } + + packCtx.cleanup(); + + if (decision.isCommit()) { + submitStrategy.stop(); + consumer.commit(); + break; + } else { + submitStrategy.update(decision.getReprocessMap()); + } + } + } + + private TbRuleEngineSubmitStrategy getSubmitStrategy(Queue queue) { + return ctx.getSubmitStrategyFactory().newInstance(queue.getName(), queue.getSubmitStrategy()); + } + + private TbRuleEngineProcessingStrategy getProcessingStrategy(Queue queue) { + return ctx.getProcessingStrategyFactory().newInstance(queue.getName(), queue.getProcessingStrategy()); + } + + private void submitMessage(TbMsgPackProcessingContext packCtx, UUID id, TbProtoQueueMsg msg) { + log.trace("[{}] Creating callback for topic {} message: {}", id, queue.getName(), msg.getValue()); + ToRuleEngineMsg toRuleEngineMsg = msg.getValue(); + TenantId tenantId = TenantId.fromUUID(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB())); + TbMsgCallback callback = ctx.isPrometheusStatsEnabled() ? + new TbMsgPackCallback(id, tenantId, packCtx, stats.getTimer(tenantId, SUCCESSFUL_STATUS), stats.getTimer(tenantId, FAILED_STATUS)) : + new TbMsgPackCallback(id, tenantId, packCtx); + try { + if (!toRuleEngineMsg.getTbMsg().isEmpty()) { + forwardToRuleEngineActor(queue.getName(), tenantId, toRuleEngineMsg, callback); + } else { + callback.onSuccess(); + } + } catch (Exception e) { + callback.onFailure(new RuleEngineException(e.getMessage(), e)); + } + } + + private void forwardToRuleEngineActor(String queueName, TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { + TbMsg tbMsg = TbMsg.fromBytes(queueName, toRuleEngineMsg.getTbMsg().toByteArray(), callback); + QueueToRuleEngineMsg msg; + ProtocolStringList relationTypesList = toRuleEngineMsg.getRelationTypesList(); + Set relationTypes; + if (relationTypesList.size() == 1) { + relationTypes = Collections.singleton(relationTypesList.get(0)); + } else { + relationTypes = new HashSet<>(relationTypesList); + } + msg = new QueueToRuleEngineMsg(tenantId, tbMsg, relationTypes, toRuleEngineMsg.getFailureMessage()); + ctx.getActorContext().tell(msg); + } + + private void printFirstOrAll(TbMsgPackProcessingContext ctx, Map> map, String prefix) { + boolean printAll = log.isTraceEnabled(); + log.info("[{}] {} to process [{}] messages", queueKey, prefix, map.size()); + for (Map.Entry> pending : map.entrySet()) { + ToRuleEngineMsg tmp = pending.getValue().getValue(); + TbMsg tmpMsg = TbMsg.fromBytes(queue.getName(), tmp.getTbMsg().toByteArray(), TbMsgCallback.EMPTY); + RuleNodeInfo ruleNodeInfo = ctx.getLastVisitedRuleNode(pending.getKey()); + if (printAll) { + log.trace("[{}][{}] {} to process message: {}, Last Rule Node: {}", queueKey, TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); + } else { + log.info("[{}] {} to process message: {}, Last Rule Node: {}", TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo); + break; + } + } + } + + public void printStats(long ts) { + stats.printStats(); + ctx.getStatisticsService().reportQueueStats(ts, stats); + stats.reset(); + } + + private void drainQueue(List>> consumers) { + long finishTs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(ctx.getTopicDeletionDelayInSec()); + try { + int n = 0; + while (System.currentTimeMillis() <= finishTs) { + for (TbQueueConsumer> consumer : consumers) { + List> msgs = consumer.poll(queue.getPollInterval()); + if (msgs.isEmpty()) { + continue; + } + for (TbProtoQueueMsg msg : msgs) { + try { + MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray()); + EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB())); + + TopicPartitionInfo tpi = ctx.getPartitionService().resolve(ServiceType.TB_RULE_ENGINE, queue.getName(), TenantId.SYS_TENANT_ID, originator); + ctx.getProducerProvider().getRuleEngineMsgProducer().send(tpi, msg, null); + n++; + } catch (Throwable e) { + log.warn("Failed to move message to system {}: {}", consumer.getTopic(), msg, e); + } + } + consumer.commit(); + } + } + if (n > 0) { + log.info("Moved {} messages from {} to system {}", n, queueKey, queue.getName()); + } + } catch (Exception e) { + log.error("[{}] Failed to drain queue", queueKey, e); + } + } + + private static String partitionsToString(Collection partitions) { + return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.joining(", ", "[", "]")); + } + + interface ConsumerWrapper { + + void updatePartitions(Set partitions); + + Collection getConsumers(); + + } + + class ConsumerPerPartitionWrapper implements ConsumerWrapper { + private final Map consumers = new HashMap<>(); + + @Override + public void updatePartitions(Set partitions) { + Set addedPartitions = new HashSet<>(partitions); + addedPartitions.removeAll(consumers.keySet()); + + Set removedPartitions = new HashSet<>(consumers.keySet()); + removedPartitions.removeAll(partitions); + log.info("[{}] Added partitions: {}, removed partitions: {}", queueKey, partitionsToString(addedPartitions), partitionsToString(removedPartitions)); + + removedPartitions.forEach((tpi) -> { + consumers.get(tpi).initiateStop(); + }); + removedPartitions.forEach((tpi) -> { + consumers.remove(tpi).awaitCompletion(); + }); + + addedPartitions.forEach((tpi) -> { + String key = queueKey + "-" + tpi.getPartition().orElse(-999999); + TbQueueConsumerTask consumer = new TbQueueConsumerTask(key, ctx.getQueueFactory().createToRuleEngineMsgConsumer(queue)); + consumers.put(tpi, consumer); + consumer.subscribe(Set.of(tpi)); + launchConsumer(consumer); + }); + } + + @Override + public Collection getConsumers() { + return consumers.values(); + } + } + + class SingleConsumerWrapper implements ConsumerWrapper { + private TbQueueConsumerTask consumer; + + @Override + public void updatePartitions(Set partitions) { + log.info("[{}] New partitions: {}", queueKey, partitionsToString(partitions)); + if (partitions.isEmpty()) { + if (consumer != null && consumer.isRunning()) { + consumer.initiateStop(); + consumer.awaitCompletion(); + } + consumer = null; + return; + } + + if (consumer == null) { + consumer = new TbQueueConsumerTask(queueKey, ctx.getQueueFactory().createToRuleEngineMsgConsumer(queue)); + } + consumer.subscribe(partitions); + if (!consumer.isRunning()) { + launchConsumer(consumer); + } + } + + @Override + public Collection getConsumers() { + if (consumer == null) { + return Collections.emptyList(); + } + return List.of(consumer); + } + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index b511989c8b..b8638b5f6f 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1611,7 +1611,9 @@ queue: pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}" # Time in seconds to wait in consumer thread before retries; max-pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}" # Max allowed time in seconds for pause between retries. # After a queue is deleted (or the profile's isolation option was disabled), Rule Engine will continue reading related topics during this period before deleting the actual topics - topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:30}" + topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:15}" + # Size of the thread pool that handles such operations as partition changes, config updates, queue deletion + management-thread-pool-size: "${TB_QUEUE_RULE_ENGINE_MGMT_THREAD_POOL_SIZE:12}" transport: # For high-priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" diff --git a/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java index e865f8e115..087c2a2106 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java @@ -139,7 +139,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(AlarmSeverity.MAJOR, updatedAlarm.getSeverity()); AlarmInfo foundAlarm = doGet("/api/alarm/info/" + updatedAlarm.getId(), AlarmInfo.class); - testNotifyEntityAllOneTime(foundAlarm, updatedAlarm.getId(), updatedAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, updatedAlarm.getId(), updatedAlarm.getOriginator(), tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.UPDATED); } @@ -156,7 +156,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(AlarmSeverity.MAJOR, updatedAlarm.getSeverity()); AlarmInfo foundAlarm = doGet("/api/alarm/info/" + updatedAlarm.getId(), AlarmInfo.class); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.UPDATED); alarm = updatedAlarm; diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java index 845ac49f15..8418ba67d9 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java @@ -37,12 +37,14 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.queue.SubmitStrategy; import org.thingsboard.server.common.data.queue.SubmitStrategyType; import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.timeseries.TimeseriesDao; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; import org.thingsboard.server.service.stats.DefaultRuleEngineStatisticsService; @@ -163,7 +165,7 @@ public class BaseQueueControllerTest extends AbstractControllerTest { tenantId, ruleEngineException ))); - TbRuleEngineConsumerStats testStats = new TbRuleEngineConsumerStats(queue, statsFactory); + TbRuleEngineConsumerStats testStats = new TbRuleEngineConsumerStats(new QueueKey(ServiceType.TB_RULE_ENGINE, queue), statsFactory); testStats.log(testProcessingResult, true); int queueStatsTtlDays = 14; @@ -215,7 +217,7 @@ public class BaseQueueControllerTest extends AbstractControllerTest { tenantId, ruleEngineException ))); - TbRuleEngineConsumerStats testStats = new TbRuleEngineConsumerStats(queue, statsFactory); + TbRuleEngineConsumerStats testStats = new TbRuleEngineConsumerStats(new QueueKey(ServiceType.TB_RULE_ENGINE, queue), statsFactory); testStats.log(testProcessingResult, true); ruleEngineStatisticsService.reportQueueStats(System.currentTimeMillis(), testStats); diff --git a/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java index 1afeaff0cb..b9e0b8cd6c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.controller; import com.datastax.oss.driver.api.core.uuid.Uuids; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -34,8 +35,10 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Primary; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.StringUtils; @@ -852,6 +855,11 @@ public class EdgeControllerTest extends AbstractControllerTest { Edge edge = doPost("/api/edge", constructEdge("Test Sync Edge", "test"), Edge.class); + // simulate edge activation + ObjectNode attributes = JacksonUtil.newObjectNode(); + attributes.put("active", true); + doPost("/api/plugins/telemetry/EDGE/" + edge.getId() + "/attributes/" + DataConstants.SERVER_SCOPE, attributes); + doPost("/api/edge/" + edge.getId().getId().toString() + "/device/" + savedDevice.getId().getId().toString(), Device.class); doPost("/api/edge/" + edge.getId().getId().toString() @@ -860,13 +868,12 @@ public class EdgeControllerTest extends AbstractControllerTest { EdgeImitator edgeImitator = new EdgeImitator(EDGE_HOST, EDGE_PORT, edge.getRoutingKey(), edge.getSecret()); edgeImitator.ignoreType(UserCredentialsUpdateMsg.class); - edgeImitator.expectMessageAmount(25); + edgeImitator.expectMessageAmount(24); edgeImitator.connect(); assertThat(edgeImitator.waitForMessages()).as("await for messages on first connect").isTrue(); verifyFetchersMsgs(edgeImitator); // verify queue msgs - Assert.assertTrue(popRuleChainMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE, "Edge Root Rule Chain")); Assert.assertTrue(popDeviceProfileMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "default")); Assert.assertTrue(popDeviceMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "Test Sync Edge Device 1")); Assert.assertTrue(popAssetProfileMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "test")); diff --git a/application/src/test/java/org/thingsboard/server/controller/EdgeEventControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdgeEventControllerTest.java index 04842df6a0..3303c64c27 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdgeEventControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdgeEventControllerTest.java @@ -16,6 +16,7 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import org.awaitility.Awaitility; import org.junit.After; @@ -26,6 +27,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.TestPropertySource; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.edge.Edge; @@ -86,6 +89,11 @@ public class EdgeEventControllerTest extends AbstractControllerTest { Edge edge = constructEdge("TestEdge", "default"); edge = doPost("/api/edge", edge, Edge.class); + // simulate edge activation + ObjectNode attributes = JacksonUtil.newObjectNode(); + attributes.put("active", true); + doPost("/api/plugins/telemetry/EDGE/" + edge.getId() + "/attributes/" + DataConstants.SERVER_SCOPE, attributes); + Device device = constructDevice("TestDevice", "default"); Device savedDevice = doPost("/api/device", device, Device.class); @@ -99,14 +107,13 @@ public class EdgeEventControllerTest extends AbstractControllerTest { EntityRelation relation = new EntityRelation(savedAsset.getId(), savedDevice.getId(), EntityRelation.CONTAINS_TYPE); - awaitForNumberOfEdgeEvents(edgeId, 3); + awaitForNumberOfEdgeEvents(edgeId, 2); doPost("/api/relation", relation); - awaitForNumberOfEdgeEvents(edgeId, 4); + awaitForNumberOfEdgeEvents(edgeId, 3); List edgeEvents = findEdgeEvents(edgeId); - Assert.assertTrue(popEdgeEvent(edgeEvents, EdgeEventType.RULE_CHAIN)); // root rule chain Assert.assertTrue(popEdgeEvent(edgeEvents, EdgeEventType.DEVICE)); // TestDevice Assert.assertTrue(popEdgeEvent(edgeEvents, EdgeEventType.ASSET)); // TestAsset Assert.assertTrue(popEdgeEvent(edgeEvents, EdgeEventType.RELATION)); diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java index 4b594d55f9..27a9872ba5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java @@ -42,6 +42,7 @@ import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; @@ -54,12 +55,12 @@ import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; -import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.discovery.PartitionService; @@ -67,9 +68,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; -import java.util.Deque; import java.util.HashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Random; @@ -680,26 +679,29 @@ public class TenantControllerTest extends AbstractControllerTest { .until(() -> partitionService.resolve(ServiceType.TB_RULE_ENGINE, MAIN_QUEUE_NAME, tenantId, tenantId) .getTenantId().get().isSysTenantId()); - Deque submittedMsgs = new LinkedList<>(); - await().atLeast(8, TimeUnit.SECONDS) // due to topic-deletion-delay - .atMost(20, TimeUnit.SECONDS) - .pollInterval(1, TimeUnit.SECONDS) - .untilAsserted(() -> { - TbMsg tbMsg = publishTbMsg(tenantId, tpi); - submittedMsgs.add(tbMsg.getId()); - - verify(queueAdmin, times(1)).deleteTopic(eq(isolatedTopic)); - }); - submittedMsgs.removeLast(); - for (UUID msgId : submittedMsgs) { - verify(actorContext, timeout(2000)).tell(argThat(msg -> { - return msg instanceof QueueToRuleEngineMsg && ((QueueToRuleEngineMsg) msg).getMsg().getId().equals(msgId); - })); + List submittedMsgs = new ArrayList<>(); + long timeLeft = TimeUnit.SECONDS.toMillis(7); // based on topic-deletion-delay + int msgs = 100; + for (int i = 1; i <= msgs; i++) { + TbMsg tbMsg = publishTbMsg(tenantId, tpi); + submittedMsgs.add(tbMsg.getId()); + Thread.sleep(timeLeft / msgs); } + await().atMost(15, TimeUnit.SECONDS).untilAsserted(() -> { + verify(queueAdmin, times(1)).deleteTopic(eq(isolatedTopic)); + }); + + await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> { + for (UUID msgId : submittedMsgs) { + verify(actorContext).tell(argThat(msg -> { + return msg instanceof QueueToRuleEngineMsg && ((QueueToRuleEngineMsg) msg).getMsg().getId().equals(msgId); + })); + } + }); } private TbMsg publishTbMsg(TenantId tenantId, TopicPartitionInfo tpi) { - TbMsg tbMsg = TbMsg.newMsg("POST_TELEMETRY_REQUEST", tenantId, TbMsgMetaData.EMPTY, "{\"test\":1}"); + TbMsg tbMsg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, tenantId, TbMsgMetaData.EMPTY, "{\"test\":1}"); TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) diff --git a/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java index 689847416c..43544dd5cf 100644 --- a/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java @@ -130,7 +130,7 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { installation(); edgeImitator = new EdgeImitator("localhost", 7070, edge.getRoutingKey(), edge.getSecret()); - edgeImitator.expectMessageAmount(26); + edgeImitator.expectMessageAmount(21); edgeImitator.connect(); requestEdgeRuleChainMetadata(); @@ -163,9 +163,7 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { @After public void teardownEdgeTest() { try { - edgeImitator.expectMessageAmount(2); loginTenantAdmin(); - Assert.assertTrue(edgeImitator.waitForMessages()); doDelete("/api/edge/" + edge.getId().toString()) .andExpect(status().isOk()); @@ -228,33 +226,33 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { // 1 message from queue fetcher validateQueues(); - // 2 messages - 1 from rule chain fetcher and 1 from rule chain controller + // 1 from rule chain fetcher UUID ruleChainUUID = validateRuleChains(); // 1 from request message validateRuleChainMetadataUpdates(ruleChainUUID); - // 4 messages - 4 messages from fetcher - 2 from system level ('mail', 'mailTemplates') and 2 from admin level ('mail', 'mailTemplates') + // 4 messages + // - 2 from fetcher - system level ('mail', 'mailTemplates') + // - 2 from fetcher - admin level ('mail', 'mailTemplates') validateAdminSettings(); - // 5 messages + // 4 messages // - 1 from default profile fetcher // - 2 from device profile fetcher (default and thermostat) // - 1 from device fetcher - // - 1 from device controller (thermostat) validateDeviceProfiles(); - // 4 messages + // 3 messages // - 1 from default profile fetcher // - 1 message from asset profile fetcher // - 1 message from asset fetcher - // - 1 message from asset controller validateAssetProfiles(); - // 2 messages - 1 from device fetcher and 1 from device controller + // 1 from device fetcher validateDevices(); - // 2 messages - 1 from asset fetcher and 1 from asset controller + // 1 from asset fetcher validateAssets(); // 1 message from public customer fetcher @@ -308,8 +306,7 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { // default msg device profile from fetcher // thermostat msg from device profile fetcher // thermostat msg from device fetcher - // thermostat msg from creation of device - Assert.assertEquals(5, deviceProfileUpdateMsgList.size()); + Assert.assertEquals(4, deviceProfileUpdateMsgList.size()); Optional thermostatProfileUpdateMsgOpt = deviceProfileUpdateMsgList.stream().filter(dfum -> THERMOSTAT_DEVICE_PROFILE_NAME.equals(dfum.getName())).findAny(); Assert.assertTrue(thermostatProfileUpdateMsgOpt.isPresent()); @@ -326,10 +323,9 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { } private void validateDevices() throws Exception { - List deviceUpdateMsgs = edgeImitator.findAllMessagesByType(DeviceUpdateMsg.class); - Assert.assertEquals(2, deviceUpdateMsgs.size()); - validateDevice(deviceUpdateMsgs.get(0)); - validateDevice(deviceUpdateMsgs.get(1)); + Optional deviceUpdateMsgOpt = edgeImitator.findMessageByType(DeviceUpdateMsg.class); + Assert.assertTrue(deviceUpdateMsgOpt.isPresent()); + validateDevice(deviceUpdateMsgOpt.get()); } private void validateDevice(DeviceUpdateMsg deviceUpdateMsg) throws Exception { @@ -345,10 +341,9 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { } private void validateAssets() throws Exception { - List assetUpdateMsgs = edgeImitator.findAllMessagesByType(AssetUpdateMsg.class); - Assert.assertEquals(2, assetUpdateMsgs.size()); - validateAsset(assetUpdateMsgs.get(0)); - validateAsset(assetUpdateMsgs.get(1)); + Optional assetUpdateMsgOpt = edgeImitator.findMessageByType(AssetUpdateMsg.class); + Assert.assertTrue(assetUpdateMsgOpt.isPresent()); + validateAsset(assetUpdateMsgOpt.get()); } private void validateAsset(AssetUpdateMsg assetUpdateMsg) throws Exception { @@ -365,12 +360,10 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { } private UUID validateRuleChains() throws Exception { - List ruleChainUpdateMsgs = edgeImitator.findAllMessagesByType(RuleChainUpdateMsg.class); - Assert.assertEquals(2, ruleChainUpdateMsgs.size()); - RuleChainUpdateMsg ruleChainCreateMsg = ruleChainUpdateMsgs.get(0); - RuleChainUpdateMsg ruleChainUpdateMsg = ruleChainUpdateMsgs.get(1); - validateRuleChain(ruleChainCreateMsg, UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE); - validateRuleChain(ruleChainUpdateMsg, UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE); + Optional ruleChainUpdateMsgOpt = edgeImitator.findMessageByType(RuleChainUpdateMsg.class); + Assert.assertTrue(ruleChainUpdateMsgOpt.isPresent()); + RuleChainUpdateMsg ruleChainUpdateMsg = ruleChainUpdateMsgOpt.get(); + validateRuleChain(ruleChainUpdateMsg, UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE); return new UUID(ruleChainUpdateMsg.getIdMSB(), ruleChainUpdateMsg.getIdLSB()); } @@ -429,7 +422,7 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { private void validateAssetProfiles() throws Exception { List assetProfileUpdateMsgs = edgeImitator.findAllMessagesByType(AssetProfileUpdateMsg.class); - Assert.assertEquals(4, assetProfileUpdateMsgs.size()); + Assert.assertEquals(3, assetProfileUpdateMsgs.size()); AssetProfileUpdateMsg assetProfileUpdateMsg = assetProfileUpdateMsgs.get(0); Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, assetProfileUpdateMsg.getMsgType()); UUID assetProfileUUID = new UUID(assetProfileUpdateMsg.getIdMSB(), assetProfileUpdateMsg.getIdLSB()); diff --git a/application/src/test/java/org/thingsboard/server/edge/AlarmEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/AlarmEdgeTest.java index afccc634ed..259572851c 100644 --- a/application/src/test/java/org/thingsboard/server/edge/AlarmEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/AlarmEdgeTest.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.google.protobuf.AbstractMessage; import org.junit.Assert; import org.junit.Test; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; @@ -102,20 +101,6 @@ public class AlarmEdgeTest extends AbstractEdgeTest { Assert.assertEquals(savedAlarm.getStatus().name(), alarmUpdateMsg.getStatus()); Assert.assertEquals(savedAlarm.getSeverity().name(), alarmUpdateMsg.getSeverity()); - // update alarm - String updatedDetails = "{\"testKey\":\"testValue\"}"; - savedAlarm.setDetails(JacksonUtil.OBJECT_MAPPER.readTree(updatedDetails)); - edgeImitator.expectMessageAmount(1); - savedAlarm = doPost("/api/alarm", savedAlarm, Alarm.class); - Assert.assertTrue(edgeImitator.waitForMessages()); - latestMessage = edgeImitator.getLatestMessage(); - Assert.assertTrue(latestMessage instanceof AlarmUpdateMsg); - alarmUpdateMsg = (AlarmUpdateMsg) latestMessage; - Assert.assertEquals(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE, alarmUpdateMsg.getMsgType()); - Assert.assertEquals(savedAlarm.getUuidId().getMostSignificantBits(), alarmUpdateMsg.getIdMSB()); - Assert.assertEquals(savedAlarm.getUuidId().getLeastSignificantBits(), alarmUpdateMsg.getIdLSB()); - Assert.assertEquals(updatedDetails, alarmUpdateMsg.getDetails()); - // ack alarm edgeImitator.expectMessageAmount(1); doPost("/api/alarm/" + savedAlarm.getUuidId() + "/ack"); diff --git a/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java index aed99b4cba..ca8dd5dbf0 100644 --- a/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java @@ -274,9 +274,7 @@ public class DeviceEdgeTest extends AbstractEdgeTest { tenantProfile.getProfileData().setConfiguration(profileConfiguration); doPost("/api/tenantProfile/", tenantProfile, TenantProfile.class); - edgeImitator.expectMessageAmount(2); loginTenantAdmin(); - Assert.assertTrue(edgeImitator.waitForMessages()); UUID uuid = Uuids.timeBased(); diff --git a/application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java index 7ec32f8aff..6f665ce462 100644 --- a/application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java @@ -32,6 +32,7 @@ import org.thingsboard.server.gen.edge.v1.TenantUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import java.util.List; +import java.util.Optional; import static org.assertj.core.api.Assertions.assertThat; @@ -75,17 +76,21 @@ public class TenantProfileEdgeTest extends AbstractEdgeTest { TenantProfileQueueConfiguration mainQueueConfiguration = createQueueConfig(DataConstants.MAIN_QUEUE_NAME, DataConstants.MAIN_QUEUE_TOPIC); TenantProfileQueueConfiguration isolatedQueueConfiguration = createQueueConfig("IsolatedHighPriority", "tb_rule_engine.isolated_hp"); edgeTenantProfile.getProfileData().setQueueConfiguration(List.of(mainQueueConfiguration, isolatedQueueConfiguration)); - edgeImitator.expectMessageAmount(1); + edgeImitator.expectMessageAmount(3); edgeTenantProfile = doPost("/api/tenantProfile", edgeTenantProfile, TenantProfile.class); Assert.assertTrue(edgeImitator.waitForMessages()); - AbstractMessage latestMessage = edgeImitator.getLatestMessage(); - Assert.assertTrue(latestMessage instanceof TenantProfileUpdateMsg); - TenantProfileUpdateMsg tenantProfileUpdateMsg = (TenantProfileUpdateMsg) latestMessage; + + Optional tenantProfileUpdateMsgOpt = edgeImitator.findMessageByType(TenantProfileUpdateMsg.class); + Assert.assertTrue(tenantProfileUpdateMsgOpt.isPresent()); + TenantProfileUpdateMsg tenantProfileUpdateMsg = tenantProfileUpdateMsgOpt.get(); Assert.assertEquals(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE, tenantProfileUpdateMsg.getMsgType()); Assert.assertEquals(edgeTenantProfile.getUuidId().getMostSignificantBits(), tenantProfileUpdateMsg.getIdMSB()); Assert.assertEquals(edgeTenantProfile.getUuidId().getLeastSignificantBits(), tenantProfileUpdateMsg.getIdLSB()); Assert.assertEquals(edgeTenantProfile.getDescription(), tenantProfileUpdateMsg.getDescription()); + List queueUpdateMsgs = edgeImitator.findAllMessagesByType(QueueUpdateMsg.class); + Assert.assertEquals(2, queueUpdateMsgs.size()); + loginTenantAdmin(); edgeImitator.expectMessageAmount(21); @@ -95,7 +100,7 @@ public class TenantProfileEdgeTest extends AbstractEdgeTest { Assert.assertTrue(edgeImitator.getDownlinkMsgs().get(0) instanceof TenantUpdateMsg); Assert.assertTrue(edgeImitator.getDownlinkMsgs().get(1) instanceof TenantProfileUpdateMsg); - List queueUpdateMsgs = edgeImitator.findAllMessagesByType(QueueUpdateMsg.class); + queueUpdateMsgs = edgeImitator.findAllMessagesByType(QueueUpdateMsg.class); Assert.assertEquals(2, queueUpdateMsgs.size()); for (QueueUpdateMsg queueUpdateMsg : queueUpdateMsgs) { Assert.assertEquals(tenantId.getId().getMostSignificantBits(), queueUpdateMsg.getTenantIdMSB()); diff --git a/application/src/test/java/org/thingsboard/server/edge/UserEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/UserEdgeTest.java index 3fd39328d5..3dc398b9f3 100644 --- a/application/src/test/java/org/thingsboard/server/edge/UserEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/UserEdgeTest.java @@ -79,9 +79,7 @@ public class UserEdgeTest extends AbstractEdgeTest { Assert.assertEquals(savedTenantAdmin.getLastName(), userUpdateMsg.getLastName()); // update user credentials - edgeImitator.expectMessageAmount(2); login(savedTenantAdmin.getEmail(), "tenant"); - Assert.assertTrue(edgeImitator.waitForMessages()); edgeImitator.expectMessageAmount(1); ChangePasswordRequest changePasswordRequest = new ChangePasswordRequest(); @@ -96,9 +94,7 @@ public class UserEdgeTest extends AbstractEdgeTest { Assert.assertEquals(savedTenantAdmin.getUuidId().getLeastSignificantBits(), userCredentialsUpdateMsg.getUserIdLSB()); Assert.assertTrue(passwordEncoder.matches(changePasswordRequest.getNewPassword(), userCredentialsUpdateMsg.getPassword())); - edgeImitator.expectMessageAmount(2); loginTenantAdmin(); - Assert.assertTrue(edgeImitator.waitForMessages()); // delete user edgeImitator.expectMessageAmount(1); @@ -164,9 +160,7 @@ public class UserEdgeTest extends AbstractEdgeTest { Assert.assertEquals(savedCustomerUser.getLastName(), userUpdateMsg.getLastName()); // update user credentials - edgeImitator.expectMessageAmount(2); login(savedCustomerUser.getEmail(), "customer"); - Assert.assertTrue(edgeImitator.waitForMessages()); edgeImitator.expectMessageAmount(1); ChangePasswordRequest changePasswordRequest = new ChangePasswordRequest(); @@ -181,9 +175,7 @@ public class UserEdgeTest extends AbstractEdgeTest { Assert.assertEquals(savedCustomerUser.getUuidId().getLeastSignificantBits(), userCredentialsUpdateMsg.getUserIdLSB()); Assert.assertTrue(passwordEncoder.matches(changePasswordRequest.getNewPassword(), userCredentialsUpdateMsg.getPassword())); - edgeImitator.expectMessageAmount(2); loginTenantAdmin(); - Assert.assertTrue(edgeImitator.waitForMessages()); // delete user edgeImitator.expectMessageAmount(1); diff --git a/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java index f2531a2561..0f57d5ef14 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java @@ -420,11 +420,11 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { notificationRulesCache.evict(TenantId.SYS_TENANT_ID); int n = 10; - updateDefaultTenantProfile(profileConfiguration -> { - profileConfiguration.getProfileConfiguration().get().setTenantEntityExportRateLimit(n + ":600"); - profileConfiguration.getProfileConfiguration().get().setCustomerServerRestLimitsConfiguration(n + ":600"); - profileConfiguration.getProfileConfiguration().get().setTenantNotificationRequestsPerRuleRateLimit(n + ":600"); - profileConfiguration.getProfileConfiguration().get().setTransportDeviceTelemetryMsgRateLimit(n + ":600"); + updateDefaultTenantProfileConfig(profileConfiguration -> { + profileConfiguration.setTenantEntityExportRateLimit(n + ":600"); + profileConfiguration.setCustomerServerRestLimitsConfiguration(n + ":600"); + profileConfiguration.setTenantNotificationRequestsPerRuleRateLimit(n + ":600"); + profileConfiguration.setTransportDeviceTelemetryMsgRateLimit(n + ":600"); }); loginTenantAdmin(); NotificationRule rule = createNotificationRule(AlarmCommentNotificationRuleTriggerConfig.builder() @@ -434,11 +434,14 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { rateLimitService.checkRateLimit(LimitedApi.ENTITY_EXPORT, tenantId); rateLimitService.checkRateLimit(LimitedApi.REST_REQUESTS_PER_CUSTOMER, tenantId, customerId); rateLimitService.checkRateLimit(LimitedApi.NOTIFICATION_REQUESTS_PER_RULE, tenantId, rule.getId()); + Thread.sleep(100); } loginTenantAdmin(); - List notifications = await().atMost(30, TimeUnit.SECONDS) - .until(() -> getMyNotifications(true, 10), list -> list.size() == 3); + List notifications = await().atMost(15, TimeUnit.SECONDS) + .until(() -> getMyNotifications(true, 10).stream() + .filter(notification -> notification.getType() == NotificationType.RATE_LIMITS) + .collect(Collectors.toList()), list -> list.size() == 3); assertThat(notifications).allSatisfy(notification -> { assertThat(notification.getSubject()).isEqualTo("Rate limits exceeded"); }); @@ -455,12 +458,14 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { }); loginSysAdmin(); - notifications = await().atMost(30, TimeUnit.SECONDS) - .until(() -> getMyNotifications(true, 10), list -> list.size() == 1); - assertThat(notifications).allSatisfy(notification -> { + notifications = await().atMost(15, TimeUnit.SECONDS) + .until(() -> getMyNotifications(true, 10).stream() + .filter(notification -> notification.getType() == NotificationType.RATE_LIMITS) + .collect(Collectors.toList()), list -> list.size() == 1); + assertThat(notifications).singleElement().satisfies(notification -> { assertThat(notification.getSubject()).isEqualTo("Rate limits exceeded for tenant " + TEST_TENANT_NAME); + assertThat(notification.getText()).isEqualTo("Rate limits for entity version creation exceeded"); }); - assertThat(notifications.get(0).getText()).isEqualTo("Rate limits for entity version creation exceeded"); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ProtoUtilsTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ProtoUtilsTest.java new file mode 100644 index 0000000000..e235076635 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/queue/ProtoUtilsTest.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.gen.transport.TransportProtos; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +class ProtoUtilsTest { + + TenantId tenantId = TenantId.fromUUID(UUID.fromString("35e10f77-16e7-424d-ae46-ee780f87ac4f")); + EntityId entityId = new RuleChainId(UUID.fromString("c640b635-4f0f-41e6-b10b-25a86003094e")); + @Test + void protoComponentLifecycleSerialization() { + ComponentLifecycleMsg msg = new ComponentLifecycleMsg(tenantId, entityId, ComponentLifecycleEvent.UPDATED); + assertThat(ProtoUtils.fromProto(ProtoUtils.toProto(msg))).as("deserialized").isEqualTo(msg); + msg = new ComponentLifecycleMsg(tenantId, entityId, ComponentLifecycleEvent.STARTED); + assertThat(ProtoUtils.fromProto(ProtoUtils.toProto(msg))).as("deserialized").isEqualTo(msg); + } + + @Test + void protoEntityTypeSerialization() { + for(EntityType entityType : EntityType.values()){ + assertThat(ProtoUtils.fromProto(ProtoUtils.toProto(entityType))).as(entityType.getNormalName()).isEqualTo(entityType); + } + } + +} \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java new file mode 100644 index 0000000000..05eacc68e3 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java @@ -0,0 +1,773 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.ruleengine; + +import lombok.Getter; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomUtils; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.QueueId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.data.queue.ProcessingStrategy; +import org.thingsboard.server.common.data.queue.ProcessingStrategyType; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.data.queue.SubmitStrategy; +import org.thingsboard.server.common.data.queue.SubmitStrategyType; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory; +import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory; +import org.thingsboard.server.service.stats.RuleEngineStatisticsService; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.after; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +@Slf4j +@RunWith(MockitoJUnitRunner.class) +public class TbRuleEngineQueueConsumerManagerTest { + + @Mock + private ActorSystemContext actorContext; + @Mock + private StatsFactory statsFactory; + @Mock + private TbRuleEngineQueueFactory queueFactory; + @Mock + private RuleEngineStatisticsService statisticsService; + @Mock + private TbServiceInfoProvider serviceInfoProvider; + @Mock + private PartitionService partitionService; + @Mock + private TbQueueProducerProvider producerProvider; + private TbQueueProducer> ruleEngineMsgProducer; + @Mock + private TbQueueAdmin queueAdmin; + private TbRuleEngineConsumerContext ruleEngineConsumerContext; + + private TbRuleEngineQueueConsumerManager consumerManager; + private Queue queue; + + private Set consumers; + private boolean generateQueueMsgs; + private AtomicInteger totalConsumedMsgs; + private AtomicInteger totalProcessedMsgs; + + @Before + public void beforeEach() { + ruleEngineConsumerContext = new TbRuleEngineConsumerContext( + actorContext, statsFactory, spy(new TbRuleEngineSubmitStrategyFactory()), + spy(new TbRuleEngineProcessingStrategyFactory()), queueFactory, statisticsService, + serviceInfoProvider, partitionService, producerProvider, queueAdmin + ); + consumers = ConcurrentHashMap.newKeySet(); + generateQueueMsgs = true; + totalConsumedMsgs = new AtomicInteger(); + totalProcessedMsgs = new AtomicInteger(); + doAnswer(inv -> { + QueueToRuleEngineMsg msg = inv.getArgument(0); + msg.getMsg().getCallback().onSuccess(); + totalProcessedMsgs.incrementAndGet(); + log.trace("totalProcessedMsgs = {}", totalProcessedMsgs); + return null; + }).when(actorContext).tell(any()); + ruleEngineMsgProducer = mock(TbQueueProducer.class); + when(producerProvider.getRuleEngineMsgProducer()).thenReturn(ruleEngineMsgProducer); + ruleEngineConsumerContext.setMgmtThreadPoolSize(2); + ruleEngineConsumerContext.setTopicDeletionDelayInSec(5); + ruleEngineConsumerContext.init(); + ruleEngineConsumerContext.setReady(false); + + queue = new Queue(); + queue.setName("Test"); + queue.setTenantId(TenantId.SYS_TENANT_ID); + queue.setId(new QueueId(UUID.randomUUID())); + queue.setTopic("tb_test"); + queue.setPartitions(10); + queue.setConsumerPerPartition(true); + queue.setPollInterval(250); + queue.setPackProcessingTimeout(2000); + SubmitStrategy submitStrategy = new SubmitStrategy(); + submitStrategy.setType(SubmitStrategyType.BURST); + submitStrategy.setBatchSize(200); + queue.setSubmitStrategy(submitStrategy); + ProcessingStrategy processingStrategy = new ProcessingStrategy(); + processingStrategy.setType(ProcessingStrategyType.SKIP_ALL_FAILURES_AND_TIMED_OUT); + processingStrategy.setRetries(0); + queue.setProcessingStrategy(processingStrategy); + + doAnswer(i -> { + TestConsumer consumer = spy(new TestConsumer(queue.getTopic())); + if (generateQueueMsgs) { + consumer.setUpTestMsg(); + } + consumers.add(consumer); + return consumer; + }).when(queueFactory).createToRuleEngineMsgConsumer(any()); + + QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queue); + consumerManager = new TbRuleEngineQueueConsumerManager(ruleEngineConsumerContext, queueKey); + } + + @After + public void afterEach() { + consumerManager.stop(); + consumerManager.awaitStop(); + ruleEngineConsumerContext.stop(); + + if (generateQueueMsgs) { + await().atMost(10, TimeUnit.SECONDS) + .untilAsserted(() -> { + log.debug("totalConsumedMsgs = {}, totalProcessedMsgs = {}", totalConsumedMsgs.get(), totalProcessedMsgs.get()); + assertThat(totalProcessedMsgs.get()).isEqualTo(totalConsumedMsgs.get()); + }); + } + } + + @Test + public void testInit_consumerPerPartition() { + queue.setConsumerPerPartition(true); + consumerManager.init(queue); + + Set partitions = createTpis(2, 3, 4); + consumerManager.update(partitions); + partitions = createTpis(3, 4, 5); + consumerManager.update(partitions); + partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + // simulated multiple partition change events before consumer is ready; only latest partitions should be processed + verifyNoInteractions(queueFactory); + + ruleEngineConsumerContext.setReady(true); + await().atMost(2, TimeUnit.SECONDS) + .until(() -> consumers.size() == 3); + for (TopicPartitionInfo partition : partitions) { + TestConsumer consumer = getConsumer(partition); + verifySubscribedAndLaunched(consumer, Set.of(partition)); + } + } + + @Test + public void testInit_singleConsumer() { + queue.setConsumerPerPartition(false); + consumerManager.init(queue); + + Set partitions = createTpis(2, 3, 4); + consumerManager.update(partitions); + partitions = createTpis(3, 4, 5); + consumerManager.update(partitions); + partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + + verifyNoInteractions(queueFactory); + + ruleEngineConsumerContext.setReady(true); + await().atMost(2, TimeUnit.SECONDS) + .until(() -> consumers.size() == 1); + TestConsumer consumer = getConsumer(); + verifySubscribedAndLaunched(consumer, partitions); + } + + @Test + public void testPartitionsUpdate_singleConsumer() { + queue.setConsumerPerPartition(false); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + + Set partitions = Collections.emptySet(); + consumerManager.update(partitions); + verify(queueFactory, after(1000).never()).createToRuleEngineMsgConsumer(any()); + + partitions = createTpis(1); + consumerManager.update(partitions); + TestConsumer consumer = getConsumer(); + verifySubscribedAndLaunched(consumer, partitions); + + partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + verifySubscribedAndLaunched(consumer, partitions); + + partitions = createTpis(4, 5, 6); + consumerManager.update(partitions); + verifySubscribedAndLaunched(consumer, partitions); + + partitions = Collections.emptySet(); + consumerManager.update(partitions); + verifyUnsubscribedAndStopped(consumer); + + partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + consumer = getConsumer(); + verifySubscribedAndLaunched(consumer, partitions); + } + + @Test + public void testPartitionsUpdate_consumerPerPartition() { + queue.setConsumerPerPartition(true); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + + consumerManager.update(Collections.emptySet()); + verify(queueFactory, after(1000).never()).createToRuleEngineMsgConsumer(any()); + + consumerManager.update(createTpis(1)); + TestConsumer consumer1 = getConsumer(1); + verifySubscribedAndLaunched(consumer1, 1); + + consumerManager.update(createTpis(1, 2, 3)); + TestConsumer consumer2 = getConsumer(2); + TestConsumer consumer3 = getConsumer(3); + verifySubscribedAndLaunched(consumer2, 2); + verifySubscribedAndLaunched(consumer3, 3); + verifyNotTouched(consumer1); + + consumerManager.update(createTpis(3, 4, 5)); + TestConsumer consumer4 = getConsumer(4); + TestConsumer consumer5 = getConsumer(5); + verifySubscribedAndLaunched(consumer4, 4); + verifySubscribedAndLaunched(consumer5, 5); + verifyUnsubscribedAndStopped(consumer1); + verifyUnsubscribedAndStopped(consumer2); + verifyNotTouched(consumer3); + + consumerManager.update(Collections.emptySet()); + verifyUnsubscribedAndStopped(consumer3); + verifyUnsubscribedAndStopped(consumer4); + verifyUnsubscribedAndStopped(consumer5); + + consumerManager.update(createTpis(1, 2, 3)); + consumer1 = getConsumer(1); + consumer2 = getConsumer(2); + consumer3 = getConsumer(3); + verifySubscribedAndLaunched(consumer1, 1); + verifySubscribedAndLaunched(consumer2, 2); + verifySubscribedAndLaunched(consumer3, 3); + } + + @Test + public void testConfigUpdate_singleConsumer() { + queue.setConsumerPerPartition(false); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + Set partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + TestConsumer consumer = getConsumer(); + verifySubscribedAndLaunched(consumer, partitions); + + Queue newConfig = JacksonUtil.clone(queue); + newConfig.setPollInterval(queue.getPollInterval() / 2); + newConfig.setPartitions(queue.getPartitions() / 2); + newConfig.setPackProcessingTimeout(queue.getPackProcessingTimeout() * 2); + newConfig.getSubmitStrategy().setType(SubmitStrategyType.SEQUENTIAL_BY_ORIGINATOR); + newConfig.getProcessingStrategy().setType(ProcessingStrategyType.RETRY_ALL); + consumerManager.update(newConfig); + + await().atMost(2, TimeUnit.SECONDS) + .untilAsserted(() -> { + verify(consumer, atLeastOnce()).poll(eq((long) newConfig.getPollInterval())); + verify(ruleEngineConsumerContext.getSubmitStrategyFactory(), atLeastOnce()).newInstance(any(), eq(newConfig.getSubmitStrategy())); + verify(ruleEngineConsumerContext.getProcessingStrategyFactory(), atLeastOnce()).newInstance(any(), eq(newConfig.getProcessingStrategy())); + }); + } + + @Test + public void testConfigUpdate_consumerPerPartition() { + queue.setConsumerPerPartition(true); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + Set partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + TestConsumer consumer1 = getConsumer(1); + TestConsumer consumer2 = getConsumer(2); + TestConsumer consumer3 = getConsumer(3); + verifySubscribedAndLaunched(consumer1, 1); + verifySubscribedAndLaunched(consumer2, 2); + verifySubscribedAndLaunched(consumer3, 3); + + Queue newConfig = JacksonUtil.clone(queue); + newConfig.setPollInterval(queue.getPollInterval() / 2); + newConfig.setPartitions(queue.getPartitions() / 2); + newConfig.setPackProcessingTimeout(queue.getPackProcessingTimeout() * 2); + newConfig.getSubmitStrategy().setType(SubmitStrategyType.SEQUENTIAL_BY_ORIGINATOR); + newConfig.getProcessingStrategy().setType(ProcessingStrategyType.RETRY_ALL); + consumerManager.update(newConfig); + + await().atMost(2, TimeUnit.SECONDS) + .untilAsserted(() -> { + verify(consumer1, atLeastOnce()).poll(eq((long) newConfig.getPollInterval())); + verify(consumer2, atLeastOnce()).poll(eq((long) newConfig.getPollInterval())); + verify(consumer3, atLeastOnce()).poll(eq((long) newConfig.getPollInterval())); + }); + verifyNotTouched(consumer1); + verifyNotTouched(consumer2); + verifyNotTouched(consumer3); + } + + @Test + public void testConfigUpdate_fromSingleToConsumerPerPartition() { + queue.setConsumerPerPartition(false); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + Set partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + TestConsumer consumer = getConsumer(); + verifySubscribedAndLaunched(consumer, partitions); + + Queue newConfig = JacksonUtil.clone(queue); + newConfig.setConsumerPerPartition(true); + consumerManager.update(newConfig); + + verifyUnsubscribedAndStopped(consumer); + verifySubscribedAndLaunched(getConsumer(1), 1); + verifySubscribedAndLaunched(getConsumer(2), 2); + verifySubscribedAndLaunched(getConsumer(3), 3); + } + + @Test + public void testConfigUpdate_fromConsumerPerPartitionToSingle() { + queue.setConsumerPerPartition(true); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + Set partitions = createTpis(1, 2, 3); + consumerManager.update(partitions); + TestConsumer consumer1 = getConsumer(1); + TestConsumer consumer2 = getConsumer(2); + TestConsumer consumer3 = getConsumer(3); + verifySubscribedAndLaunched(consumer1, 1); + verifySubscribedAndLaunched(consumer2, 2); + verifySubscribedAndLaunched(consumer3, 3); + + Queue newConfig = JacksonUtil.clone(queue); + newConfig.setConsumerPerPartition(false); + consumerManager.update(newConfig); + + verifyUnsubscribedAndStopped(consumer1); + verifyUnsubscribedAndStopped(consumer2); + verifyUnsubscribedAndStopped(consumer3); + verifySubscribedAndLaunched(getConsumer(), partitions); + } + + @Test + public void testStop() { + queue.setConsumerPerPartition(true); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + consumerManager.update(createTpis(1)); + TestConsumer consumer = getConsumer(1); + verifySubscribedAndLaunched(consumer, 1); + verify(queueFactory, times(1)).createToRuleEngineMsgConsumer(any()); + + consumerManager.stop(); + consumerManager.update(createTpis(1, 2, 3, 4)); // to check that no new tasks after stop are processed + consumerManager.update(createTpis(5, 6, 7)); + + verifyUnsubscribedAndStopped(consumer); + verifyNoMoreInteractions(queueFactory); + } + + @Test + public void testDelete_consumerPerPartition() { + queue.setConsumerPerPartition(true); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + Set partitions = createTpis(1, 2); + consumerManager.update(partitions); + TestConsumer consumer1 = getConsumer(1); + TestConsumer consumer2 = getConsumer(2); + verifySubscribedAndLaunched(consumer1, 1); + verifySubscribedAndLaunched(consumer2, 2); + verifyMsgProcessed(consumer1.testMsg); + verifyMsgProcessed(consumer2.testMsg); + + consumerManager.delete(); + + await().atMost(2, TimeUnit.SECONDS) + .untilAsserted(() -> { + verify(ruleEngineMsgProducer).send(any(), any(), any()); + }); + clearInvocations(actorContext); + verify(consumer1, never()).unsubscribe(); + verify(consumer2, never()).unsubscribe(); + int msgCount = totalConsumedMsgs.get(); + + await().atLeast(4, TimeUnit.SECONDS) // based on topicDeletionDelayInSec + .atMost(7, TimeUnit.SECONDS) + .untilAsserted(() -> { + partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .forEach(topic -> { + verify(queueAdmin).deleteTopic(eq(topic)); + }); + }); + verify(consumer1).unsubscribe(); + verify(consumer2).unsubscribe(); + + int totalMovedMsgs = totalConsumedMsgs.get() - msgCount; + assertThat(totalMovedMsgs).isNotZero(); + verify(ruleEngineMsgProducer, atLeast(totalMovedMsgs)).send(any(), any(), any()); + verify(actorContext, never()).tell(any()); + generateQueueMsgs = false; + } + + @Test + public void testDelete_singleConsumer() { + queue.setConsumerPerPartition(false); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + Set partitions = createTpis(1, 2); + consumerManager.update(partitions); + TestConsumer consumer = getConsumer(); + verifySubscribedAndLaunched(consumer, partitions); + verifyMsgProcessed(consumer.testMsg); + + consumerManager.delete(); + + await().atMost(2, TimeUnit.SECONDS) + .untilAsserted(() -> { + verify(ruleEngineMsgProducer).send(any(), any(), any()); + }); + clearInvocations(actorContext); + verify(consumer, never()).unsubscribe(); + int msgCount = totalConsumedMsgs.get(); + + await().atLeast(4, TimeUnit.SECONDS) + .atMost(7, TimeUnit.SECONDS) + .untilAsserted(() -> { + partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .forEach(topic -> { + verify(queueAdmin).deleteTopic(eq(topic)); + }); + }); + verify(consumer).unsubscribe(); + + int movedMsgs = totalConsumedMsgs.get() - msgCount; + assertThat(movedMsgs).isNotZero(); + verify(ruleEngineMsgProducer, atLeast(movedMsgs)).send(any(), any(), any()); + verify(actorContext, never()).tell(any()); + generateQueueMsgs = false; + } + + @Test + public void testManyDifferentUpdates() throws Exception { + queue.setConsumerPerPartition(RandomUtils.nextBoolean()); + consumerManager.init(queue); + ruleEngineConsumerContext.setReady(true); + + Supplier queueConfigUpdater = () -> { + Queue oldConfig = consumerManager.getQueue(); + Queue newConfig = JacksonUtil.clone(oldConfig); + newConfig.setConsumerPerPartition(RandomUtils.nextBoolean()); + newConfig.setPollInterval(RandomUtils.nextInt(100, 501)); + newConfig.setPartitions(RandomUtils.nextInt(1, 10)); + newConfig.setPackProcessingTimeout(RandomUtils.nextLong(100, 5001)); + newConfig.getSubmitStrategy().setType(SubmitStrategyType.values()[RandomUtils.nextInt(0, SubmitStrategyType.values().length)]); + newConfig.getProcessingStrategy().setType(ProcessingStrategyType.values()[RandomUtils.nextInt(0, ProcessingStrategyType.values().length)]); + log.info("Generated new config: consumerPerPartition={}, pollInterval={}, processingStrategy={}", + newConfig.isConsumerPerPartition(), newConfig.getPollInterval(), newConfig.getProcessingStrategy().getType()); + return newConfig; + }; + Supplier> partitionsUpdater = () -> { + int partitionsCount = RandomUtils.nextInt(0, 20); + int[] partitions = IntStream.generate(() -> RandomUtils.nextInt(0, 20)) + .distinct().limit(partitionsCount) + .sorted().toArray(); + log.info("Generated new partitions: {}", Arrays.toString(partitions)); + return createTpis(partitions); + }; + + int iterations = 100; + Queue latestConfig = queue; + Set latestPartitions = Collections.emptySet(); + for (int i = 1; i <= iterations; i++) { + boolean updateQueueConfig = RandomUtils.nextBoolean(); + boolean updatePartitions = !updateQueueConfig; + if (updateQueueConfig) { + latestConfig = queueConfigUpdater.get(); + consumerManager.update(latestConfig); + } + if (updatePartitions) { + latestPartitions = partitionsUpdater.get(); + consumerManager.update(latestPartitions); + } + Thread.sleep(RandomUtils.nextLong(0, 200)); + } + if (latestPartitions.isEmpty()) { + do { + latestPartitions = partitionsUpdater.get(); + } while (latestPartitions.isEmpty()); + consumerManager.update(latestPartitions); + } + + Queue expectedConfig = latestConfig; + Set expectedPartitions = latestPartitions; + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + assertThat(consumerManager.getQueue()).isEqualTo(expectedConfig); + assertThat(consumerManager.getPartitions()).isEqualTo(expectedPartitions); + }); + + if (expectedConfig.isConsumerPerPartition()) { + await().atMost(5, TimeUnit.SECONDS).until(() -> { + for (TopicPartitionInfo partition : expectedPartitions) { + if (consumers.stream().noneMatch(consumer -> consumer.subscribed && + consumer.pollingStarted && Set.of(partition).equals(consumer.getPartitions()))) { + return false; + } + } + return consumers.size() == expectedPartitions.size(); + }); + } else { + await().atMost(5, TimeUnit.SECONDS).until(() -> { + return consumers.size() == 1 && consumers.stream() + .anyMatch(consumer -> consumer.subscribed && consumer.pollingStarted && + expectedPartitions.equals(consumer.getPartitions())); + }); + } + Mockito.reset(ruleEngineConsumerContext.getSubmitStrategyFactory()); + Mockito.reset(ruleEngineConsumerContext.getProcessingStrategyFactory()); + consumers.forEach(Mockito::clearInvocations); + + await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> { + for (TestConsumer consumer : consumers) { + verify(consumer, atLeastOnce().description("consumer " + consumer.topics)).poll(expectedConfig.getPollInterval()); + } + verify(ruleEngineConsumerContext.getSubmitStrategyFactory(), atLeastOnce()).newInstance(any(), eq(expectedConfig.getSubmitStrategy())); + verify(ruleEngineConsumerContext.getProcessingStrategyFactory(), atLeastOnce()).newInstance(any(), eq(expectedConfig.getProcessingStrategy())); + }); + } + + private void verifySubscribedAndLaunched(TestConsumer consumer, Set expectedPartitions) { + await().atMost(2, TimeUnit.SECONDS) + .until(() -> consumer.subscribed && consumer.getPartitions().equals(expectedPartitions) && consumer.pollingStarted); + verify(consumer, times(1)).subscribe(any()); + verify(consumer).subscribe(eq(expectedPartitions)); + verify(consumer).doSubscribe(argThat(topics -> topics.containsAll(expectedPartitions.stream() + .map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList())))); + verify(consumer, atLeastOnce()).poll(eq((long) queue.getPollInterval())); + verify(consumer, atLeastOnce()).doPoll(eq((long) queue.getPollInterval())); + verify(consumer, never()).unsubscribe(); + Mockito.reset(consumer); + } + + private void verifySubscribedAndLaunched(TestConsumer consumer, int... expectedPartitions) { + verifySubscribedAndLaunched(consumer, createTpis(expectedPartitions)); + } + + private void verifyUnsubscribedAndStopped(TestConsumer consumer) { + await().atMost(2, TimeUnit.SECONDS) + .until(() -> !consumer.subscribed && !consumer.topics.isEmpty()); + verify(consumer, never()).subscribe(any()); + verify(consumer, never()).doSubscribe(any()); + assertThat(consumers).doesNotContain(consumer); + Mockito.reset(consumer); + } + + private void verifyNotTouched(TestConsumer consumer) { + verify(consumer, never()).subscribe(any()); + verify(consumer, never()).subscribe(); + verify(consumer, never()).doSubscribe(any()); + verify(consumer, never()).unsubscribe(); + verify(consumer, never()).doUnsubscribe(); + } + + private void verifyMsgProcessed(TbMsg tbMsg) { + await().atMost(2, TimeUnit.SECONDS).untilAsserted(() -> { + verify(actorContext, atLeastOnce()).tell(argThat(msg -> { + return ((QueueToRuleEngineMsg) msg).getMsg().getId().equals(tbMsg.getId()); + })); + }); + } + + // for consumer-per-partition + private TestConsumer getConsumer(TopicPartitionInfo tpi) { + return await().atMost(5, TimeUnit.SECONDS) + .until(() -> consumers.stream() + .filter(consumer -> consumer.getPartitions() != null && + consumer.getPartitions().size() == 1 && + consumer.getPartitions().contains(tpi)) + .findFirst().orElse(null), Objects::nonNull); + } + + private TestConsumer getConsumer(int partition) { + return await().atMost(5, TimeUnit.SECONDS) + .until(() -> consumers.stream() + .filter(consumer -> consumer.getPartitions() != null && + consumer.getPartitions().size() == 1 && + consumer.getPartitions().stream() + .anyMatch(tpi -> tpi.getPartition().get().equals(partition))) + .findFirst().orElse(null), Objects::nonNull); + } + + // for single consumer + private TestConsumer getConsumer() { + return await().atMost(5, TimeUnit.SECONDS) + .until(() -> consumers.size() == 1 ? consumers.iterator().next() : null, Objects::nonNull); + } + + private Set createTpis(int... partitions) { + return Arrays.stream(partitions) + .mapToObj(n -> TopicPartitionInfo.builder() + .tenantId(queue.getTenantId()) + .topic(queue.getTopic()) + .partition(n) + .myPartition(true) + .build()) + .collect(Collectors.toSet()); + } + + + class TestConsumer extends AbstractTbQueueConsumerTemplate> { + + @Getter + private List topics; + + private boolean subscribed; + private boolean pollingStarted; + + private TbMsg testMsg; + + public TestConsumer(String topic) { + super(topic); + } + + @SneakyThrows + @Override + protected List doPoll(long durationInMillis) { + log.debug("doPoll({} ms)", durationInMillis); + if (!subscribed) { + throw new IllegalStateException("Cannot poll because not subscribed"); + } + pollingStarted = true; + if (testMsg != null && RandomUtils.nextBoolean()) { + Thread.sleep(100); + return List.of(testMsg); + } + return Collections.emptyList(); + } + + @Override + protected TbProtoQueueMsg decode(TbMsg tbMsg) throws IOException { + log.debug("decode()"); + UUID tenantId = UUID.randomUUID(); + return new TbProtoQueueMsg<>(UUID.randomUUID(), ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(tenantId.getMostSignificantBits()) + .setTenantIdLSB(tenantId.getLeastSignificantBits()) + .addRelationTypes("Success") + .setTbMsg(TbMsg.toByteString(tbMsg)) + .build()); + } + + @Override + protected void doSubscribe(List topicNames) { + log.debug("doSubscribe({})", topicNames); + this.topics = topicNames; + subscribed = true; + } + + @Override + protected void doCommit() { + if (!subscribed) { + throw new IllegalStateException("Cannot commit because not subscribed"); + } + log.debug("doCommit() totalConsumedMsgs = {}", totalConsumedMsgs.incrementAndGet()); + } + + @Override + public void unsubscribe() { + super.unsubscribe(); + consumers.remove(this); + } + + @Override + protected void doUnsubscribe() { + log.debug("doUnsubscribe()"); + if (!subscribed) { + throw new IllegalStateException("Already unsubscribed!"); + } + subscribed = false; + } + + @Override + protected boolean isLongPollingSupported() { + return false; + } + + public Set getPartitions() { + return partitions; + } + + public void setUpTestMsg() { + testMsg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, new DeviceId(UUID.randomUUID()), new TbMsgMetaData(), "{}"); + } + } + +} diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java index 9c41f9d342..21216e164e 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java @@ -28,6 +28,8 @@ public interface TbQueueConsumer { void subscribe(Set partitions); + void stop(); + void unsubscribe(); List poll(long durationInMillis); @@ -36,10 +38,6 @@ public interface TbQueueConsumer { boolean isStopped(); - void onQueueDelete(); - - boolean isQueueDeleted(); - List getFullTopicNames(); } diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 60dfcae1de..8202f1c749 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -20,6 +20,42 @@ package transport; option java_package = "org.thingsboard.server.gen.transport"; option java_outer_classname = "TransportProtos"; +/** + * Common data structures + */ +enum EntityTypeProto { + UNSPECIFIED = 0; + TENANT = 1; + CUSTOMER = 2; + USER = 3; + DASHBOARD = 4; + ASSET = 5; + DEVICE = 6; + ALARM = 7; + // next 3 reserved for PE; + RULE_CHAIN = 11; + RULE_NODE = 12; + // next 2 reserved for PE; + ENTITY_VIEW = 15; + WIDGETS_BUNDLE = 16; + WIDGET_TYPE = 17; + // next 2 reserved for PE; + TENANT_PROFILE = 20; + DEVICE_PROFILE = 21; + ASSET_PROFILE = 22; + API_USAGE_STATE = 23; + TB_RESOURCE = 24; + OTA_PACKAGE = 25; + EDGE = 26; + RPC = 27; + QUEUE = 28; + NOTIFICATION_TARGET = 29; + NOTIFICATION_TEMPLATE = 30; + NOTIFICATION_REQUEST = 31; + NOTIFICATION = 32; + NOTIFICATION_RULE = 33; +} + /** * Service Discovery Data Structures; */ @@ -731,6 +767,25 @@ message FromDeviceRPCResponseProto { int32 error = 4; } +enum ComponentLifecycleEvent { + CREATED = 0; + STARTED = 1; + ACTIVATED = 2; + SUSPENDED = 3; + UPDATED = 4; + STOPPED = 5; + DELETED = 6; +} + +message ComponentLifecycleMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + EntityTypeProto entityType = 3; + int64 entityIdMSB = 4; + int64 entityIdLSB = 5; + ComponentLifecycleEvent event = 6; +} + message EdgeNotificationMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; @@ -980,10 +1035,11 @@ message ToCoreMsg { } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ +/* Please, adjust the TbCoreConsumerStats when modifying the ToCoreNotificationMsg */ message ToCoreNotificationMsg { LocalSubscriptionServiceMsgProto toLocalSubscriptionServiceMsg = 1; FromDeviceRPCResponseProto fromDeviceRpcResponse = 2; - bytes componentLifecycleMsg = 3; + bytes componentLifecycleMsg = 3; //will be removed in 3.6.1 in favour of ComponentLifecycleMsgProto bytes edgeEventUpdateMsg = 4; QueueUpdateMsg queueUpdateMsg = 5; QueueDeleteMsg queueDeleteMsg = 6; @@ -992,6 +1048,7 @@ message ToCoreNotificationMsg { bytes fromEdgeSyncResponseMsg = 9; SubscriptionMgrMsgProto toSubscriptionMgrMsg = 10; NotificationRuleProcessorMsg notificationRuleProcessorMsg = 11; + ComponentLifecycleMsgProto componentLifecycle = 12; } /* Messages that are handled by ThingsBoard RuleEngine Service */ @@ -1004,10 +1061,11 @@ message ToRuleEngineMsg { } message ToRuleEngineNotificationMsg { - bytes componentLifecycleMsg = 1; + bytes componentLifecycleMsg = 1; // will be removed in 3.6.1 in favour of ComponentLifecycleMsgProto FromDeviceRPCResponseProto fromDeviceRpcResponse = 2; QueueUpdateMsg queueUpdateMsg = 3; QueueDeleteMsg queueDeleteMsg = 4; + ComponentLifecycleMsgProto componentLifecycle = 5; } /* Messages that are handled by ThingsBoard Transport Service */ diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index 8ca6585718..ce87edf43a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -26,38 +26,46 @@ import java.util.stream.Collectors; * @author Andrew Shvayka */ public enum EntityType { - TENANT, - CUSTOMER, - USER, - DASHBOARD, - ASSET, - DEVICE, - ALARM, - RULE_CHAIN, - RULE_NODE, - ENTITY_VIEW { + TENANT(1), + CUSTOMER(2), + USER(3), + DASHBOARD(4), + ASSET(5), + DEVICE(6), + ALARM (7), + RULE_CHAIN (11), + RULE_NODE (12), + + ENTITY_VIEW (15) { // backward compatibility for TbOriginatorTypeSwitchNode to return correct rule node connection. @Override - public String getNormalName() { + public String getNormalName () { return "Entity View"; } }, - WIDGETS_BUNDLE, - WIDGET_TYPE, - TENANT_PROFILE, - DEVICE_PROFILE, - ASSET_PROFILE, - API_USAGE_STATE, - TB_RESOURCE, - OTA_PACKAGE, - EDGE, - RPC, - QUEUE, - NOTIFICATION_TARGET, - NOTIFICATION_TEMPLATE, - NOTIFICATION_REQUEST, - NOTIFICATION, - NOTIFICATION_RULE; + WIDGETS_BUNDLE (16), + WIDGET_TYPE (17), + TENANT_PROFILE (20), + DEVICE_PROFILE (21), + ASSET_PROFILE (22), + API_USAGE_STATE (23), + TB_RESOURCE (24), + OTA_PACKAGE (25), + EDGE (26), + RPC (27), + QUEUE (28), + NOTIFICATION_TARGET (29), + NOTIFICATION_TEMPLATE (30), + NOTIFICATION_REQUEST (31), + NOTIFICATION (32), + NOTIFICATION_RULE (33); + + @Getter + private final int protoNumber; // Corresponds to EntityTypeProto + + private EntityType(int protoNumber) { + this.protoNumber = protoNumber; + } public static final List NORMAL_NAMES = EnumSet.allOf(EntityType.class).stream() .map(EntityType::getNormalName).collect(Collectors.toUnmodifiableList()); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index 0cdf3ad1eb..da92eb65d6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -29,6 +29,10 @@ public class EntityIdFactory { return getByTypeAndUuid(EntityType.values()[type], UUID.fromString(uuid)); } + public static EntityId getByTypeAndUuid(int type, UUID uuid) { + return getByTypeAndUuid(EntityType.values()[type], uuid); + } + public static EntityId getByTypeAndUuid(String type, String uuid) { return getByTypeAndUuid(EntityType.valueOf(type), UUID.fromString(uuid)); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java index a7ae9599ea..969baee088 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java @@ -21,5 +21,6 @@ import java.io.Serializable; * @author Andrew Shvayka */ public enum ComponentLifecycleEvent implements Serializable { + // In sync with ComponentLifecycleEvent proto CREATED, STARTED, ACTIVATED, SUSPENDED, UPDATED, STOPPED, DELETED } \ No newline at end of file diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index b25a8cd3ea..f293b5dc24 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -15,8 +15,7 @@ */ package org.thingsboard.server.common.msg.plugin; -import lombok.Getter; -import lombok.ToString; +import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; @@ -31,21 +30,14 @@ import java.util.Optional; /** * @author Andrew Shvayka */ -@ToString +@Data public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { - @Getter + private static final long serialVersionUID = -5303421482781273062L; + private final TenantId tenantId; - @Getter private final EntityId entityId; - @Getter private final ComponentLifecycleEvent event; - public ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event) { - this.tenantId = tenantId; - this.entityId = entityId; - this.event = event; - } - public Optional getRuleChainId() { return entityId.getEntityType() == EntityType.RULE_CHAIN ? Optional.of((RuleChainId) entityId) : Optional.empty(); } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/RuleNodeUpdatedMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/RuleNodeUpdatedMsg.java index 0a5dfa484f..d912aaa29c 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/RuleNodeUpdatedMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/RuleNodeUpdatedMsg.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.msg.plugin; +import lombok.EqualsAndHashCode; import lombok.ToString; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -23,8 +24,11 @@ import org.thingsboard.server.common.msg.MsgType; /** * @author Andrew Shvayka + * This class used only to tell local rule-node actor like 'existing.getSelfActor().tellWithHighPriority(new RuleNodeUpdatedMs( ...' + * Never serialized to/from proto, otherwise you need to change proto mappers in ProtoUtils class */ -@ToString +@ToString(callSuper = true) +@EqualsAndHashCode(callSuper = true) public class RuleNodeUpdatedMsg extends ComponentLifecycleMsg { public RuleNodeUpdatedMsg(TenantId tenantId, EntityId entityId) { diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java index 7122d5ef2a..d2144e8111 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -42,7 +42,7 @@ public class TopicPartitionInfo { this.myPartition = myPartition; String tmp = topic; if (tenantId != null && !tenantId.isNullUid()) { - tmp += "." + tenantId.getId().toString(); + tmp += ".isolated." + tenantId.getId().toString(); } if (partition != null) { tmp += "." + partition; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 2ebe41850d..073371ebb5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -44,7 +44,6 @@ public abstract class AbstractTbQueueConsumerTemplate i protected volatile Set partitions; protected final ReentrantLock consumerLock = new ReentrantLock(); //NonfairSync final Queue> subscribeQueue = new ConcurrentLinkedQueue<>(); - protected volatile boolean queueDeleted = false; @Getter private final String topic; @@ -55,7 +54,7 @@ public abstract class AbstractTbQueueConsumerTemplate i @Override public void subscribe() { - log.info("enqueue topic subscribe {} ", topic); + log.debug("enqueue topic subscribe {} ", topic); if (stopped) { log.error("trying subscribe, but consumer stopped for topic {}", topic); return; @@ -65,7 +64,7 @@ public abstract class AbstractTbQueueConsumerTemplate i @Override public void subscribe(Set partitions) { - log.info("enqueue topics subscribe {} ", partitions); + log.debug("enqueue topics subscribe {} ", partitions); if (stopped) { log.error("trying subscribe, but consumer stopped for topic {}", topic); return; @@ -78,7 +77,8 @@ public abstract class AbstractTbQueueConsumerTemplate i List records; long startNanos = System.nanoTime(); if (stopped) { - return errorAndReturnEmpty(); + log.error("poll invoked but consumer stopped for topic " + topic, new RuntimeException("stacktrace")); + return emptyList(); } if (!subscribed && partitions == null && subscribeQueue.isEmpty()) { return sleepAndReturnEmpty(startNanos, durationInMillis); @@ -96,6 +96,7 @@ public abstract class AbstractTbQueueConsumerTemplate i } if (!subscribed) { List topicNames = getFullTopicNames(); + log.info("Subscribing to topics {}", topicNames); doSubscribe(topicNames); subscribed = true; } @@ -127,11 +128,6 @@ public abstract class AbstractTbQueueConsumerTemplate i return result; } - List errorAndReturnEmpty() { - log.error("poll invoked but consumer stopped for topic" + topic, new RuntimeException("stacktrace")); - return emptyList(); - } - List sleepAndReturnEmpty(final long startNanos, final long durationInMillis) { long durationNanos = TimeUnit.MILLISECONDS.toNanos(durationInMillis); long spentNanos = System.nanoTime() - startNanos; @@ -163,15 +159,20 @@ public abstract class AbstractTbQueueConsumerTemplate i } } + @Override + public void stop() { + stopped = true; + } + @Override public void unsubscribe() { - log.info("Unsubscribing from topics and stopping consumer for topics {}", partitions.stream() - .map(TopicPartitionInfo::getFullTopicName) - .collect(Collectors.joining(", "))); + log.info("Unsubscribing and stopping consumer for topics {}", getFullTopicNames()); stopped = true; consumerLock.lock(); try { - doUnsubscribe(); + if (subscribed) { + doUnsubscribe(); + } } finally { consumerLock.unlock(); } @@ -192,17 +193,11 @@ public abstract class AbstractTbQueueConsumerTemplate i abstract protected void doUnsubscribe(); - @Override - public void onQueueDelete() { - queueDeleted = true; - } - - public boolean isQueueDeleted() { - return queueDeleted; - } - @Override public List getFullTopicNames() { + if (partitions == null) { + return Collections.emptyList(); + } return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 9f58446966..ff3f0cc9b3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -73,7 +73,6 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue protected void doSubscribe(List topicNames) { if (!topicNames.isEmpty()) { topicNames.forEach(admin::createTopicIfNotExists); - log.info("subscribe topics {}", topicNames); consumer.subscribe(topicNames); } else { log.info("unsubscribe due to empty topic list"); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index 8711cbbcf1..a7f8cadd0d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -31,7 +31,6 @@ public class InMemoryTbQueueConsumer implements TbQueueCon private volatile Set partitions; private volatile boolean stopped; private volatile boolean subscribed; - private volatile boolean queueDeleted; public InMemoryTbQueueConsumer(InMemoryStorage storage, String topic) { this.storage = storage; @@ -58,9 +57,15 @@ public class InMemoryTbQueueConsumer implements TbQueueCon subscribed = true; } + @Override + public void stop() { + stopped = true; + } + @Override public void unsubscribe() { stopped = true; + subscribed = false; } @Override @@ -104,16 +109,6 @@ public class InMemoryTbQueueConsumer implements TbQueueCon return stopped; } - @Override - public void onQueueDelete() { - queueDeleted = true; - } - - @Override - public boolean isQueueDeleted() { - return queueDeleted; - } - @Override public List getFullTopicNames() { return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 22a16de64f..55050ce08c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -187,7 +187,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(configuration.getTopic()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId("re-" + queueName + (configuration.getTenantId().isSysTenantId() ? "" : ("-" + configuration.getTenantId())) + "-consumer"); + consumerBuilder.groupId("re-" + queueName + (configuration.getTenantId().isSysTenantId() ? "" : ("-isolated-" + configuration.getTenantId())) + "-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(ruleEngineAdmin); consumerBuilder.statsService(consumerStatsService); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 2e3bf784d7..cb063b6a7f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -166,7 +166,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(configuration.getTopic()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId("re-" + queueName + (configuration.getTenantId().isSysTenantId() ? "" : ("-" + configuration.getTenantId())) + "-consumer"); + consumerBuilder.groupId("re-" + queueName + (configuration.getTenantId().isSysTenantId() ? "" : ("-isolated-" + configuration.getTenantId())) + "-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(ruleEngineAdmin); consumerBuilder.statsService(consumerStatsService); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 386447045d..bb1943a7d5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -82,7 +82,6 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory * @return * @param configuration */ - //TODO 2.5 ybondarenko: make sure you use queueName to distinct consumers where necessary TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration); /** diff --git a/common/util/pom.xml b/common/util/pom.xml index 3d37912ba1..f8dba98bb8 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -36,6 +36,14 @@ + + org.bouncycastle + bcprov-jdk15on + + + org.bouncycastle + bcpkix-jdk15on + org.springframework spring-core diff --git a/common/util/src/main/java/org/thingsboard/common/util/SslUtil.java b/common/util/src/main/java/org/thingsboard/common/util/SslUtil.java new file mode 100644 index 0000000000..889436671a --- /dev/null +++ b/common/util/src/main/java/org/thingsboard/common/util/SslUtil.java @@ -0,0 +1,105 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.common.util; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.openssl.PEMDecryptorProvider; +import org.bouncycastle.openssl.PEMEncryptedKeyPair; +import org.bouncycastle.openssl.PEMKeyPair; +import org.bouncycastle.openssl.PEMParser; +import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter; +import org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder; +import org.bouncycastle.operator.InputDecryptorProvider; +import org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo; +import org.bouncycastle.pkcs.jcajce.JcePKCSPBEInputDecryptorProviderBuilder; +import org.thingsboard.server.common.data.StringUtils; + +import java.io.StringReader; +import java.security.PrivateKey; +import java.security.Security; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.List; + +@Slf4j +public class SslUtil { + + public static final char[] EMPTY_PASS = {}; + + public static final BouncyCastleProvider DEFAULT_PROVIDER = new BouncyCastleProvider(); + + static { + if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { + Security.addProvider(DEFAULT_PROVIDER); + } + } + + private SslUtil() { + } + + @SneakyThrows + public static List readCertFile(String fileContent) { + List certificates = new ArrayList<>(); + JcaX509CertificateConverter certConverter = new JcaX509CertificateConverter(); + try (PEMParser pemParser = new PEMParser(new StringReader(fileContent))) { + Object object; + while ((object = pemParser.readObject()) != null) { + if (object instanceof X509CertificateHolder) { + X509Certificate x509Cert = certConverter.getCertificate((X509CertificateHolder) object); + certificates.add(x509Cert); + } + } + } + return certificates; + } + + @SneakyThrows + public static PrivateKey readPrivateKey(String fileContent, String passStr) { + char[] password = StringUtils.isEmpty(passStr) ? EMPTY_PASS : passStr.toCharArray(); + + PrivateKey privateKey = null; + JcaPEMKeyConverter keyConverter = new JcaPEMKeyConverter(); + if (StringUtils.isNotEmpty(fileContent)) { + try (PEMParser pemParser = new PEMParser(new StringReader(fileContent))) { + Object object; + while ((object = pemParser.readObject()) != null) { + if (object instanceof PEMEncryptedKeyPair) { + PEMDecryptorProvider decProv = new JcePEMDecryptorProviderBuilder().build(password); + privateKey = keyConverter.getKeyPair(((PEMEncryptedKeyPair) object).decryptKeyPair(decProv)).getPrivate(); + break; + } else if (object instanceof PKCS8EncryptedPrivateKeyInfo) { + InputDecryptorProvider decProv = + new JcePKCSPBEInputDecryptorProviderBuilder().setProvider(DEFAULT_PROVIDER).build(password); + privateKey = keyConverter.getPrivateKey(((PKCS8EncryptedPrivateKeyInfo) object).decryptPrivateKeyInfo(decProv)); + break; + } else if (object instanceof PEMKeyPair) { + privateKey = keyConverter.getKeyPair((PEMKeyPair) object).getPrivate(); + break; + } else if (object instanceof PrivateKeyInfo) { + privateKey = keyConverter.getPrivateKey((PrivateKeyInfo) object); + } + } + } + } + return privateKey; + } + +} diff --git a/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardThreadFactory.java b/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardThreadFactory.java index 1f41c48f43..9eaca3ca47 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardThreadFactory.java +++ b/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardThreadFactory.java @@ -22,6 +22,7 @@ import java.util.concurrent.atomic.AtomicInteger; * Copy of Executors.DefaultThreadFactory but with ability to set name of the pool */ public class ThingsBoardThreadFactory implements ThreadFactory { + public static final String THREAD_TOPIC_SEPARATOR = " | "; private static final AtomicInteger poolNumber = new AtomicInteger(1); private final ThreadGroup group; private final AtomicInteger threadNumber = new AtomicInteger(1); @@ -40,6 +41,17 @@ public class ThingsBoardThreadFactory implements ThreadFactory { "-thread-"; } + public static void updateCurrentThreadName(String threadSuffix) { + String name = Thread.currentThread().getName(); + int spliteratorIndex = name.indexOf(THREAD_TOPIC_SEPARATOR); + if (spliteratorIndex > 0) { + name = name.substring(0, spliteratorIndex); + } + name = name + THREAD_TOPIC_SEPARATOR + threadSuffix; + Thread.currentThread().setName(name); + } + + @Override public Thread newThread(Runnable r) { Thread t = new Thread(group, r, diff --git a/dao/src/main/java/org/thingsboard/server/dao/eventsourcing/SaveEntityEvent.java b/dao/src/main/java/org/thingsboard/server/dao/eventsourcing/SaveEntityEvent.java index 205f592d43..cc2e854f59 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/eventsourcing/SaveEntityEvent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/eventsourcing/SaveEntityEvent.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.id.TenantId; public class SaveEntityEvent { private final TenantId tenantId; private final T entity; + private final T oldEntity; private final EntityId entityId; private final Boolean added; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java index f6250e98f0..2ed2364621 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java @@ -36,6 +36,8 @@ public interface EntityAlarmRepository extends JpaRepository findByRequestId(UUID requestId, Pageable pageable); @Transactional - int deleteByIdAndRecipientId(UUID id, UUID recipientId); + @Modifying + @Query("DELETE FROM NotificationEntity n WHERE n.id = :id AND n.recipientId = :recipientId") + int deleteByIdAndRecipientId(@Param("id") UUID id, @Param("recipientId") UUID recipientId); @Transactional - void deleteByRequestId(UUID requestId); + @Modifying + @Query("DELETE FROM NotificationEntity n WHERE n.requestId = :requestId") + void deleteByRequestId(@Param("requestId") UUID requestId); @Transactional - void deleteByRecipientId(UUID recipientId); + @Modifying + @Query("DELETE FROM NotificationEntity n WHERE n.recipientId = :recipientId") + void deleteByRecipientId(@Param("recipientId") UUID recipientId); @Modifying @Transactional diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationRequestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationRequestRepository.java index 126295616f..e7f3ac529b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationRequestRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationRequestRepository.java @@ -70,9 +70,13 @@ public interface NotificationRequestRepository extends JpaRepository findByTenantIdAndIdIn(UUID tenantId, List ids); @Transactional - void deleteByTenantId(UUID tenantId); + @Modifying + @Query("DELETE FROM NotificationTargetEntity t WHERE t.tenantId = :tenantId") + void deleteByTenantId(@Param("tenantId") UUID tenantId); long countByTenantId(UUID tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationTemplateRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationTemplateRepository.java index 73a25b98c3..3e68819b26 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationTemplateRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationTemplateRepository.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.notification; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; @@ -42,7 +43,9 @@ public interface NotificationTemplateRepository extends JpaRepository certHolders = readCertFile(cert); - Object keyObject = readPrivateKeyFile(privateKey); - char[] passwordCharArray = "".toCharArray(); - if (!StringUtils.isEmpty(password)) { - passwordCharArray = password.toCharArray(); - } - - JcaPEMKeyConverter keyConverter = new JcaPEMKeyConverter().setProvider("BC"); - - PrivateKey privateKey; - if (keyObject instanceof PEMEncryptedKeyPair) { - PEMDecryptorProvider provider = new JcePEMDecryptorProviderBuilder().build(passwordCharArray); - KeyPair key = keyConverter.getKeyPair(((PEMEncryptedKeyPair) keyObject).decryptKeyPair(provider)); - privateKey = key.getPrivate(); - } else if (keyObject instanceof PEMKeyPair) { - KeyPair key = keyConverter.getKeyPair((PEMKeyPair) keyObject); - privateKey = key.getPrivate(); - } else if (keyObject instanceof PrivateKey) { - privateKey = (PrivateKey) keyObject; - } else { - throw new RuntimeException("Unable to get private key from object: " + keyObject.getClass()); - } - - KeyStore clientKeyStore = KeyStore.getInstance(KeyStore.getDefaultType()); - clientKeyStore.load(null, null); - for (X509Certificate certHolder : certHolders) { - clientKeyStore.setCertificateEntry("cert-" + certHolder.getSubjectDN().getName(), certHolder); - } - clientKeyStore.setKeyEntry("private-key", - privateKey, - passwordCharArray, - certHolders.toArray(new Certificate[]{})); - KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - keyManagerFactory.init(clientKeyStore, passwordCharArray); - return keyManagerFactory; - } - protected TrustManagerFactory createAndInitTrustManagerFactory() throws Exception { - List caCertHolders = readCertFile(caCert); + List caCerts = SslUtil.readCertFile(caCert); KeyStore caKeyStore = KeyStore.getInstance(KeyStore.getDefaultType()); caKeyStore.load(null, null); - for (X509Certificate caCertHolder : caCertHolders) { - caKeyStore.setCertificateEntry("caCert-cert-" + caCertHolder.getSubjectDN().getName(), caCertHolder); + for (X509Certificate caCert : caCerts) { + caKeyStore.setCertificateEntry(CA_CERT_CERT_ALIAS_PREFIX + caCert.getSubjectDN().getName(), caCert); } TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); @@ -158,170 +85,31 @@ public class CertPemCredentials implements ClientCredentials { return trustManagerFactory; } - List readCertFile(String fileContent) throws Exception { - if (fileContent == null || fileContent.trim().isEmpty()) { - return Collections.emptyList(); - } - - List certificates = new ArrayList<>(); - String[] pems = fileContent.trim().split("-----END CERTIFICATE-----"); - for (String pem : pems) { - if (pem.trim().isEmpty()) { - continue; - } - pem = pem.replace("-----BEGIN CERTIFICATE-----", "") - .replace("-----END CERTIFICATE-----", "") - .replaceAll("\\s", ""); - byte[] decoded = Base64.decodeBase64(pem); - CertificateFactory certFactory = CertificateFactory.getInstance("X.509"); - try (InputStream inStream = new ByteArrayInputStream(decoded)) { - certificates.add((X509Certificate) certFactory.generateCertificate(inStream)); - } - } - return certificates; - } - - private PrivateKey readPrivateKeyFile(String fileContent) throws Exception { - PrivateKey privateKey = null; - if (fileContent != null && !fileContent.isEmpty()) { - KeyFactory keyFactory = KeyFactory.getInstance("RSA"); - KeySpec keySpec = getKeySpec(fileContent); - privateKey = keyFactory.generatePrivate(keySpec); - } - return privateKey; + private KeyManagerFactory createAndInitKeyManagerFactory() throws Exception { + KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + kmf.init(loadKeyStore(), password.toCharArray()); + return kmf; } - private KeySpec getKeySpec(String encodedKey) throws Exception { - KeySpec keySpec = null; - Matcher matcher = OPENSSL_ENCRYPTED_RSA_PRIVATEKEY_PATTERN.matcher(encodedKey); - if (matcher.matches()) { - String encryptionDetails = matcher.group(1).trim(); - String encryptedKey = matcher.group(2).replaceAll("\\s", ""); - byte[] encryptedBinaryKey = java.util.Base64.getDecoder().decode(encryptedKey); - String[] encryptionDetailsParts = encryptionDetails.split(","); - if (encryptionDetailsParts.length == 2) { - String encryptionAlgorithm = encryptionDetailsParts[0]; - String encryptedAlgorithmParams = encryptionDetailsParts[1]; - byte[] pw = password.getBytes(); - byte[] iv = Hex.decode(encryptedAlgorithmParams); - - MessageDigest digest = MessageDigest.getInstance("MD5"); - digest.update(pw); - digest.update(iv, 0, 8); - - byte[] round1Digest = digest.digest(); - digest.update(round1Digest); - digest.update(pw); - digest.update(iv, 0, 8); - - byte[] round2Digest = digest.digest(); - Cipher cipher = null; - SecretKey secretKey = null; - byte[] key = null; + protected KeyStore loadKeyStore() throws Exception { + List certificates = SslUtil.readCertFile(this.cert); + PrivateKey privateKey = SslUtil.readPrivateKey(this.privateKey, password); - switch(encryptionAlgorithm) { - case "AES-256-CBC": - cipher = Cipher.getInstance("AES/CBC/PKCS5Padding"); - key = new byte[32]; - System.arraycopy(round1Digest, 0, key, 0, 16); - System.arraycopy(round2Digest, 0, key, 16, 16); - secretKey = new SecretKeySpec(key, "AES"); - break; - case "AES-192-CBC": - cipher = Cipher.getInstance("AES/CBC/PKCS5Padding"); - key = new byte[24]; - System.arraycopy(round1Digest, 0, key, 0, 16); - System.arraycopy(round2Digest, 0, key, 16, 8); - secretKey = new SecretKeySpec(key, "AES"); - break; - case "AES-128-CBC": - cipher = Cipher.getInstance("AES/CBC/PKCS5Padding"); - key = new byte[16]; - System.arraycopy(round1Digest, 0, key, 0, 16); - secretKey = new SecretKeySpec(key, "AES"); - break; - case "DES-EDE3-CBC": - cipher = Cipher.getInstance("DESede/CBC/PKCS5Padding"); - key = new byte[24]; - System.arraycopy(round1Digest, 0, key, 0, 16); - System.arraycopy(round2Digest, 0, key, 16, 8); - secretKey = new SecretKeySpec(key, "DESede"); - break; - case "DES-CBC": - cipher = Cipher.getInstance("DES/CBC/PKCS5Padding"); - key = new byte[8]; - System.arraycopy(round1Digest, 0, key, 0, 8); - secretKey = new SecretKeySpec(key, "DES"); - break; - } - if (cipher != null) { - cipher.init(Cipher.DECRYPT_MODE, secretKey, new IvParameterSpec(iv)); - byte[] pkcs1 = cipher.doFinal(encryptedBinaryKey); - keySpec = decodeRSAPrivatePKCS1(pkcs1); - } else { - throw new RuntimeException("Unknown Encryption algorithm!"); - } - } else { - throw new RuntimeException("Wrong encryption details!"); - } - } else { - encodedKey = encodedKey.replaceAll(".*BEGIN.*PRIVATE KEY.*", "") - .replaceAll(".*END.*PRIVATE KEY.*", "") - .replaceAll("\\s", ""); - byte[] decoded = Base64.decodeBase64(encodedKey); - if (password == null || password.isEmpty()) { - keySpec = new PKCS8EncodedKeySpec(decoded); - } else { - PBEKeySpec pbeKeySpec = new PBEKeySpec(password.toCharArray()); - - EncryptedPrivateKeyInfo privateKeyInfo = new EncryptedPrivateKeyInfo(decoded); - String algorithmName = privateKeyInfo.getAlgName(); - Cipher cipher = Cipher.getInstance(algorithmName); - SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance(algorithmName); - - Key pbeKey = secretKeyFactory.generateSecret(pbeKeySpec); - AlgorithmParameters algParams = privateKeyInfo.getAlgParameters(); - cipher.init(Cipher.DECRYPT_MODE, pbeKey, algParams); - keySpec = privateKeyInfo.getKeySpec(cipher); - } + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + keyStore.load(null); + List unique = certificates.stream().distinct().collect(Collectors.toList()); + for (X509Certificate cert : unique) { + keyStore.setCertificateEntry(CERT_ALIAS_PREFIX + cert.getSubjectDN().getName(), cert); } - return keySpec; - } - - private static BigInteger derint(ByteBuffer input) { - int len = der(input, 0x02); - byte[] value = new byte[len]; - input.get(value); - return new BigInteger(+1, value); - } - private static int der(ByteBuffer input, int exp) { - int tag = input.get() & 0xFF; - if (tag != exp) throw new IllegalArgumentException("Unexpected tag"); - int n = input.get() & 0xFF; - if (n < 128) return n; - n &= 0x7F; - if ((n < 1) || (n > 2)) throw new IllegalArgumentException("Invalid length"); - int len = 0; - while (n-- > 0) { - len <<= 8; - len |= input.get() & 0xFF; + if (privateKey != null) { + CertificateFactory factory = CertificateFactory.getInstance(X_509); + CertPath certPath = factory.generateCertPath(certificates); + List path = certPath.getCertificates(); + Certificate[] x509Certificates = path.toArray(new Certificate[0]); + keyStore.setKeyEntry(PRIVATE_KEY_ALIAS, privateKey, password.toCharArray(), x509Certificates); } - return len; + return keyStore; } - static RSAPrivateCrtKeySpec decodeRSAPrivatePKCS1(byte[] encoded) { - ByteBuffer input = ByteBuffer.wrap(encoded); - if (der(input, 0x30) != input.remaining()) throw new IllegalArgumentException("Excess data"); - if (!BigInteger.ZERO.equals(derint(input))) throw new IllegalArgumentException("Unsupported version"); - BigInteger n = derint(input); - BigInteger e = derint(input); - BigInteger d = derint(input); - BigInteger p = derint(input); - BigInteger q = derint(input); - BigInteger ep = derint(input); - BigInteger eq = derint(input); - BigInteger c = derint(input); - return new RSAPrivateCrtKeySpec(n, e, d, p, q, ep, eq, c); - } } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java index 904f470001..2cecd1490d 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java @@ -18,21 +18,36 @@ package org.thingsboard.rule.engine.credentials; import org.apache.commons.io.FileUtils; import org.junit.Assert; import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.thingsboard.common.util.SslUtil; import java.io.File; import java.io.IOException; +import java.security.Key; +import java.security.KeyStore; +import java.security.cert.Certificate; import java.security.cert.X509Certificate; import java.util.List; +import java.util.stream.Stream; + +import static org.thingsboard.rule.engine.credentials.CertPemCredentials.CERT_ALIAS_PREFIX; +import static org.thingsboard.rule.engine.credentials.CertPemCredentials.PRIVATE_KEY_ALIAS; public class CertPemCredentialsTest { - private final CertPemCredentials credentials = new CertPemCredentials(); + private static final String PASS = "test"; + private static final String EMPTY_PASS = ""; + private static final String RSA = "RSA"; + private static final String EC = "EC"; @Test public void testChainOfCertificates() throws Exception { String fileContent = fileContent("pem/tb-cloud-chain.pem"); - List x509Certificates = credentials.readCertFile(fileContent); + List x509Certificates = SslUtil.readCertFile(fileContent); Assert.assertEquals(4, x509Certificates.size()); Assert.assertEquals("CN=*.thingsboard.cloud, O=\"ThingsBoard, Inc.\", ST=New York, C=US", @@ -49,7 +64,7 @@ public class CertPemCredentialsTest { public void testSingleCertificate() throws Exception { String fileContent = fileContent("pem/tb-cloud.pem"); - List x509Certificates = credentials.readCertFile(fileContent); + List x509Certificates = SslUtil.readCertFile(fileContent); Assert.assertEquals(1, x509Certificates.size()); Assert.assertEquals("CN=*.thingsboard.cloud, O=\"ThingsBoard, Inc.\", ST=New York, C=US", @@ -60,11 +75,43 @@ public class CertPemCredentialsTest { public void testEmptyFileContent() throws Exception { String fileContent = fileContent("pem/empty.pem"); - List x509Certificates = credentials.readCertFile(fileContent); + List x509Certificates = SslUtil.readCertFile(fileContent); Assert.assertEquals(0, x509Certificates.size()); } + private static Stream testLoadKeyStore() { + return Stream.of( + Arguments.of("pem/rsa_cert.pem", "pem/rsa_key.pem", EMPTY_PASS, RSA), + Arguments.of("pem/rsa_encrypted_cert.pem", "pem/rsa_encrypted_key.pem", PASS, RSA), + Arguments.of("pem/rsa_encrypted_traditional_cert.pem", "pem/rsa_encrypted_traditional_key.pem", PASS, RSA), + Arguments.of("pem/ec_cert.pem", "pem/ec_key.pem", EMPTY_PASS, EC) + ); + } + + @ParameterizedTest + @MethodSource + public void testLoadKeyStore(String certPath, String keyPath, String password, String algorithm) throws Exception { + CertPemCredentials certPemCredentials = new CertPemCredentials(); + String certContent = fileContent(certPath); + certPemCredentials.setCert(certContent); + certPemCredentials.setPrivateKey(fileContent(keyPath)); + certPemCredentials.setPassword(password); + KeyStore keyStore = certPemCredentials.loadKeyStore(); + Assertions.assertNotNull(keyStore); + Key key = keyStore.getKey(PRIVATE_KEY_ALIAS, password.toCharArray()); + Assertions.assertNotNull(key); + Assertions.assertEquals(algorithm, key.getAlgorithm()); + + List certs = SslUtil.readCertFile(certContent); + for (X509Certificate cert : certs) { + String alias = CERT_ALIAS_PREFIX + cert.getIssuerDN().getName(); + Certificate certificate = keyStore.getCertificate(alias); + Assertions.assertNotNull(certificate); + Assertions.assertEquals(new String(cert.getEncoded()), new String(certificate.getEncoded())); + } + } + private String fileContent(String fileName) throws IOException { ClassLoader classLoader = getClass().getClassLoader(); File file = new File(classLoader.getResource(fileName).getFile()); diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/ec_cert.pem b/rule-engine/rule-engine-components/src/test/resources/pem/ec_cert.pem new file mode 100644 index 0000000000..f22f61d3a1 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/ec_cert.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIICCDCCAa2gAwIBAgIUGx/SZqIWza/i/gaKFUVIyTEu2oMwCgYIKoZIzj0EAwIw +WTELMAkGA1UEBhMCVUExDTALBgNVBAgMBEtZSVYxDTALBgNVBAcMBEtZSVYxCzAJ +BgNVBAoMAlRCMQswCQYDVQQLDAJUQjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIz +MTAxNjEyMjMyMVoXDTI0MTAxNTEyMjMyMVowWTELMAkGA1UEBhMCVUExDTALBgNV +BAgMBEtZSVYxDTALBgNVBAcMBEtZSVYxCzAJBgNVBAoMAlRCMQswCQYDVQQLDAJU +QjESMBAGA1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE +z4MgawieJfVc5zUOPiw5WFxfHGJf7dOMsHvudDxdOs27PXPbJfi09BVJ3+JjNxA2 +wQz9KUk877oWRYrN/e+MbKNTMFEwHQYDVR0OBBYEFDTV8VD3m+8IBQOBJ+V/bcbl +4preMB8GA1UdIwQYMBaAFDTV8VD3m+8IBQOBJ+V/bcbl4preMA8GA1UdEwEB/wQF +MAMBAf8wCgYIKoZIzj0EAwIDSQAwRgIhAOgIkl8j8m51W7pWlNUAuUnHnOVhVjGr +h8Rc6cbwTapKAiEA2CLrduTweXEF5fBRtWyOsG8c9af6+MWHKmwHL1IDw9Q= +-----END CERTIFICATE----- diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/ec_key.pem b/rule-engine/rule-engine-components/src/test/resources/pem/ec_key.pem new file mode 100644 index 0000000000..74e0716259 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/ec_key.pem @@ -0,0 +1,8 @@ +-----BEGIN EC PARAMETERS----- +BggqhkjOPQMBBw== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIIEd0mMh0EEy3fMbOpbUY6kW0oAYcaYoTvoVpZxDr5qZoAoGCCqGSM49 +AwEHoUQDQgAEz4MgawieJfVc5zUOPiw5WFxfHGJf7dOMsHvudDxdOs27PXPbJfi0 +9BVJ3+JjNxA2wQz9KUk877oWRYrN/e+MbA== +-----END EC PRIVATE KEY----- \ No newline at end of file diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/rsa_cert.pem b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_cert.pem new file mode 100644 index 0000000000..f89fce7444 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFkzCCA3ugAwIBAgIUUQa3cWUVoF58dzg8ycb/y7SdCj8wDQYJKoZIhvcNAQEL +BQAwWTELMAkGA1UEBhMCVUExDTALBgNVBAgMBEtZSVYxDTALBgNVBAcMBEtZSVYx +CzAJBgNVBAoMAlRCMQswCQYDVQQLDAJUQjESMBAGA1UEAwwJbG9jYWxob3N0MB4X +DTIzMTAxMzEyMzcwMVoXDTI0MTAxMjEyMzcwMVowWTELMAkGA1UEBhMCVUExDTAL +BgNVBAgMBEtZSVYxDTALBgNVBAcMBEtZSVYxCzAJBgNVBAoMAlRCMQswCQYDVQQL +DAJUQjESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8A +MIICCgKCAgEAsHn27cH+pYFI0eJYer8ww29g/xlKgr9aarYlkILeXnBhPPHBCXG+ +FegeMpHa8FUPANIqYJiwM13altO6hMLPa0J7+nQhwF5NCbxzAdi/kU8ofhIwJH+K +gOsD3BKdR7Ua7KMDQFnGTFRR9ZxsuYZ/0AHuzPHwxSLUvvMbiWbu5P2FYMrEyyLo +uVVihZPkeBhcnI6SJRyCdMdMy282nWQ+47gAUI3cFa7dXxUcXvRbbToMNPTIDUy4 +VhxJYhL4T6ED0Ds7tZRsG71LcMfw2RQUgiS1FuYh+O7N8lUMukMy2/umQluM0+qB +CYWa2p1UCbVzlrW1qgKQm1Q8E91XSR9KL/zdO8m9/uNeI1jyJu6i1cibWR7gnh6J +ChLxouQlrBzuLzSz7PG8q1MOWi+oHYJWSvmsckbQDhwEsfhFrYVgndJdxnmlkzvS +1OP7RGSYXLfMF+ZxC2YEJiU65QACCl2IHknyNiL8Jg5ahXgZMNshyfvOv5RB5jnz +4vzRpGhUYCcyLzORT+5gY9ZYbX/51cOomQV1ryTTQs+zA8mfEVLjbbLqvYdI84LC +3chMdcOm8Z9U1xdb2FX/c724XDyPnQNy1PLggzqvOFZzLeey0nBVUWyVrcCydbS5 +PAvVoAucO8kqP6b7uB7QnDeGaCiAVF+9QaXxjyQEdLEu3z5JMM7uH4UCAwEAAaNT +MFEwHQYDVR0OBBYEFHXrT3L+O3kJ2xNZ4Lh1ThGG6M1vMB8GA1UdIwQYMBaAFHXr +T3L+O3kJ2xNZ4Lh1ThGG6M1vMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL +BQADggIBABFDqkTdxsJyu5L2x3WSpw4jw4vgJYlgUvTSeU8i54DaSzncLZdpWsqb +37LFHkvlquIfvOi9f9EBT2KuZwaajPQBNE4m7kLchoAv8Mc8a2EXhN2caXamnN3F +vWAb4QW/VHKKz2vWprfARwqQO58TEPgzU4FcW1lPpX2ULBeoS5kZDDEgyfaFZETF +FnsSb9E3/YuH6sJCu880kbW8BIyQmbUytrbn+16J/iaZBwc+iD49t2VBLDOsr1x4 +5qzxknG3h9wiz9ob9v6hWFfMpdiK12S0P5FVsUkCpxoae8jc8rPS7W3HaYowFjVR +OHOjtWy5/SV2rypKShjg9manf6iwGdTGkD0qoqsRs9JQFabjNR23IQv+1OUbrEVC +DbS65IjwLJlIZBX8JuJaU3I8zqj/9q7TtRDp1NCiG5W0NgipERRCciWaLJ+Fz6Lu +QzhI2ZOJrl49hmr6e0bsyNUv9l89WcbKm3/IC+V7o80uADYCOaz2jDGfKbvcPHzN +mTma8qVsjpcedttsvNMyZOsM/Rpk+dbChgReRVvcmzQV0izEvJJBWFr4HrfcM6Ev +sZrnUiT8ENUZqiK40d+T3Q6JheHwm+ENI1aUDkYCpoWZ/PzKe+Bj8lR8dPvmeVrc +eiwS37nMFO/5X7aIkszTouScNO99cN0UqPldfJo+8ZTbai5VFxGD +-----END CERTIFICATE----- diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_cert.pem b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_cert.pem new file mode 100644 index 0000000000..2f4e55a612 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_cert.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDkzCCAnugAwIBAgIUKAylzm/K5OfbXSjm1zY9bX1a8HQwDQYJKoZIhvcNAQEL +BQAwWTELMAkGA1UEBhMCVUExDTALBgNVBAgMBEtZSVYxDTALBgNVBAcMBEtZSVYx +CzAJBgNVBAoMAlRCMQswCQYDVQQLDAJUQjESMBAGA1UEAwwJbG9jYWxob3N0MB4X +DTIzMTAxNjIxMDQwNVoXDTI0MTAxNTIxMDQwNVowWTELMAkGA1UEBhMCVUExDTAL +BgNVBAgMBEtZSVYxDTALBgNVBAcMBEtZSVYxCzAJBgNVBAoMAlRCMQswCQYDVQQL +DAJUQjESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEA87nLliszEWml8QvyAC+H80NZCxf4TcG826NBOp0AUPJ8xQBHCzc1 +t1ohVm2/fn2VJZAYXG2xSVcHyXjjjv3iGLE2AIDbXh06/yFg4TVjlbrWrAHFehyN +FwrK8ez36oGLa3ZVq+mx1fLfBQw5mStbh09NXmKTzqP6m9ggKtt63cUwoWdUTemT +qrjryJd69LiJi+MVqtbKO2j30/lgAZmaHtbojl9EcvWfeXLb20TnXRIctaIS1VGo +SluzjbNQErdN/VRW4RAOP6UFsK0xID2EuLODBmAWnI49fXO/OS+u3Kd3suABE0o9 +slfDXqNTp0r5N0OoSAFcc4EsV3+9Gf+mqwIDAQABo1MwUTAdBgNVHQ4EFgQUhS5K +XQDxGvaBCpKY1de+JZl8zjYwHwYDVR0jBBgwFoAUhS5KXQDxGvaBCpKY1de+JZl8 +zjYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAxez4vLtBCBNM +l6AQghViNAR9iiwYMxUwKwlU+uZftRGnT+6dXgfTR3PV6LCfMMmtuNs0JTGy0ff8 +erbzfZxExvHfIFXCwepwTWawQhvRRn9GHOJXIzESDRRhsXoJDzd0JVOx0wWxp1cz +EUts+ZbKLoC+kIhsOGY+0a+sopeV2rMO5bUMpA8P0mKZlGynEGMLzKxz65E/IA9h +EQKpJjpvYfN+7eUkF6ZRXNV2LI/8BCoG6mOVoOMEXnloPwwBtOevoCB43U3sT9Er +WQWgZdbeI4gEyEqgMTibNogZZF0KW+5as3iv7avDd8pCgONvD0iwKSlvi9RNjiw8 +p6bwNmBcuA== +-----END CERTIFICATE----- diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_key.pem b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_key.pem new file mode 100644 index 0000000000..045a44ce28 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_key.pem @@ -0,0 +1,30 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIFLTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQILTHGLs8mGUkCAggA +MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBAUnb+mChJ9Wu9F7q6ingLYBIIE +0Mwe8Zl6fs5kiT1AL7gXrSSXmyOVvxDFt0V3TX1w399VIadcUUO0RQeEqXoMEUzl +5at99Xmoo7ByvZSPWCdV4d/j5Yw+Z15euxzclSZJnmBgvQx8cFPLCTTaqlgv5r/Y +lTzBrczbgruMFKtzkvHgvZYiagccOtFHDNC1fUBcUR8dkOOsgTiy4QCVo5kkXHt/ +rblE/uVWI8/E318WZBZaz68HcmGIG6ivdMEsKskSKrH6zA3eLjyGB+zSAIPRB/Mp +s7Rj+RK74zFSYyaq6fgdTG8lug2f3rHImSOtQThcfme5XL4P66rUgJsh/sml1vqH +e848VArGoVy3wfvkss6CyXIJevhFh2xVWRyVqG1nraw2QssEnVqIZvdAnaJJONX8 +r1trjHkZ1JD0nO9Mns1c/bw6hjK6W3UwGfgEMM9VQ7wNI2B6CFOXHKTHg6r3us4k +UqaQtfbpTv+d0YKF/rolDcK+mK/rkxP3rtJA7Ud8nQ4VjxyYX4jTs3/BzDkP7Tsj +5gKy9e1zuTF+MUWs3G5oKGQUKVcbgoYJ+iOqgVSd1JbecRo7Pl8XgDv3I9RWHzUr +EAMjVJjRU9tJuPvILFBkpl1/OPC9sGxJz9Hy9qLtEGhGLUhNz6XmIy/aWPCyA4ea +ZES/n7f+aYmXIxulcxS7MUejkwl1EtNqVyrKvLiRBXjBk2HPCb7Te8fRu/LpHZXN +D7wjymg1fGZPPFzdKh7wMdAKiK50KIMGXTxS6kHb6qW/755oSUjWRLPGcPCfdbjn +UiC5WC9FCog6jfRq1rMlz5b8yjyb+UbJ6N4qFSHeQf+7WLeS0Di8k3cLDSWl6T7M +z82ePof2V2TrADNpXvAcR78uiDfUpfa7DhkimvbBZpRVaaQVU7unxUPVc93WgCWV +a7kBuFJAGt+Wt1LPPD/5KOQ5pRINSoh4VhiZzmnY/m7RDPWEaL5gsMjF5bFoP2UZ +MuyAiTmvO299lJDrdQyQds7yafO9PrTE4msuqpuZSHW7ZZIdRO6EXlfZ1We3icWr ++jE47bUIEl04k1PvXyv9LeoqlHZJTagxZIerMOEwq976MaVR7RJbqUpRUV9FFNCL +gTouPCwUcVtLCaTYQjz/+12/YeVkiBHIWkI8Vv5Mn3Vkwy303ygGCQ+brht1e8x+ +BbgzSpiX8aHiEuDAKooewxnKrf3Dk9BcwbnftxajOZcZ3iphk07t5VLRy86zLKCq +ZOY+KymcDCGaOPnSHFrZK3lZOOT+BB9Vi6EYAkxZCZgoDsb/voMEdpPlxK7ultf6 +is5/JQeQbeP9wbNh4Ru2x3p5Ir4wffhh1KT3UsMobusosTo55ErhMHPvH5amppwq +IrxdM7heo7JMaNKmtol4y45IqSt58iluF5m2Ds4m85xjDteRgEOjtNBStFxPCMAB +KUEzRxEaplAcJfzYzpYtoHZuZ8W3Gi7yeXQ+BV8Q9DeaZc5DDDhIcIkOoHOAKhit +d7Gpr8hpwc60AgHRjua8OdbhM4ntT1xnyDEqZbP8mN+UBAohOHMrqo+f4DL1ibB9 +qSwfdLiVItsEBqlfANV3i9rEeKNH5tOFwFCmmH1yBCSDCtWPmPUJ5tZae6fIetK5 +uSstFXLaDpm6fcHgkeqrnyteWpnk5X5SQQ+fMHPjQ2vp +-----END ENCRYPTED PRIVATE KEY----- diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_traditional_cert.pem b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_traditional_cert.pem new file mode 100644 index 0000000000..38ab2e42fe --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_traditional_cert.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDkzCCAnugAwIBAgIUIo+5l07ZrQR/LxEEmUbnn4yxCwIwDQYJKoZIhvcNAQEL +BQAwWTELMAkGA1UEBhMCVUExDTALBgNVBAgMBEtZSVYxDTALBgNVBAcMBEtZSVYx +CzAJBgNVBAoMAlRCMQswCQYDVQQLDAJUQjESMBAGA1UEAwwJbG9jYWxob3N0MB4X +DTIzMTAxNjIxMDMxNVoXDTI0MTAxNTIxMDMxNVowWTELMAkGA1UEBhMCVUExDTAL +BgNVBAgMBEtZSVYxDTALBgNVBAcMBEtZSVYxCzAJBgNVBAoMAlRCMQswCQYDVQQL +DAJUQjESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEA87nLliszEWml8QvyAC+H80NZCxf4TcG826NBOp0AUPJ8xQBHCzc1 +t1ohVm2/fn2VJZAYXG2xSVcHyXjjjv3iGLE2AIDbXh06/yFg4TVjlbrWrAHFehyN +FwrK8ez36oGLa3ZVq+mx1fLfBQw5mStbh09NXmKTzqP6m9ggKtt63cUwoWdUTemT +qrjryJd69LiJi+MVqtbKO2j30/lgAZmaHtbojl9EcvWfeXLb20TnXRIctaIS1VGo +SluzjbNQErdN/VRW4RAOP6UFsK0xID2EuLODBmAWnI49fXO/OS+u3Kd3suABE0o9 +slfDXqNTp0r5N0OoSAFcc4EsV3+9Gf+mqwIDAQABo1MwUTAdBgNVHQ4EFgQUhS5K +XQDxGvaBCpKY1de+JZl8zjYwHwYDVR0jBBgwFoAUhS5KXQDxGvaBCpKY1de+JZl8 +zjYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAiUTgjnsVIg90 +Dm+XSlscIPbZEj/mJanoFFfAfbVJz1DadygG9viVUMf3jVQBcsJGeBDckR2b3OHY +82cQVpdu3Heqld+gnfsyi8QBi7EdK4i0q8NVqFgpw83KxNm9xt7xrgHtxhE0kWfW +dpTgeIu0hFf0qLUObw/g8+0awBuxNY2crLtLXQM/dRgtv5Zt/DilW3jMLAE5wke+ +/HM4/emOJO6DSI9BC8iUsmNpIpq45267jcjpczNBo3ap7Bad+jM/paRDng9Uavvr +VCsaJFaL5HG6TtNXN60npBouOWnivPzUeuTI4PnjGRgdp3lgb0IuXbuwxIW6FVG/ +73RHc0gGOA== +-----END CERTIFICATE----- diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_traditional_key.pem b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_traditional_key.pem new file mode 100644 index 0000000000..0b0e128a25 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_encrypted_traditional_key.pem @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,FB5DE36A7A8B25DA + +skR15rUvZmdLzGqU5/BF4Yc3E6dxtXTvlOhuGnqH/idItMKUMWIIlQ7ZfWYF/CrC +CkeAUqhF4y+y+2eR4ejUzZKs6bYjTtkXAXAqQvCsTrTBdQSCcLwbHWLWMro0UG24 +e23Rx8kD0YC7VHqyr08NlLh94wR7kanhEeRUbmKBonZT/I9AZ5ntiBxq9QVBtc8M +f7LKIsnQDcd39cVXSo3LOJ/x7YVB//Ln1R1dexwxE0sXOyLq2hhrxzfHGuGXXW41 +/3+CeTgmX9Rzpawrq9vbVabPUgFcJlrogNRSnUAm9kz4b2zadCxEaCejVmhBy4wx +z2AJGcmE+D4VkQK1AAj5+AQQrPOIIFQnyGjwHkJSTGVTcKmttRYZBvUjdfQfj1fK +NsKOSZLzZGknM8Pz5MHgHqk70C7f+nm0uVhVuAiykA4PY4JdCAuTWJxMM2LWM3/q +rYCEMwxCGa6U92dakfC+W+d9pAbeN+xYOWkDrqG7BdAYg1P70cuMRPdd5bsq3YPp +G4n5NVyQvLlocGhZgC7NVzUtc18+rGblV4D657+GZwJnBZJN4TYey4+r2D5fv9rO +kcRVwfR704BbUBkjIzVzD7nXtBbr3ni8HSqde3g4aVL4WyV9XNvjUsYyrZ0u9Mt1 +IccAsa1xBquUNMxwO1H1mFLtzPKmFzKtlzqiiDsRQoRylwUa1k03sHKUflZRa8Sf +g4MpTRzK+vo1opMemlonty5YbvWXKlH68ioo49L8N457Y0hIUJOQgywg80NxT33t +x8y66lawd1Iv+Q7pptVxJtA4JmcdPvGwBLJZY4DacMyp/JqchAQfSQfmQ0tC+RfJ +z2By/s5wOEuVDksgp8RF1gn+VmvLyOoLK7tq+zpMO1mhfYTCMgSiz2GkNdiW6i25 +gjNWN/F62YL+9VJo4+olrcsYDFiiJq+deQk3H1tQJzu6qECfDqKDyw7IunvTwFil +5/d43LvLbRj75Kf/++xwTjfHudeTgw02/yPyELnURkUazvkOFsn7n8tU46Qm5TWh +fPFXSYxRf3m9rkKZB98YOJo595RuZyiYg9dEQX8Gybl1/7H7l4Cvw6yp71kgLrrI +JRYEt9pmWbQX97UFC3WTVMdKWakFziYVGPvFKkIzrHgcutbQVNsZ7GbO+rdWMIxr +SfUe6jCEclzGQjI9Ep4PTLjZvbusUMkoUjGasAluXFXDC4RKtpuXd4RmbOLdVuyN +OnZ5KZHFjrv4ch5PakRTViWFWSddV5CJ4fMkCG9qUHKrUWGjOvzu5rbcUzq3xJZG +9loIvlA4ekEAhQPHwx69uBmUwnCgyB9CosQGmUlwmC3KALA/EiXklTA6w0fGiiPk +uLA9oBGrVcD8Peug9Owfmj4fWbxJGP7x1UR/nZWpynIfzME0AD8MK5uqoWmQTG8I +cLSjVAB1CO//AZe4LpYQulMPq4dipmE+YnKLi0WPXuZVARDciAGGV2BfH6Iv8j9k +o9IoklcpGg22zXLoGn4tu+7Y5GcoV/mx68Gun1E/QFuY59damAqoc82EEsxr+UX0 +4vGX+KZn283xSYiilE3qpCZOER0ZUFInphUwJzzYfW3mW/AWR78tFQIQiuVKV2/a +-----END RSA PRIVATE KEY----- diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/rsa_key.pem b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_key.pem new file mode 100644 index 0000000000..8a92a68908 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/rsa_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCwefbtwf6lgUjR +4lh6vzDDb2D/GUqCv1pqtiWQgt5ecGE88cEJcb4V6B4ykdrwVQ8A0ipgmLAzXdqW +07qEws9rQnv6dCHAXk0JvHMB2L+RTyh+EjAkf4qA6wPcEp1HtRrsowNAWcZMVFH1 +nGy5hn/QAe7M8fDFItS+8xuJZu7k/YVgysTLIui5VWKFk+R4GFycjpIlHIJ0x0zL +bzadZD7juABQjdwVrt1fFRxe9FttOgw09MgNTLhWHEliEvhPoQPQOzu1lGwbvUtw +x/DZFBSCJLUW5iH47s3yVQy6QzLb+6ZCW4zT6oEJhZranVQJtXOWtbWqApCbVDwT +3VdJH0ov/N07yb3+414jWPIm7qLVyJtZHuCeHokKEvGi5CWsHO4vNLPs8byrUw5a +L6gdglZK+axyRtAOHASx+EWthWCd0l3GeaWTO9LU4/tEZJhct8wX5nELZgQmJTrl +AAIKXYgeSfI2IvwmDlqFeBkw2yHJ+86/lEHmOfPi/NGkaFRgJzIvM5FP7mBj1lht +f/nVw6iZBXWvJNNCz7MDyZ8RUuNtsuq9h0jzgsLdyEx1w6bxn1TXF1vYVf9zvbhc +PI+dA3LU8uCDOq84VnMt57LScFVRbJWtwLJ1tLk8C9WgC5w7ySo/pvu4HtCcN4Zo +KIBUX71BpfGPJAR0sS7fPkkwzu4fhQIDAQABAoICABm8z+yA/Hh60Hn7vte4Bo6a +MdVChQFokvE5O2VGENRJI4VV5MdR1V0wiybo6rteTF/cRt3rptb2+yhAHNW767BC +8/3k7f82QZoH5+X/DIFOwCMS1/6as0J2BAwWkuWgXhrg81pxPWBoc8OUWq78FKvr +fD5bkrfNiqWGox946aJv7wHc0LKnlrVg5IuCtDFnrCoRCPNsowIRBvwsbhSqSBnB +/hnBdrWa2SJC2+5lSOg3LQyUHpEB/Whhm7o39gr2+q1l1iF3UgUBqHz8S/381bjd +TaPXUGETwulyyfZoUoSOwQKwg2tsqgEPgTQc+eKomgEC40m2MgzVTiW/hDlf3NvA +aEaUUU1izN/t8tuXS/UBWSsVwPeVm2oPTWTVovoj9PFMSLrJ4oM9iMHl27lKP/Xt +aShnCIu0qwVSLqwCM0HxZNZLEvJIFJe3OV7dvFlbnMiEOlDsz4k2sylFdICOOqxC +Nb61hX8n6iYmAID9hahExOAFcJfpV/MrGnF1IfNDdOim5az1k0PUZlA+50NLjOzK +umfAQpsa7ZUpjfNq6HkX5mhJXelvv+pWuvbBogG10nio13I4J/YwydC/0qaijrhp +XTuV3Or2HhGr5Fe9lpzrnWB90q7iqAgGzevds8AagE0EdUIFupx6vr28Gb3mmlvD +yObUj/9cB+eYIae1jmzJAoIBAQC9pN8I0ltyR5IuuPBcEZcqdrRV04iwZylxEcYy +TIj1YBYvU6LP8AOg671Q4vGXOCo1uq/UsCzZMPn5yFa5fMg2AHu2B51nM/NILvD0 +QCgpyvNV64q+Mci9VWoctWZs93QiQyUKe1c/vUMYlWsYXbz/8osChX+r5doxBIQx +w60aXr9FLKfVfYfBfn1nakdjOVVvAFDPyoV6Dmfn/cAfPH50NIeOGYRpVTPHwHYr +ZCcIRW9MIzmS00GogAH1BM8JjRr0F9F+rRESeJKdLSqJbgLy2LtJnvU8YjlRUVWO +FLzqaUyT2PJS9Vqbohrlk52Znq5Gl+SbGhSEx+oTH3JM/n0jAoIBAQDuOZ8bqtRY +p4BuBOPOaiHRIor+ng48m+nhXec4TuKUlwKLFJHXu+lsEZfs7BWijbRqMH4I5GZl +10EmE4mpkp843kqbFi71s7l9xWnM7jgXWSauH0O4Cleq8/9l2ZOissYzakLhNz3C +9IT0JcnHFmPOPlH4McjoKM3zWDniKI2fRn9q4DAEvRxDuB6PYxbz2NY2OAVI6xSA +bNevnyYA0bwvWeigr6dNCAs1z9QwX2oDGfIEUk3ixdGIqIkpL3WcVPXva2hRPAm4 +1gaI39+q86rEPiZJWfpasUEBY2Ho1eyENqaPTGHCTfBq5fMVntOVhs79TbQ+s70c +1Wyfo6sjHh83AoIBAQCuqdLhhRzEPDbe4WY+5dScP4gIJDOYhOseQIiSevsJQ94q +6JTjfuNYqsZKYTqxVAFMSwz2juw/fWQ+Mc3uOIcNdZR7KrhF/QrsSI+T5iMXmtxT +HgVC9wczmh+JIWmcoqxLghvzc3YANog9dCCW6H7SHMj7IYldAO3ch5RZYSdlSi5P +v7k0X9FQ3PcS8Eefk4akHV5Qgu48ZFg+yu7P1h+BV4Ah2E6j1N1D9Hbhr/RjIdBI +B4lXOUsXrg4fZLZqzZMtjWJdkXhP0sz2BktPGAuPLx4PyF+FpdG0m3x4x5DXNPRa +l01YKrGw9bRgDXzxp7xLOEpMr9CGGrnzstrLHviRAoIBAFoyuwmQvuHqWfhOJasM +CE3VFGeflKhiKEXKdjedtrCoFLBwU2ApqBHg/3MXWIG5wavLPI1FXXgF7obqMt9f +wqWXlQvvdExXhk4Wpx6Ou/IrMTgQYmWWlOcHh5YasYmSwvTIsRXxApOEXarLfADD +e3qlogelYfp1KLWQnCoDTMwXtzrSM5w3tjH1zqxfylr9qO3SfD3FtHeDvo6iZZM9 +1lDfa/MbTu8dspDnZeIC3nLaKgZ020SXveROW9CaRZ+xk4TZWCAZ6VxwvPyqN1fU +9r1jAsAXL3GTV5ec939fMDRHNP1g4Erfk74F3uo6vsYIyuqhtzNefqYiMQSoxa2A +RDUCggEAFlN3ih1gpyvErW4Vy/wUd1ckSH/lojlNjbbyXocKE2eiUnJUwTPerVwX +dI8vqvlPohfDIZqfuBVV+8hiJQGeMiAts6roTQ0pu/w1+euQ4DsOpzUErqadVSOj +h8SpvfxDxrftZSMaN6F7g0Pxlix6qt79XH3Kpfzf9BGOfCG7lslXRAjfuk+HUptK +PijoVHwMwFuZVlN8GBh3uzg+wvME92c4Vr1tHpwqjTqDwZN4RmdnrfGdDb1HJUJW +kv+fD65qKnJz1fZ0RTAcWv4bVFi5GJhZarXD3Vr3C5SH8zNZxDeR29OrSuG7+23g +wOqb/axEbvcj5sV6/4p2zz6AzFPEmQ== +-----END PRIVATE KEY----- diff --git a/ui-ngx/src/app/core/http/entity.service.ts b/ui-ngx/src/app/core/http/entity.service.ts index 69ca114001..41b5360b7a 100644 --- a/ui-ngx/src/app/core/http/entity.service.ts +++ b/ui-ngx/src/app/core/http/entity.service.ts @@ -92,6 +92,7 @@ import { NotificationService } from '@core/http/notification.service'; import { TenantProfileService } from '@core/http/tenant-profile.service'; import { NotificationType } from '@shared/models/notification.models'; import { UserId } from '@shared/models/id/user-id'; +import { AlarmService } from '@core/http/alarm.service'; @Injectable({ providedIn: 'root' @@ -119,7 +120,8 @@ export class EntityService { private assetProfileService: AssetProfileService, private utils: UtilsService, private queueService: QueueService, - private notificationService: NotificationService + private notificationService: NotificationService, + private alarmService: AlarmService ) { } private getEntityObservable(entityType: EntityType, entityId: string, @@ -155,7 +157,7 @@ export class EntityService { observable = this.ruleChainService.getRuleChain(entityId, config); break; case EntityType.ALARM: - console.error('Get Alarm Entity is not implemented!'); + observable = this.alarmService.getAlarm(entityId, config); break; case EntityType.OTA_PACKAGE: observable = this.otaPackageService.getOtaPackageInfo(entityId, config); diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts index 26d6611a3c..3f048ab2e3 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts @@ -56,7 +56,7 @@ export class TenantProfileComponent extends EntityComponent { const mainQueue = [ { id: guid(), - consumerPerPartition: true, + consumerPerPartition: false, name: 'Main', packProcessingTimeout: 10000, partitions: 1, @@ -84,7 +84,7 @@ export class TenantProfileComponent extends EntityComponent { topic: 'tb_rule_engine.hp', pollInterval: 2000, partitions: 1, - consumerPerPartition: true, + consumerPerPartition: false, packProcessingTimeout: 10000, submitStrategy: { type: 'BURST', @@ -108,7 +108,7 @@ export class TenantProfileComponent extends EntityComponent { topic: 'tb_rule_engine.sq', pollInterval: 2000, partitions: 1, - consumerPerPartition: true, + consumerPerPartition: false, packProcessingTimeout: 10000, submitStrategy: { type: 'SEQUENTIAL_BY_ORIGINATOR',