diff --git a/monitoring/pom.xml b/monitoring/pom.xml index d096275089..4d9d3115c9 100644 --- a/monitoring/pom.xml +++ b/monitoring/pom.xml @@ -79,6 +79,10 @@ org.apache.httpcomponents httpclient + + com.slack.api + slack-api-client + org.eclipse.leshan leshan-client-cf @@ -118,6 +122,11 @@ ch.qos.logback logback-classic + + org.springframework.boot + spring-boot-starter-test + test + diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportInfo.java b/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportInfo.java index 251b5e1d85..cc9801dcad 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportInfo.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportInfo.java @@ -16,13 +16,21 @@ package org.thingsboard.monitoring.config.transport; import lombok.Data; +import org.thingsboard.monitoring.data.notification.ShortNameProvider; @Data -public class TransportInfo { +public class TransportInfo implements ShortNameProvider { private final TransportType type; private final TransportMonitoringTarget target; + public String getShortName() { + if (target.getQueue().equals("Main")) { + return type.getName(); + } + return type.getName() + " " + target.getQueue(); + } + @Override public String toString() { if (target.getQueue().equals("Main")) { diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/data/MonitoredServiceKey.java b/monitoring/src/main/java/org/thingsboard/monitoring/data/MonitoredServiceKey.java index 28fa6a9e18..c566e8f896 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/data/MonitoredServiceKey.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/data/MonitoredServiceKey.java @@ -18,6 +18,9 @@ package org.thingsboard.monitoring.data; public class MonitoredServiceKey { public static final String GENERAL = "Monitoring"; + public static final String LOGIN = "Login"; + public static final String WS_CONNECT = "WS Connect"; + public static final String WS_SUBSCRIBE = "WS Subscribe"; public static final String EDQS = "*EDQS*"; } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/AffectedService.java b/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/AffectedService.java new file mode 100644 index 0000000000..b30a398223 --- /dev/null +++ b/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/AffectedService.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.monitoring.data.notification; + +public record AffectedService(String name, Status status, int failureCount) { + + public enum Status { FAILING, RECOVERED, HIGH_LATENCY } + + public static AffectedService failing(String name, int failureCount) { + return new AffectedService(name, Status.FAILING, failureCount); + } + + public static AffectedService recovered(String name) { + return new AffectedService(name, Status.RECOVERED, 0); + } + + public static AffectedService highLatency(String name) { + return new AffectedService(name, Status.HIGH_LATENCY, 0); + } + +} diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/HighLatencyNotification.java b/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/HighLatencyNotification.java index 9a319eb6c9..3e739ec4dc 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/HighLatencyNotification.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/HighLatencyNotification.java @@ -18,6 +18,8 @@ package org.thingsboard.monitoring.data.notification; import org.thingsboard.monitoring.data.Latency; import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; public class HighLatencyNotification implements Notification { @@ -39,4 +41,11 @@ public class HighLatencyNotification implements Notification { return text.toString(); } + @Override + public List getAffectedServices() { + return highLatencies.stream() + .map(latency -> AffectedService.highLatency(latency.getKey())) + .collect(Collectors.toList()); + } + } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/InfoNotification.java b/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/InfoNotification.java index b2b73414de..6906e8c732 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/InfoNotification.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/InfoNotification.java @@ -24,4 +24,9 @@ public class InfoNotification implements Notification { public String getText() { return message; } + + @Override + public boolean isIncident() { + return false; + } } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/Notification.java b/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/Notification.java index 33ed7f8328..7734fc62ee 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/Notification.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/Notification.java @@ -15,8 +15,18 @@ */ package org.thingsboard.monitoring.data.notification; +import java.util.List; + public interface Notification { String getText(); + default boolean isIncident() { + return true; + } + + default List getAffectedServices() { + return List.of(); + } + } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/ServiceFailureNotification.java b/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/ServiceFailureNotification.java index dd93f232a6..69e336f334 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/ServiceFailureNotification.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/ServiceFailureNotification.java @@ -18,6 +18,10 @@ package org.thingsboard.monitoring.data.notification; import lombok.Getter; import org.apache.commons.lang3.exception.ExceptionUtils; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + @Getter public class ServiceFailureNotification implements Notification { @@ -43,7 +47,65 @@ public class ServiceFailureNotification implements Notification { if (errorMsg == null) { errorMsg = error.getClass().getSimpleName(); } + errorMsg = stripResponseBody(errorMsg); + errorMsg = linkifyRequestUrl(errorMsg); return String.format("%s - Failure: %s (number of subsequent failures: %s)", serviceKey, errorMsg, failuresCount); } + // Spring RestClient: '... request for ""' + private static final Pattern REQUEST_FOR_URL_PATTERN = Pattern.compile("request for \"(https?://[^\"\\s]+)\""); + // Apache HttpClient wrapped by Spring: 'I/O error on POST request: Connect to failed: ' + private static final Pattern REQUEST_CONNECT_PATTERN = Pattern.compile("request: Connect to (https?://\\S+?) failed:"); + + static String linkifyRequestUrl(String msg) { + if (msg == null) { + return null; + } + // Slack mrkdwn link: + Matcher m = REQUEST_FOR_URL_PATTERN.matcher(msg); + if (m.find()) { + return m.replaceAll("<$1|request>"); + } + Matcher m2 = REQUEST_CONNECT_PATTERN.matcher(msg); + if (m2.find()) { + return m2.replaceAll("<$1|request>:"); + } + return msg; + } + + static String stripResponseBody(String msg) { + if (msg == null) { + return null; + } + int htmlIdx = -1; + for (String marker : new String[]{"= 0 && (htmlIdx < 0 || idx < htmlIdx)) { + htmlIdx = idx; + } + } + if (htmlIdx > 0) { + msg = msg.substring(0, htmlIdx).stripTrailing(); + if (msg.endsWith("\"")) { + msg = msg.substring(0, msg.length() - 1).stripTrailing(); + } + if (msg.endsWith(":")) { + msg = msg.substring(0, msg.length() - 1).stripTrailing(); + } + } + return msg; + } + + @Override + public List getAffectedServices() { + return List.of(AffectedService.failing(shortName(serviceKey), failuresCount)); + } + + static String shortName(Object serviceKey) { + if (serviceKey instanceof ShortNameProvider provider) { + return provider.getShortName(); + } + return serviceKey.toString(); + } + } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/ServiceRecoveryNotification.java b/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/ServiceRecoveryNotification.java index b6e0c7c695..31b32422a1 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/ServiceRecoveryNotification.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/ServiceRecoveryNotification.java @@ -15,6 +15,8 @@ */ package org.thingsboard.monitoring.data.notification; +import java.util.List; + public class ServiceRecoveryNotification implements Notification { private final Object serviceKey; @@ -28,4 +30,9 @@ public class ServiceRecoveryNotification implements Notification { return String.format("%s is OK", serviceKey); } + @Override + public List getAffectedServices() { + return List.of(AffectedService.recovered(ServiceFailureNotification.shortName(serviceKey))); + } + } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/ShortNameProvider.java b/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/ShortNameProvider.java new file mode 100644 index 0000000000..96c63023af --- /dev/null +++ b/monitoring/src/main/java/org/thingsboard/monitoring/data/notification/ShortNameProvider.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.monitoring.data.notification; + +public interface ShortNameProvider { + + String getShortName(); + +} diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/notification/NotificationService.java b/monitoring/src/main/java/org/thingsboard/monitoring/notification/NotificationService.java index e813b3205d..091a331438 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/notification/NotificationService.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/notification/NotificationService.java @@ -51,7 +51,7 @@ public class NotificationService { return notificationChannels.stream().map(notificationChannel -> notificationExecutor.submit(() -> { try { - notificationChannel.sendNotification(message); + notificationChannel.sendNotification(message, notification); } catch (Exception e) { log.error("Failed to send notification to {}", notificationChannel.getClass().getSimpleName(), e); } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/notification/channels/NotificationChannel.java b/monitoring/src/main/java/org/thingsboard/monitoring/notification/channels/NotificationChannel.java index 126942cb3f..614fddda31 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/notification/channels/NotificationChannel.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/notification/channels/NotificationChannel.java @@ -15,8 +15,10 @@ */ package org.thingsboard.monitoring.notification.channels; +import org.thingsboard.monitoring.data.notification.Notification; + public interface NotificationChannel { - void sendNotification(String message); + void sendNotification(String message, Notification notification); } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/notification/channels/impl/SlackApiClient.java b/monitoring/src/main/java/org/thingsboard/monitoring/notification/channels/impl/SlackApiClient.java new file mode 100644 index 0000000000..4ad74d7811 --- /dev/null +++ b/monitoring/src/main/java/org/thingsboard/monitoring/notification/channels/impl/SlackApiClient.java @@ -0,0 +1,125 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.monitoring.notification.channels.impl; + +import com.slack.api.Slack; +import com.slack.api.SlackConfig; +import com.slack.api.methods.MethodsClient; +import com.slack.api.methods.SlackApiTextResponse; +import com.slack.api.methods.request.chat.ChatPostMessageRequest; +import com.slack.api.methods.request.chat.ChatUpdateRequest; +import com.slack.api.methods.response.chat.ChatPostMessageResponse; +import com.slack.api.methods.response.chat.ChatUpdateResponse; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class SlackApiClient { + + private static final int DEFAULT_CALL_TIMEOUT_MS = 5000; + + private final Slack slack; + private final String botToken; + + public SlackApiClient(String botToken) { + this(botToken, DEFAULT_CALL_TIMEOUT_MS); + } + + public SlackApiClient(String botToken, int callTimeoutMs) { + this.botToken = botToken; + SlackConfig config = new SlackConfig(); + config.setHttpClientCallTimeoutMillis(callTimeoutMs); + config.setHttpClientReadTimeoutMillis(callTimeoutMs); + config.setHttpClientWriteTimeoutMillis(callTimeoutMs); + this.slack = Slack.getInstance(config); + } + + public String postMessage(String channelId, String text) { + ChatPostMessageRequest request = ChatPostMessageRequest.builder() + .channel(channelId) + .text(text) + .build(); + ChatPostMessageResponse response = sendRequest(request); + return response.getTs(); + } + + public String postThreadReply(String channelId, String threadTs, String text) { + ChatPostMessageRequest request = ChatPostMessageRequest.builder() + .channel(channelId) + .text(text) + .threadTs(threadTs) + .build(); + ChatPostMessageResponse response = sendRequest(request); + return response.getTs(); + } + + public void close() { + try { + slack.close(); + } catch (Exception e) { + log.warn("Failed to close Slack client", e); + } + } + + public void updateMessage(String channelId, String ts, String text) { + ChatUpdateRequest request = ChatUpdateRequest.builder() + .channel(channelId) + .ts(ts) + .text(text) + .build(); + MethodsClient client = slack.methods(botToken); + ChatUpdateResponse response; + try { + response = client.chatUpdate(request); + } catch (Exception e) { + throw new RuntimeException("Failed to update Slack message: " + e.getMessage(), e); + } + checkResponse(response); + } + + private ChatPostMessageResponse sendRequest(ChatPostMessageRequest request) { + MethodsClient client = slack.methods(botToken); + ChatPostMessageResponse response; + try { + response = client.chatPostMessage(request); + } catch (Exception e) { + throw new RuntimeException("Failed to send Slack message: " + e.getMessage(), e); + } + checkResponse(response); + return response; + } + + private void checkResponse(SlackApiTextResponse response) { + if (response.isOk()) { + return; + } + String error = response.getError(); + if (error != null) { + switch (error) { + case "missing_scope" -> { + String neededScope = response.getNeeded(); + error = "bot token scope '" + neededScope + "' is needed"; + } + case "not_in_channel" -> error = "app needs to be added to the channel"; + } + } else if (response.getWarning() != null) { + error = "warning: " + response.getWarning(); + } else { + error = "unknown error"; + } + throw new RuntimeException("Slack API error: " + error); + } + +} diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/notification/channels/impl/SlackIncidentTransport.java b/monitoring/src/main/java/org/thingsboard/monitoring/notification/channels/impl/SlackIncidentTransport.java new file mode 100644 index 0000000000..8812714d62 --- /dev/null +++ b/monitoring/src/main/java/org/thingsboard/monitoring/notification/channels/impl/SlackIncidentTransport.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.monitoring.notification.channels.impl; + +import org.thingsboard.monitoring.notification.incident.IncidentTransport; + +public class SlackIncidentTransport implements IncidentTransport { + + private final SlackApiClient slackApiClient; + private final String channelId; + + public SlackIncidentTransport(SlackApiClient slackApiClient, String channelId) { + this.slackApiClient = slackApiClient; + this.channelId = channelId; + } + + @Override + public String postIncident(String text) { + return slackApiClient.postMessage(channelId, text); + } + + @Override + public void postThreadReply(String threadId, String text) { + slackApiClient.postThreadReply(channelId, threadId, text); + } + + @Override + public void updateIncident(String threadId, String text) { + slackApiClient.updateMessage(channelId, threadId, text); + } + +} diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/notification/channels/impl/SlackNotificationChannel.java b/monitoring/src/main/java/org/thingsboard/monitoring/notification/channels/impl/SlackNotificationChannel.java index 5c0df4f6fe..a135a22794 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/notification/channels/impl/SlackNotificationChannel.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/notification/channels/impl/SlackNotificationChannel.java @@ -16,13 +16,16 @@ package org.thingsboard.monitoring.notification.channels.impl; import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.stereotype.Component; import org.springframework.web.client.RestTemplate; +import org.thingsboard.monitoring.data.notification.Notification; import org.thingsboard.monitoring.notification.channels.NotificationChannel; +import org.thingsboard.monitoring.notification.incident.IncidentManager; import java.time.Duration; import java.util.Map; @@ -35,19 +38,73 @@ public class SlackNotificationChannel implements NotificationChannel { @Value("${monitoring.notifications.slack.webhook_url}") private String webhookUrl; + @Value("${monitoring.notifications.slack.bot_token:}") + private String botToken; + + @Value("${monitoring.notifications.slack.channel_id:}") + private String channelId; + + @Value("${monitoring.notifications.incident.enabled:}") + private boolean incidentEnabled; + + @Value("${monitoring.notifications.incident.resolution_timeout_s:}") + private long resolutionTimeoutSeconds; + + @Value("${monitoring.notifications.incident.tag_channel:}") + private boolean tagChannel; + + @Value("${monitoring.notifications.message_prefix:}") + private String messagePrefix; + private RestTemplate restTemplate; + private SlackApiClient slackApiClient; + private IncidentManager incidentManager; @PostConstruct private void init() { - restTemplate = new RestTemplateBuilder() - .connectTimeout(Duration.ofSeconds(5)) - .readTimeout(Duration.ofSeconds(2)) - .build(); + boolean hasBotConfig = botToken != null && !botToken.isEmpty() && channelId != null && !channelId.isEmpty(); + if (hasBotConfig) { + slackApiClient = new SlackApiClient(botToken); + log.info("Slack API mode enabled (channel: {})", channelId); + if (incidentEnabled) { + incidentManager = new IncidentManager(new SlackIncidentTransport(slackApiClient, channelId), + resolutionTimeoutSeconds, messagePrefix, tagChannel); + log.info("Incident grouping enabled via Slack (resolution timeout: {}s)", resolutionTimeoutSeconds); + } + } else { + if (incidentEnabled) { + log.warn("Incident grouping is enabled but Slack bot_token/channel_id are not set; " + + "falling back to plain webhook mode without incident support"); + } + restTemplate = new RestTemplateBuilder() + .connectTimeout(Duration.ofSeconds(5)) + .readTimeout(Duration.ofSeconds(2)) + .build(); + log.info("Slack webhook mode enabled"); + } } @Override - public void sendNotification(String message) { - restTemplate.postForObject(webhookUrl, Map.of("text", message), String.class); + public void sendNotification(String message, Notification notification) { + if (incidentManager != null && notification.isIncident()) { + // Pass the raw notification text: IncidentManager already puts the prefix into the + // incident header, so pre-prefixing the thread reply would double it up. + incidentManager.sendAlert(notification.getText(), notification.getAffectedServices()); + } else if (slackApiClient != null) { + slackApiClient.postMessage(channelId, message); + } else { + restTemplate.postForObject(webhookUrl, Map.of("text", message), String.class); + } + } + + @PreDestroy + private void destroy() { + if (incidentManager != null) { + incidentManager.shutdown(); + } + if (slackApiClient != null) { + slackApiClient.close(); + } } } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/notification/incident/IncidentManager.java b/monitoring/src/main/java/org/thingsboard/monitoring/notification/incident/IncidentManager.java new file mode 100644 index 0000000000..c25abfc081 --- /dev/null +++ b/monitoring/src/main/java/org/thingsboard/monitoring/notification/incident/IncidentManager.java @@ -0,0 +1,292 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.monitoring.notification.incident; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.monitoring.data.notification.AffectedService; + +import java.time.Duration; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +/** + * Thread-safety: all public entry points and scheduled callbacks are {@code synchronized} on the + * manager instance. Transport I/O is performed while the monitor is held. This is safe under the + * assumptions that (a) the transport enforces a short per-call timeout, and (b) notification + * producers are single-threaded; see the Slack client (default 5s) for the Slack-based transport. + */ +@Slf4j +public class IncidentManager { + + private final IncidentTransport transport; + private final long resolutionTimeoutSeconds; + private final String messagePrefix; + private final boolean tagChannel; + private final ScheduledExecutorService scheduler; + + private String activeIncidentThreadId; + private ScheduledFuture resolutionTask; + private ScheduledFuture durationUpdateTask; + private Instant incidentStartTime; + private Instant lastAlertTime; + private final Map failingServices = new LinkedHashMap<>(); + private final Map recoveredServices = new LinkedHashMap<>(); + private final Set highLatencyServices = new LinkedHashSet<>(); + + public IncidentManager(IncidentTransport transport, long resolutionTimeoutSeconds, + String messagePrefix, boolean tagChannel) { + this.transport = transport; + this.resolutionTimeoutSeconds = resolutionTimeoutSeconds; + this.messagePrefix = messagePrefix; + this.tagChannel = tagChannel; + this.scheduler = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "incident-manager"); + t.setDaemon(true); + return t; + }); + } + + public synchronized void sendAlert(String message, List affectedServices) { + try { + if (activeIncidentThreadId == null) { + if (affectedServices.stream().allMatch(s -> s.status() == AffectedService.Status.RECOVERED)) { + return; + } + incidentStartTime = Instant.now(); + failingServices.clear(); + recoveredServices.clear(); + highLatencyServices.clear(); + applyAffectedServices(affectedServices); + activeIncidentThreadId = transport.postIncident(buildOngoingMessageText()); + startDurationUpdater(); + log.info("New incident created, thread id: {}", activeIncidentThreadId); + } else if (applyAffectedServices(affectedServices)) { + safeUpdateHeader(); + } + + try { + transport.postThreadReply(activeIncidentThreadId, message); + log.debug("Alert added to incident thread {}", activeIncidentThreadId); + } catch (Exception e) { + log.error("Failed to post alert to incident thread {}", activeIncidentThreadId, e); + } + } finally { + if (activeIncidentThreadId != null) { + lastAlertTime = Instant.now(); + // High latency is a warning only — it has no explicit recovery signal + // (HighLatencyNotification fires only when something is above threshold), + // so resolution hinges on failing services alone. + if (failingServices.isEmpty()) { + resetResolutionTimer(); + } else { + cancelResolutionTimer(); + } + } + } + } + + private boolean applyAffectedServices(List affectedServices) { + boolean changed = false; + Set latencySnapshot = null; + for (AffectedService service : affectedServices) { + String name = service.name(); + switch (service.status()) { + case FAILING -> { + Integer prev = failingServices.put(name, service.failureCount()); + if (prev == null || prev.intValue() != service.failureCount()) { + changed = true; + } + if (recoveredServices.remove(name) != null) { + changed = true; + } + } + case RECOVERED -> { + Integer lastFailureCount = failingServices.remove(name); + if (lastFailureCount != null) { + recoveredServices.put(name, lastFailureCount); + changed = true; + } + } + case HIGH_LATENCY -> { + if (latencySnapshot == null) { + latencySnapshot = new LinkedHashSet<>(); + } + latencySnapshot.add(name); + } + } + } + // HighLatencyNotification carries the full current set of high latencies, so treat it as a + // snapshot: replace highLatencyServices entirely. Without this, a brief spike would stay + // yellow in the header until the incident resolves. + if (latencySnapshot != null && !latencySnapshot.equals(highLatencyServices)) { + highLatencyServices.clear(); + highLatencyServices.addAll(latencySnapshot); + changed = true; + } + return changed; + } + + private String buildOngoingMessageText() { + StringBuilder sb = new StringBuilder(); + if (tagChannel) { + sb.append(" "); + } + if (messagePrefix != null && !messagePrefix.isEmpty()) { + sb.append("*").append(messagePrefix).append("*"); + } + sb.append(" :rotating_light:"); + Duration elapsed = Duration.between(incidentStartTime, Instant.now()); + if (elapsed.toMinutes() >= 1) { + sb.append(" (").append(formatDuration(elapsed)).append(")"); + } + if (hasAffected()) { + sb.append(" | ").append(formatAffectedServices()); + } + return sb.toString(); + } + + private boolean hasAffected() { + return !failingServices.isEmpty() || !recoveredServices.isEmpty() || !highLatencyServices.isEmpty(); + } + + private void safeUpdateHeader() { + try { + transport.updateIncident(activeIncidentThreadId, buildOngoingMessageText()); + } catch (Exception e) { + log.error("Failed to update incident message", e); + } + } + + private void resetResolutionTimer() { + cancelResolutionTimer(); + resolutionTask = scheduler.schedule(this::resolveIncident, resolutionTimeoutSeconds, TimeUnit.SECONDS); + } + + private void cancelResolutionTimer() { + if (resolutionTask != null) { + resolutionTask.cancel(false); + resolutionTask = null; + } + } + + private void startDurationUpdater() { + if (durationUpdateTask != null) { + durationUpdateTask.cancel(false); + } + durationUpdateTask = scheduler.scheduleAtFixedRate(this::updateDuration, 60, 60, TimeUnit.SECONDS); + } + + private synchronized void updateDuration() { + if (activeIncidentThreadId == null) { + return; + } + safeUpdateHeader(); + } + + private void stopDurationUpdater() { + if (durationUpdateTask != null) { + durationUpdateTask.cancel(false); + durationUpdateTask = null; + } + } + + static String formatDuration(Duration duration) { + long totalMinutes = duration.toMinutes(); + if (totalMinutes < 60) { + return totalMinutes + "m"; + } + long hours = totalMinutes / 60; + long minutes = totalMinutes % 60; + return minutes > 0 ? hours + "h" + minutes + "m" : hours + "h"; + } + + synchronized void resolveIncident() { + if (activeIncidentThreadId == null) { + return; + } + String threadId = activeIncidentThreadId; + stopDurationUpdater(); + String resolutionMessage = buildResolutionMessage(); + activeIncidentThreadId = null; + resolutionTask = null; + failingServices.clear(); + recoveredServices.clear(); + highLatencyServices.clear(); + try { + transport.updateIncident(threadId, resolutionMessage); + log.info("Incident resolved (thread was {})", threadId); + } catch (Exception e) { + log.error("Failed to send incident resolution message", e); + } + } + + private String buildResolutionMessage() { + Duration totalDuration = lastAlertTime != null + ? Duration.between(incidentStartTime, lastAlertTime) + : Duration.between(incidentStartTime, Instant.now()); + StringBuilder sb = new StringBuilder(); + if (messagePrefix != null && !messagePrefix.isEmpty()) { + sb.append("*").append(messagePrefix).append("*"); + } + sb.append(" :white_check_mark:"); + sb.append(" (").append(formatDuration(totalDuration)).append(")"); + if (hasAffected()) { + sb.append(" | ").append(formatAffectedServices()).append("\n"); + } + return sb.toString(); + } + + private String formatAffectedServices() { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (Map.Entry entry : failingServices.entrySet()) { + if (!first) sb.append(", "); + sb.append(":red_circle: ").append(entry.getKey()).append(" (").append(entry.getValue()).append(")"); + first = false; + } + for (String name : highLatencyServices) { + if (!first) sb.append(", "); + sb.append(":large_yellow_circle: ").append(name); + first = false; + } + for (Map.Entry entry : recoveredServices.entrySet()) { + if (!first) sb.append(", "); + sb.append(":large_green_circle: ").append(entry.getKey()).append(" (").append(entry.getValue()).append(")"); + first = false; + } + return sb.toString(); + } + + public void shutdown() { + scheduler.shutdownNow(); + try { + if (!scheduler.awaitTermination(5, TimeUnit.SECONDS)) { + log.warn("Incident scheduler did not terminate in time"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + +} diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/notification/incident/IncidentTransport.java b/monitoring/src/main/java/org/thingsboard/monitoring/notification/incident/IncidentTransport.java new file mode 100644 index 0000000000..0d6b4cac2a --- /dev/null +++ b/monitoring/src/main/java/org/thingsboard/monitoring/notification/incident/IncidentTransport.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.monitoring.notification.incident; + +public interface IncidentTransport { + + String postIncident(String text); + + void postThreadReply(String threadId, String text); + + void updateIncident(String threadId, String text); + +} diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java index b73194469f..c0e85da1ac 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseHealthChecker.java @@ -89,11 +89,10 @@ public abstract class BaseHealthChecker { diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseMonitoringService.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseMonitoringService.java index ea154e49c0..9c8c8fc533 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseMonitoringService.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/BaseMonitoringService.java @@ -117,28 +117,60 @@ public abstract class BaseMonitoringService, T ext } try { log.info("Starting {}", getName()); - stopWatch.start(); - String accessToken = tbClient.logIn(); - reporter.reportLatency(Latencies.LOG_IN, stopWatch.getTime()); - try (WsClient wsClient = wsClientFactory.createClient(accessToken)) { + String accessToken; + try { stopWatch.start(); - wsClient.subscribeForTelemetry(devices, getTestTelemetryKeys()).waitForReply(); - reporter.reportLatency(Latencies.WS_SUBSCRIBE, stopWatch.getTime()); + accessToken = tbClient.logIn(); + reporter.reportLatency(Latencies.LOG_IN, stopWatch.getTime()); + reporter.serviceIsOk(MonitoredServiceKey.LOGIN); + } catch (Exception e) { + reporter.serviceFailure(MonitoredServiceKey.LOGIN, e); + return; + } + + WsClient wsClient; + try { + wsClient = wsClientFactory.createClient(accessToken); + reporter.serviceIsOk(MonitoredServiceKey.WS_CONNECT); + } catch (Exception e) { + reporter.serviceFailure(MonitoredServiceKey.WS_CONNECT, e); + return; + } + + try (WsClient ws = wsClient) { + try { + stopWatch.start(); + ws.subscribeForTelemetry(devices, getTestTelemetryKeys()).waitForReply(); + reporter.reportLatency(Latencies.WS_SUBSCRIBE, stopWatch.getTime()); + reporter.serviceIsOk(MonitoredServiceKey.WS_SUBSCRIBE); + } catch (Exception e) { + reporter.serviceFailure(MonitoredServiceKey.WS_SUBSCRIBE, e); + return; + } for (BaseHealthChecker healthChecker : healthCheckers) { - check(healthChecker, wsClient); + check(healthChecker, ws); } } if (checkEdqs) { - stopWatch.start(); - checkEdqs(); - reporter.reportLatency(Latencies.EDQS_QUERY, stopWatch.getTime()); - reporter.serviceIsOk(MonitoredServiceKey.EDQS); + try { + stopWatch.start(); + checkEdqs(); + reporter.reportLatency(Latencies.EDQS_QUERY, stopWatch.getTime()); + reporter.serviceIsOk(MonitoredServiceKey.EDQS); + } catch (ServiceFailureException e) { + reporter.serviceFailure(e.getServiceKey(), e); + return; + } catch (Exception e) { + reporter.serviceFailure(MonitoredServiceKey.EDQS, e); + return; + } } reporter.reportLatencies(); + reporter.serviceIsOk(MonitoredServiceKey.GENERAL); log.debug("Finished {}", getName()); } catch (ServiceFailureException e) { reporter.serviceFailure(e.getServiceKey(), e); diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringReporter.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringReporter.java index b13394dbdc..75a8a819c0 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringReporter.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/MonitoringReporter.java @@ -25,7 +25,6 @@ import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.monitoring.client.TbClient; import org.thingsboard.monitoring.data.Latency; -import org.thingsboard.monitoring.data.MonitoredServiceKey; import org.thingsboard.monitoring.data.notification.HighLatencyNotification; import org.thingsboard.monitoring.data.notification.ServiceFailureNotification; import org.thingsboard.monitoring.data.notification.ServiceRecoveryNotification; @@ -119,9 +118,7 @@ public class MonitoringReporter { public void serviceIsOk(Object serviceKey) { ServiceRecoveryNotification notification = new ServiceRecoveryNotification(serviceKey); - if (!serviceKey.equals(MonitoredServiceKey.GENERAL)) { - log.info(notification.getText()); - } + log.info(notification.getText()); AtomicInteger failuresCounter = failuresCounters.get(serviceKey); if (failuresCounter != null) { if (failuresCounter.get() >= failuresThreshold) { diff --git a/monitoring/src/main/resources/tb-monitoring.yml b/monitoring/src/main/resources/tb-monitoring.yml index 053e42bb9d..768fd2e4c7 100644 --- a/monitoring/src/main/resources/tb-monitoring.yml +++ b/monitoring/src/main/resources/tb-monitoring.yml @@ -121,11 +121,24 @@ monitoring: notifications: message_prefix: '${NOTIFICATION_MESSAGE_PREFIX:}' + # Incident grouping (threads alerts into incidents, auto-resolves after timeout). + # Requires a channel that supports it — currently only Slack API (bot_token + channel_id). + incident: + # Enable incident grouping + enabled: '${INCIDENT_ENABLED:false}' + # Incident resolution timeout in seconds + resolution_timeout_s: '${INCIDENT_RESOLUTION_TIMEOUT_S:90}' + # Tag @channel in incident messages + tag_channel: '${INCIDENT_TAG_CHANNEL:false}' slack: # Enable notifying via Slack enabled: '${SLACK_NOTIFICATION_CHANNEL_ENABLED:false}' # Slack webhook url webhook_url: '${SLACK_WEBHOOK_URL:}' + # Slack Bot OAuth token (xoxb-...) for API-based messaging with incident support, requires chat:write:bot scope + bot_token: '${SLACK_BOT_TOKEN:}' + # Slack channel ID (e.g. C01234ABCDE) - required when incident feature is enabled + channel_id: '${SLACK_CHANNEL_ID:}' latency: # Enable latencies reporting diff --git a/monitoring/src/test/java/org/thingsboard/monitoring/data/notification/ServiceFailureNotificationTest.java b/monitoring/src/test/java/org/thingsboard/monitoring/data/notification/ServiceFailureNotificationTest.java new file mode 100644 index 0000000000..500a713f7b --- /dev/null +++ b/monitoring/src/test/java/org/thingsboard/monitoring/data/notification/ServiceFailureNotificationTest.java @@ -0,0 +1,97 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.monitoring.data.notification; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class ServiceFailureNotificationTest { + + @Test + void stripResponseBodyRemovesNginxErrorHtml() { + String msg = "503 Service Temporarily Unavailable on POST request for \"https://domain/api/auth/login\": \"" + + "503 Service Temporarily Unavailable" + + "

503 Service Temporarily Unavailable

nginx
\""; + + String sanitized = ServiceFailureNotification.stripResponseBody(msg); + + assertThat(sanitized) + .isEqualTo("503 Service Temporarily Unavailable on POST request for \"https://domain/api/auth/login\""); + } + + @Test + void stripResponseBodyRemovesDoctypeHtml() { + String msg = "500 Internal Server Error: \"...\""; + + String sanitized = ServiceFailureNotification.stripResponseBody(msg); + + assertThat(sanitized).isEqualTo("500 Internal Server Error"); + } + + @Test + void stripResponseBodyLeavesPlainMessagesUntouched() { + String msg = "Connection refused"; + assertThat(ServiceFailureNotification.stripResponseBody(msg)).isEqualTo(msg); + } + + @Test + void stripResponseBodyHandlesNull() { + assertThat(ServiceFailureNotification.stripResponseBody(null)).isNull(); + } + + @Test + void linkifyReplacesRequestForUrlWithSlackMrkdwnLink() { + String msg = "503 Service Temporarily Unavailable on POST request for \"https://example.com/api/auth/login\""; + + assertThat(ServiceFailureNotification.linkifyRequestUrl(msg)) + .isEqualTo("503 Service Temporarily Unavailable on POST "); + } + + @Test + void linkifyReplacesRequestConnectToUrlFailed() { + String msg = "I/O error on POST request: Connect to https://example.com:443 failed: Connect timed out"; + + assertThat(ServiceFailureNotification.linkifyRequestUrl(msg)) + .isEqualTo("I/O error on POST : Connect timed out"); + } + + @Test + void linkifyLeavesMessagesWithoutRequestUrlUntouched() { + String msg = "Connection refused"; + assertThat(ServiceFailureNotification.linkifyRequestUrl(msg)).isEqualTo(msg); + } + + @Test + void linkifyHandlesNull() { + assertThat(ServiceFailureNotification.linkifyRequestUrl(null)).isNull(); + } + + @Test + void shortNameUsesShortNameProviderWhenAvailable() { + ShortNameProvider provider = () -> "MQTT"; + assertThat(ServiceFailureNotification.shortName(provider)).isEqualTo("MQTT"); + } + + @Test + void shortNameFallsBackToToStringForOtherKeys() { + Object key = new Object() { + @Override public String toString() { return "LOGIN"; } + }; + assertThat(ServiceFailureNotification.shortName(key)).isEqualTo("LOGIN"); + } + +} diff --git a/monitoring/src/test/java/org/thingsboard/monitoring/notification/incident/IncidentManagerTest.java b/monitoring/src/test/java/org/thingsboard/monitoring/notification/incident/IncidentManagerTest.java new file mode 100644 index 0000000000..b7051db534 --- /dev/null +++ b/monitoring/src/test/java/org/thingsboard/monitoring/notification/incident/IncidentManagerTest.java @@ -0,0 +1,183 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.monitoring.notification.incident; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.thingsboard.monitoring.data.notification.AffectedService; + +import java.time.Duration; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; + +class IncidentManagerTest { + + private RecordingTransport transport; + private IncidentManager manager; + + @BeforeEach + void setUp() { + transport = new RecordingTransport(); + manager = new IncidentManager(transport, 3600L, "tbqa", false); + } + + @AfterEach + void tearDown() { + manager.shutdown(); + } + + @Test + void formatDurationRendersMinutesAndHours() { + assertThat(IncidentManager.formatDuration(Duration.ofSeconds(30))).isEqualTo("0m"); + assertThat(IncidentManager.formatDuration(Duration.ofMinutes(5))).isEqualTo("5m"); + assertThat(IncidentManager.formatDuration(Duration.ofMinutes(59))).isEqualTo("59m"); + assertThat(IncidentManager.formatDuration(Duration.ofMinutes(60))).isEqualTo("1h"); + assertThat(IncidentManager.formatDuration(Duration.ofMinutes(75))).isEqualTo("1h15m"); + assertThat(IncidentManager.formatDuration(Duration.ofMinutes(120))).isEqualTo("2h"); + } + + @Test + void firstFailureOpensIncidentAndPostsHeaderAndReply() { + manager.sendAlert("CoAP failure message", + List.of(AffectedService.failing("CoAP", 1))); + + assertThat(transport.incidents).hasSize(1); + assertThat(transport.incidents.get(0)).contains(":rotating_light:").contains(":red_circle: CoAP (1)"); + assertThat(transport.replies).hasSize(1); + assertThat(transport.replies.get(0).text()).isEqualTo("CoAP failure message"); + } + + @Test + void isolatedRecoveryWithoutActiveIncidentIsIgnored() { + manager.sendAlert("Login is OK", + List.of(AffectedService.recovered("Login"))); + + assertThat(transport.incidents).isEmpty(); + assertThat(transport.replies).isEmpty(); + assertThat(transport.updates).isEmpty(); + } + + @Test + void subsequentFailureUpdatesHeaderAndPostsReply() { + manager.sendAlert("CoAP failure", List.of(AffectedService.failing("CoAP", 1))); + manager.sendAlert("CoAP repeat", List.of(AffectedService.failing("CoAP", 3))); + + assertThat(transport.incidents).hasSize(1); + assertThat(transport.replies).hasSize(2); + assertThat(transport.updates).hasSize(1); + assertThat(transport.updates.get(0).text()).contains(":red_circle: CoAP (3)"); + } + + @Test + void recoveryAfterFailureMovesServiceToGreenAndKeepsFailureCount() { + manager.sendAlert("CoAP failure", List.of(AffectedService.failing("CoAP", 4))); + manager.sendAlert("CoAP is OK", List.of(AffectedService.recovered("CoAP"))); + + assertThat(transport.updates).hasSize(1); + String updated = transport.updates.get(0).text(); + assertThat(updated).contains(":large_green_circle: CoAP (4)").doesNotContain(":red_circle:"); + } + + @Test + void highLatencyIsTrackedAsYellow() { + manager.sendAlert("high latency", + List.of(AffectedService.highLatency("logInLatency"))); + + assertThat(transport.incidents.get(0)).contains(":large_yellow_circle: logInLatency"); + } + + @Test + void repeatingSameFailureCountDoesNotTriggerRedundantUpdate() { + manager.sendAlert("CoAP failure", List.of(AffectedService.failing("CoAP", 3))); + manager.sendAlert("CoAP still failing", List.of(AffectedService.failing("CoAP", 3))); + + assertThat(transport.updates).isEmpty(); + assertThat(transport.replies).hasSize(2); + } + + @Test + void fullLifecycleStartFailRecoverResolve() { + manager.sendAlert("Login failure", List.of(AffectedService.failing("Login", 1))); + manager.sendAlert("WS failure", List.of(AffectedService.failing("WS Connect", 1))); + manager.sendAlert("Login is OK", List.of(AffectedService.recovered("Login"))); + + assertThat(transport.incidents).hasSize(1); + + manager.resolveIncident(); + + assertThat(transport.updates).last() + .extracting(RecordingTransport.Message::text) + .asString() + .contains(":white_check_mark:") + .contains(":red_circle: WS Connect") + .contains(":large_green_circle: Login (1)"); + } + + @Test + void resolveWithoutActiveIncidentIsNoOp() { + manager.resolveIncident(); + assertThat(transport.updates).isEmpty(); + } + + @Test + void doesNotAutoResolveWhileServicesAreStillFailing() throws Exception { + manager.shutdown(); + transport = new RecordingTransport(); + manager = new IncidentManager(transport, 1L, "tbqa", false); + + manager.sendAlert("CoAP failure", List.of(AffectedService.failing("CoAP", 1))); + Thread.sleep(1500); + + assertThat(transport.updates) + .extracting(RecordingTransport.Message::text) + .noneMatch(t -> t.contains(":white_check_mark:")); + + manager.sendAlert("CoAP is OK", List.of(AffectedService.recovered("CoAP"))); + Thread.sleep(1500); + + assertThat(transport.updates) + .extracting(RecordingTransport.Message::text) + .anyMatch(t -> t.contains(":white_check_mark:")); + } + + private static class RecordingTransport implements IncidentTransport { + private final AtomicInteger threadCounter = new AtomicInteger(); + final java.util.List incidents = new java.util.ArrayList<>(); + final java.util.List replies = new java.util.ArrayList<>(); + final java.util.List updates = new java.util.ArrayList<>(); + + @Override + public String postIncident(String text) { + incidents.add(text); + return "thread-" + threadCounter.incrementAndGet(); + } + + @Override + public void postThreadReply(String threadId, String text) { + replies.add(new Message(threadId, text)); + } + + @Override + public void updateIncident(String threadId, String text) { + updates.add(new Message(threadId, text)); + } + + record Message(String threadId, String text) {} + } +} diff --git a/pom.xml b/pom.xml index 1ebc76dafd..a9bc21ca5c 100755 --- a/pom.xml +++ b/pom.xml @@ -170,7 +170,7 @@ 0.4.8 1.0.0 - 1.39.0 + 1.48.0 6.6.0 1.35.0 1.6.1