Housekeeper stats; Grafana dashboard

application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java

@@ -33,9 +33,9 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg;
 import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
 import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
 import org.thingsboard.server.queue.util.AfterStartUp;
-import org.thingsboard.server.queue.util.DataDecodingEncodingService;
 import org.thingsboard.server.queue.util.TbCoreComponent;
 import org.thingsboard.server.service.housekeeper.processor.HousekeeperTaskProcessor;
+import org.thingsboard.server.service.housekeeper.stats.HousekeeperStatsService;
 
 import javax.annotation.PreDestroy;
 import java.util.List;
@@ -59,27 +59,28 @@ public class DefaultHousekeeperService implements HousekeeperService {
     private final TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> consumer;
     private final TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> producer;
     private final HousekeeperReprocessingService reprocessingService;
-    private final DataDecodingEncodingService dataDecodingEncodingService;
+    private final HousekeeperStatsService statsService;
 
     private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-consumer"));
     private final ExecutorService executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-task-processor"));
 
-    @Value("${queue.core.housekeeper.poll-interval-ms:10000}")
+    @Value("${queue.core.housekeeper.task-processing-timeout-ms:120000}")
+    private int taskProcessingTimeout;
+    @Value("${queue.core.housekeeper.poll-interval-ms:500}")
     private int pollInterval;
-    private int taskProcessingTimeout = 120;
 
     private boolean stopped;
 
     public DefaultHousekeeperService(HousekeeperReprocessingService reprocessingService,
                                      TbCoreQueueFactory queueFactory,
                                      TbQueueProducerProvider producerProvider,
-                                     DataDecodingEncodingService dataDecodingEncodingService,
+                                     HousekeeperStatsService statsService,
                                      @Lazy List<HousekeeperTaskProcessor<?>> taskProcessors) {
         this.consumer = queueFactory.createHousekeeperMsgConsumer();
         this.producer = producerProvider.getHousekeeperMsgProducer();
         this.reprocessingService = reprocessingService;
+        this.statsService = statsService;
         this.taskProcessors = taskProcessors.stream().collect(Collectors.toMap(HousekeeperTaskProcessor::getTaskType, p -> p));
-        this.dataDecodingEncodingService = dataDecodingEncodingService;
     }
 
     @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
@@ -129,18 +130,23 @@ public class DefaultHousekeeperService implements HousekeeperService {
             throw new IllegalArgumentException("Unsupported task type " + task.getTaskType());
         }
 
+        if (log.isDebugEnabled()) {
+            log.debug("[{}][{}][{}] {} task {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(),
+                    msg.getTask().getErrorsCount() == 0 ? "Processing" : "Reprocessing", task.getTaskType());
+        }
         try {
             Future<Object> future = executor.submit(() -> {
                 taskProcessor.process((T) task);
                 return null;
             });
-            future.get(taskProcessingTimeout, TimeUnit.SECONDS);
+            future.get(taskProcessingTimeout, TimeUnit.MILLISECONDS);
+            statsService.reportProcessed(task, msg);
         } catch (InterruptedException e) {
             throw e;
         } catch (Throwable e) {
             Throwable error = e;
             if (e instanceof ExecutionException) {
-                error = error.getCause();
+                error = e.getCause();
             } else if (e instanceof TimeoutException) {
                 error = new TimeoutException("Timeout after " + taskProcessingTimeout + " seconds");
             }
@@ -148,6 +154,7 @@ public class DefaultHousekeeperService implements HousekeeperService {
                     task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(),
                     task.getTaskType(), msg.getTask().getAttempt(), task, error);
             reprocessingService.submitForReprocessing(queueMsg, error);
+            statsService.reportFailure(task, msg);
         }
     }
 

application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java

@@ -19,7 +19,6 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Lazy;
-import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 import org.thingsboard.common.util.ThingsBoardThreadFactory;
 import org.thingsboard.server.common.data.StringUtils;
@@ -28,20 +27,20 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
 import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
 import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto;
 import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
-import org.thingsboard.server.queue.TbQueueCallback;
-import org.thingsboard.server.queue.TbQueueMsgMetadata;
 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
 import org.thingsboard.server.queue.discovery.PartitionService;
 import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
 import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
 import org.thingsboard.server.queue.util.TbCoreComponent;
 
+import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 
 @TbCoreComponent
@@ -54,16 +53,19 @@ public class HousekeeperReprocessingService {
     private final TbCoreQueueFactory queueFactory;
     private final TbQueueProducerProvider producerProvider;
 
-    private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer"));
+    @Value("${queue.core.housekeeper.reprocessing-start-delay-sec:15}") //  fixme: to 5 minutes
-
+    private int startDelay;
-    private static final int startDelay = 15; // fixme - to 5 minutes
+    @Value("${queue.core.housekeeper.task-reprocessing-delay-sec:30}") // fixme: to 30 minutes or 1 hour
-    private static final int reprocessingDelay = 30; // seconds
+    private int reprocessingDelay;
-    private static final int maxReprocessingAttempts = 5;
+    @Value("${queue.core.housekeeper.max-reprocessing-attempts:10}")
+    private int maxReprocessingAttempts;
     @Value("${queue.core.housekeeper.poll-interval-ms:500}")
     private int pollInterval;
 
+    private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer"));
+    private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-scheduler"));
+
     private boolean stopped;
-    // todo: stats
 
     public HousekeeperReprocessingService(@Lazy DefaultHousekeeperService housekeeperService,
                                           PartitionService partitionService, TbCoreQueueFactory queueFactory,
@@ -74,7 +76,17 @@ public class HousekeeperReprocessingService {
         this.producerProvider = producerProvider;
     }
 
-    @Scheduled(initialDelay = startDelay, fixedDelay = reprocessingDelay, timeUnit = TimeUnit.SECONDS)
+    @PostConstruct
+    private void init() {
+        scheduler.scheduleWithFixedDelay(() -> {
+            try {
+                startReprocessing();
+            } catch (Throwable e) {
+                log.error("Unexpected error during reprocessing", e);
+            }
+        }, startDelay, reprocessingDelay, TimeUnit.SECONDS);
+    }
+
     public void startReprocessing() {
         if (!partitionService.isMyPartition(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID)) {
             return;
@@ -146,6 +158,7 @@ public class HousekeeperReprocessingService {
     @PreDestroy
     private void stop() {
         stopped = true;
+        scheduler.shutdownNow();
         consumerExecutor.shutdownNow();
     }
 

application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java

@@ -36,7 +36,7 @@ public class AlarmsUnassignTaskProcessor implements HousekeeperTaskProcessor<Ala
     @Override
     public void process(AlarmsUnassignHousekeeperTask task) throws Exception {
         List<AlarmId> alarms = alarmService.unassignDeletedUserAlarms(task.getTenantId(), (UserId) task.getEntityId(), task.getUserTitle(), task.getTs());
-        log.trace("[{}][{}] Unassigned {} alarms", task.getTenantId(), task.getEntityId(), alarms.size());
+        log.debug("[{}][{}] Unassigned {} alarms", task.getTenantId(), task.getEntityId(), alarms.size());
     }
 
     @Override

application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java

@@ -32,7 +32,7 @@ public class AttributesDeletionTaskProcessor implements HousekeeperTaskProcessor
     @Override
     public void process(HousekeeperTask task) throws Exception {
         int deletedCount = attributesService.removeAllByEntityId(task.getTenantId(), task.getEntityId());
-        log.trace("[{}][{}][{}] Deleted {} attributes", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount);
+        log.debug("[{}][{}][{}] Deleted {} attributes", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount);
     }
 
     @Override

application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java

@@ -24,16 +24,17 @@ import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType;
 @Component
 @RequiredArgsConstructor
 public class EventsDeletionTaskProcessor implements HousekeeperTaskProcessor<HousekeeperTask> {
+
     private final EventService eventService;
 
     @Override
     public void process(HousekeeperTask task) throws Exception {
         eventService.removeEvents(task.getTenantId(), task.getEntityId(), null, 0L, System.currentTimeMillis());
-        throw new RuntimeException("test error");
     }
 
     @Override
     public HousekeeperTaskType getTaskType() {
         return HousekeeperTaskType.DELETE_EVENTS;
     }
+
 }

application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java

@@ -40,7 +40,7 @@ public class TelemetryDeletionTaskProcessor implements HousekeeperTaskProcessor<
             DeleteTsKvQuery deleteQuery = new BaseDeleteTsKvQuery(key, 0, System.currentTimeMillis(), false, true);
             timeseriesService.remove(task.getTenantId(), task.getEntityId(), List.of(deleteQuery)).get();
         }
-        log.trace("[{}][{}][{}] Deleted {} telemetry keys", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), keys.size());
+        log.debug("[{}][{}][{}] Deleted {} telemetry keys", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), keys.size());
     }
 
     @Override

application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java

@@ -0,0 +1,117 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.housekeeper.stats;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+import org.thingsboard.server.common.stats.DefaultCounter;
+import org.thingsboard.server.common.stats.StatsCounter;
+import org.thingsboard.server.common.stats.StatsFactory;
+import org.thingsboard.server.common.stats.StatsType;
+import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask;
+import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType;
+import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
+
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+@Service
+@Slf4j
+public class HousekeeperStatsService {
+
+    private final Map<HousekeeperTaskType, HousekeeperStats> stats = new EnumMap<>(HousekeeperTaskType.class);
+
+    public HousekeeperStatsService(StatsFactory statsFactory) {
+        for (HousekeeperTaskType taskType : HousekeeperTaskType.values()) {
+            stats.put(taskType, new HousekeeperStats(taskType, statsFactory));
+        }
+    }
+
+    @Scheduled(initialDelay = 60, fixedDelay = 60, timeUnit = TimeUnit.SECONDS)
+    private void reportStats() {
+        String statsStr = stats.values().stream().map(stats -> {
+            String countersStr = stats.getCounters().stream()
+                    .filter(counter -> counter.get() > 0)
+                    .map(counter -> counter.getName() + " = " + counter.get())
+                    .collect(Collectors.joining(", "));
+            if (countersStr.isEmpty()) {
+                return null;
+            } else {
+                return stats.getTaskType() + " {" + countersStr + "}";
+            }
+        }).filter(Objects::nonNull).collect(Collectors.joining("; "));
+
+        if (!statsStr.isEmpty()) {
+            stats.values().forEach(HousekeeperStats::reset);
+            log.info("Housekeeper stats: {}", statsStr);
+        }
+    }
+
+    public void reportProcessed(HousekeeperTask task, ToHousekeeperServiceMsg msg) {
+        HousekeeperStats stats = this.stats.get(task.getTaskType());
+        if (msg.getTask().getErrorsCount() == 0) {
+            stats.getProcessedCounter().increment();
+        } else {
+            stats.getReprocessedCounter().increment();
+        }
+    }
+
+    public void reportFailure(HousekeeperTask task, ToHousekeeperServiceMsg msg) {
+        HousekeeperStats stats = this.stats.get(task.getTaskType());
+        if (msg.getTask().getErrorsCount() == 0) {
+            stats.getFailedProcessingCounter().increment();
+        } else {
+            stats.getFailedReprocessingCounter().increment();
+        }
+    }
+
+    @Getter
+    static class HousekeeperStats {
+        private final HousekeeperTaskType taskType;
+        private final List<StatsCounter> counters = new ArrayList<>();
+
+        private final StatsCounter processedCounter;
+        private final StatsCounter failedProcessingCounter;
+        private final StatsCounter reprocessedCounter;
+        private final StatsCounter failedReprocessingCounter;
+
+        public HousekeeperStats(HousekeeperTaskType taskType, StatsFactory statsFactory) {
+            this.taskType = taskType;
+            this.processedCounter = register("processed", statsFactory);
+            this.failedProcessingCounter = register("failedProcessing", statsFactory);
+            this.reprocessedCounter = register("reprocessed", statsFactory);
+            this.failedReprocessingCounter = register("failedReprocessing", statsFactory);
+        }
+
+        private StatsCounter register(String statsName, StatsFactory statsFactory) {
+            StatsCounter counter = statsFactory.createStatsCounter(StatsType.HOUSEKEEPER.getName(), statsName, Map.of("taskType", taskType.name()));
+            counters.add(counter);
+            return counter;
+        }
+
+        public void reset() {
+            counters.forEach(DefaultCounter::clear);
+        }
+    }
+
+}

application/src/main/resources/thingsboard.yml

@@ -1585,9 +1585,14 @@ queue:
       # Statistics printing interval for Core microservices
       print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}"
     housekeeper:
-      topic: "tb_housekeeper"
+      topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}"
-      reprocessing-topic: "tb_housekeeper.reprocessing"
+      reprocessing-topic: "${TB_HOUSEKEEPER_REPROCESSING_TOPIC:tb_housekeeper.reprocessing}"
-      poll-interval-ms: "1000"
+      poll-interval-ms: "${TB_HOUSEKEEPER_POLL_INTERVAL_MS:500}"
+      task-processing-timeout-ms: "${TB_HOUSEKEEPER_TASK_PROCESSING_TIMEOUT_MS:120000}"
+      reprocessing-start-delay-sec: "${TB_HOUSEKEEPER_REPROCESSING_START_DELAY_SEC:15}" # fixme: to 5 minutes
+      task-reprocessing-delay-sec: "${TB_HOUSEKEEPER_TASK_REPROCESSING_DELAY_SEC:30}" # fixme: to 30 minutes or 1 hour
+      max-reprocessing-attempts: "${TB_HOUSEKEEPER_MAX_REPROCESSING_ATTEMPTS:30}"
+
   vc:
     # Default topic name for Kafka, RabbitMQ, etc.
     topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}"

common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java

@@ -25,6 +25,10 @@ import org.springframework.stereotype.Service;
 import org.thingsboard.server.common.data.StringUtils;
 
 import javax.annotation.PostConstruct;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
 @Service
@@ -62,10 +66,24 @@ public class DefaultStatsFactory implements StatsFactory {
 
     @Override
     public StatsCounter createStatsCounter(String key, String statsName) {
+        return createStatsCounter(key, statsName, Collections.emptyMap());
+    }
+
+    @Override
+    public StatsCounter createStatsCounter(String key, String statsName, Map<String, String> tags) {
+        String[] tagsArr = new String[]{STATS_NAME_TAG, statsName};
+        if (!tags.isEmpty()) {
+            List<String> tagsList = new ArrayList<>(List.of(tagsArr));
+            tags.forEach((name, value) -> {
+                tagsList.add(name);
+                tagsList.add(value);
+            });
+            tagsArr = tagsList.toArray(String[]::new);
+        }
         return new StatsCounter(
                 new AtomicInteger(0),
                 metricsEnabled ?
-                        meterRegistry.counter(key, STATS_NAME_TAG, statsName)
+                        meterRegistry.counter(key, tagsArr)
                         : STUB_COUNTER,
                 statsName
         );

common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java

@@ -17,9 +17,14 @@ package org.thingsboard.server.common.stats;
 
 import io.micrometer.core.instrument.Timer;
 
+import java.util.Map;
+
 public interface StatsFactory {
+
     StatsCounter createStatsCounter(String key, String statsName);
 
+    StatsCounter createStatsCounter(String key, String statsName, Map<String, String> tags);
+
     DefaultCounter createDefaultCounter(String key, String... tags);
 
     <T extends Number> T createGauge(String key, T number, String... tags);

common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java

@@ -16,7 +16,12 @@
 package org.thingsboard.server.common.stats;
 
 public enum StatsType {
-    RULE_ENGINE("ruleEngine"), CORE("core"), TRANSPORT("transport"), JS_INVOKE("jsInvoke"), RATE_EXECUTOR("rateExecutor");
+    RULE_ENGINE("ruleEngine"),
+    CORE("core"),
+    TRANSPORT("transport"),
+    JS_INVOKE("jsInvoke"),
+    RATE_EXECUTOR("rateExecutor"),
+    HOUSEKEEPER("housekeeper");
 
     private String name;
 

docker/monitoring/grafana/provisioning/dashboards/housekeeper.json

@@ -0,0 +1,390 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 2,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 0,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "9BonzvTSz"
+          },
+          "exemplar": true,
+          "expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"processed\"}[1m]))",
+          "interval": "",
+          "legendFormat": "{{taskType}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Processed",
+      "type": "timeseries"
+    },
+    {
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 0,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "9BonzvTSz"
+          },
+          "exemplar": true,
+          "expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"failedProcessing\"}[1m]))",
+          "interval": "",
+          "legendFormat": "{{taskType}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Processing failures",
+      "type": "timeseries"
+    },
+    {
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 0,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 10
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "9BonzvTSz"
+          },
+          "exemplar": true,
+          "expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"reprocessed\"}[1m]))",
+          "interval": "",
+          "legendFormat": "{{taskType}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Reprocessed",
+      "type": "timeseries"
+    },
+    {
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 0,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 10
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "9BonzvTSz"
+          },
+          "exemplar": true,
+          "expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"failedReprocessing\"}[1m]))",
+          "interval": "",
+          "legendFormat": "{{taskType}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Reprocessing failures",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 35,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Housekeeper",
+  "uid": "JFJb7voIz",
+  "version": 5,
+  "weekStart": ""
+}