From dd85a79d551e7453f9167ff6e4e827688565ef19 Mon Sep 17 00:00:00 2001
From: nmemmert <67961292+nmemmert@users.noreply.github.com>
Date: Thu, 6 Aug 2026 16:50:57 -0400
Subject: [PATCH] Fix false-positive slow alerts: require 3 consecutive checks
 + use P95 baseline
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Previously a single check above 2× average response time fired an alert.
This caused false positives from normal network variance.

- scheduler.js: track consecutive slow checks per resource; only alert after
  3 in a row (resets to 0 when response time normalises)
- monitorService.js: calculateResponseTimeBaseline now returns P95 instead
  of mean, making 2× baseline a much more realistic slow threshold

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 server/monitorService.js |  7 ++++---
 server/scheduler.js      | 40 ++++++++++++++++++++++++----------------
 2 files changed, 28 insertions(+), 19 deletions(-)

diff --git a/server/monitorService.js b/server/monitorService.js
index 215c846..e353c59 100644
--- a/server/monitorService.js
+++ b/server/monitorService.js
@@ -765,9 +765,10 @@ class MonitorService {
       return null;
     }
 
-    // Return average of response times
-    const sum = checks.reduce((acc, c) => acc + (c.response_time || 0), 0);
-    return Math.round(sum / checks.length);
+    // Use P95 so the baseline reflects realistic worst-case, not the average.
+    // Threshold is 2× P95, making it very unlikely to trigger on normal variance.
+    const p95Index = Math.min(Math.floor(checks.length * 0.95), checks.length - 1);
+    return Math.round(checks[p95Index].response_time);
   }
 
   // Check if response time is anomalous
diff --git a/server/scheduler.js b/server/scheduler.js
index 4500477..861191b 100644
--- a/server/scheduler.js
+++ b/server/scheduler.js
@@ -9,6 +9,7 @@ class Scheduler {
     this.isRunning = false;
     this.lastCheckedAt = new Map();
     this.slowAlertCooldown = new Map();
+    this.slowConsecutiveCount = new Map(); // tracks consecutive slow checks per resource
   }
 
   start() {
@@ -172,26 +173,33 @@ class Scheduler {
           pendingAlerts.push({ resource, incident, stats });
         }
 
-        // Slow alert: prefer explicit threshold, fall back to 2× computed baseline
+        // Slow alert: prefer explicit threshold, fall back to 2× P95 baseline
         const effectiveThreshold = resource.response_time_threshold ||
           (resource.response_time_baseline ? resource.response_time_baseline * 2 : null);
 
-        if (
-          result.status === 'up' &&
-          effectiveThreshold &&
-          result.response_time > effectiveThreshold &&
-          !resource.maintenance_mode
-        ) {
-          const lastSlowAlert = this.slowAlertCooldown.get(resource.id) || 0;
-          if (now - lastSlowAlert > 60 * 60 * 1000) {
-            this.slowAlertCooldown.set(resource.id, now);
-            await notificationService.sendAlert(resource, {
-              type: 'slow',
-              responseTime: result.response_time,
-              threshold: effectiveThreshold,
-              id: null,
-            });
+        if (result.status === 'up' && effectiveThreshold && !resource.maintenance_mode) {
+          if (result.response_time > effectiveThreshold) {
+            const consecutive = (this.slowConsecutiveCount.get(resource.id) || 0) + 1;
+            this.slowConsecutiveCount.set(resource.id, consecutive);
+            // Require 3 consecutive slow checks before alerting (avoids single-spike false positives)
+            if (consecutive >= 3) {
+              const lastSlowAlert = this.slowAlertCooldown.get(resource.id) || 0;
+              if (now - lastSlowAlert > 60 * 60 * 1000) {
+                this.slowAlertCooldown.set(resource.id, now);
+                await notificationService.sendAlert(resource, {
+                  type: 'slow',
+                  responseTime: result.response_time,
+                  threshold: effectiveThreshold,
+                  id: null,
+                });
+              }
+            }
+          } else {
+            // Reset consecutive counter when response time is back to normal
+            this.slowConsecutiveCount.set(resource.id, 0);
           }
+        } else if (!effectiveThreshold) {
+          this.slowConsecutiveCount.set(resource.id, 0);
         }
       } catch (error) {
         // Check error handled silently