Add slow_alert_consecutive setting to control slow alert sensitivity

Exposes the consecutive-slow-checks requirement as a configurable setting
instead of hardcoding 3. Saved under key 'slow_alert_consecutive' in the
DB, readable via SLOW_ALERT_CONSECUTIVE env var. UI appears in Settings →
Monitoring alongside the existing Consecutive Failures Threshold field.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
nmemmertandClaude Sonnet 4.6 committed 2026-08-06 16:53:06 -04:00
1 parent dd85a79d55
commit 3a3d32ecac
3 files changed
+19 -2

No files matched your search

+14
View File
@@ -35,6 +35,7 @@ function SettingsWizard() {
items_per_page: 20, items_per_page: 20,
refresh_interval: 5000, refresh_interval: 5000,
incident_failure_threshold: 10, incident_failure_threshold: 10,
slow_alert_consecutive: 3,
}); });
const [testResult, setTestResult] = useState(null); const [testResult, setTestResult] = useState(null);
@@ -529,6 +530,18 @@ function SettingsWizard() {
<small>Number of consecutive failures before triggering an alert (default: 3)</small> <small>Number of consecutive failures before triggering an alert (default: 3)</small>
</div> </div>
<div className="form-group">
<label>Slow Alert Sensitivity</label>
<input
type="number"
value={settings.slow_alert_consecutive || 3}
onChange={(e) => setSettings({ ...settings, slow_alert_consecutive: parseInt(e.target.value) })}
min="1"
max="20"
/>
<small>Consecutive slow checks required before alerting — 1 = any spike, 3 = default, 5+ = only sustained slowness</small>
</div>
<div className="form-group"> <div className="form-group">
<label>Incident Failure Threshold</label> <label>Incident Failure Threshold</label>
<input <input
@@ -579,6 +592,7 @@ function SettingsWizard() {
grace_period: settings.grace_period, grace_period: settings.grace_period,
downtime_threshold: settings.downtime_threshold, downtime_threshold: settings.downtime_threshold,
incident_failure_threshold: settings.incident_failure_threshold, incident_failure_threshold: settings.incident_failure_threshold,
slow_alert_consecutive: settings.slow_alert_consecutive,
})} })}
disabled={savingSection === 'Monitoring'} disabled={savingSection === 'Monitoring'}
> >
+2
View File
@@ -147,6 +147,7 @@ function buildSettingsFromDb(dbSettings = {}) {
ntfy_url: dbSettings.ntfy_url || process.env.NTFY_URL || 'https://ntfy.sh', ntfy_url: dbSettings.ntfy_url || process.env.NTFY_URL || 'https://ntfy.sh',
ntfy_topic: dbSettings.ntfy_topic || process.env.NTFY_TOPIC || '', ntfy_topic: dbSettings.ntfy_topic || process.env.NTFY_TOPIC || '',
incident_failure_threshold: parseInt(dbSettings.incident_failure_threshold || process.env.INCIDENT_FAILURE_THRESHOLD) || 10, incident_failure_threshold: parseInt(dbSettings.incident_failure_threshold || process.env.INCIDENT_FAILURE_THRESHOLD) || 10,
slow_alert_consecutive: parseInt(dbSettings.slow_alert_consecutive || process.env.SLOW_ALERT_CONSECUTIVE) || 3,
webhook_template: dbSettings.webhook_template || process.env.WEBHOOK_TEMPLATE || '', webhook_template: dbSettings.webhook_template || process.env.WEBHOOK_TEMPLATE || '',
}; };
} }
@@ -773,6 +774,7 @@ app.post('/api/settings', (req, res) => {
ntfy_url: { env: 'NTFY_URL', coerce: v => v || 'https://ntfy.sh' }, ntfy_url: { env: 'NTFY_URL', coerce: v => v || 'https://ntfy.sh' },
ntfy_topic: { env: 'NTFY_TOPIC', coerce: v => v || '' }, ntfy_topic: { env: 'NTFY_TOPIC', coerce: v => v || '' },
incident_failure_threshold: { env: 'INCIDENT_FAILURE_THRESHOLD', coerce: v => String(v || 10) }, incident_failure_threshold: { env: 'INCIDENT_FAILURE_THRESHOLD', coerce: v => String(v || 10) },
slow_alert_consecutive: { env: 'SLOW_ALERT_CONSECUTIVE', coerce: v => String(v || 3) },
}; };
// Collect only the fields present in this request // Collect only the fields present in this request
+3 -2
View File
@@ -181,8 +181,9 @@ class Scheduler {
if (result.response_time > effectiveThreshold) { if (result.response_time > effectiveThreshold) {
const consecutive = (this.slowConsecutiveCount.get(resource.id) || 0) + 1; const consecutive = (this.slowConsecutiveCount.get(resource.id) || 0) + 1;
this.slowConsecutiveCount.set(resource.id, consecutive); this.slowConsecutiveCount.set(resource.id, consecutive);
// Require 3 consecutive slow checks before alerting (avoids single-spike false positives) const slowAlertConsecutiveSetting = db.prepare(`SELECT value FROM settings WHERE key = 'slow_alert_consecutive'`).get();
if (consecutive >= 3) { const slowAlertConsecutive = parseInt(slowAlertConsecutiveSetting?.value) || 3;
if (consecutive >= slowAlertConsecutive) {
const lastSlowAlert = this.slowAlertCooldown.get(resource.id) || 0; const lastSlowAlert = this.slowAlertCooldown.get(resource.id) || 0;
if (now - lastSlowAlert > 60 * 60 * 1000) { if (now - lastSlowAlert > 60 * 60 * 1000) {
this.slowAlertCooldown.set(resource.id, now); this.slowAlertCooldown.set(resource.id, now);