Add slow_alert_consecutive setting to control slow alert sensitivity
Exposes the consecutive-slow-checks requirement as a configurable setting instead of hardcoding 3. Saved under key 'slow_alert_consecutive' in the DB, readable via SLOW_ALERT_CONSECUTIVE env var. UI appears in Settings → Monitoring alongside the existing Consecutive Failures Threshold field. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
dd85a79d55
commit
3a3d32ecac
3 files changed
+19
-2
No files matched your search
@@ -35,6 +35,7 @@ function SettingsWizard() {
|
|||||||
items_per_page: 20,
|
items_per_page: 20,
|
||||||
refresh_interval: 5000,
|
refresh_interval: 5000,
|
||||||
incident_failure_threshold: 10,
|
incident_failure_threshold: 10,
|
||||||
|
slow_alert_consecutive: 3,
|
||||||
});
|
});
|
||||||
|
|
||||||
const [testResult, setTestResult] = useState(null);
|
const [testResult, setTestResult] = useState(null);
|
||||||
@@ -529,6 +530,18 @@ function SettingsWizard() {
|
|||||||
<small>Number of consecutive failures before triggering an alert (default: 3)</small>
|
<small>Number of consecutive failures before triggering an alert (default: 3)</small>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div className="form-group">
|
||||||
|
<label>Slow Alert Sensitivity</label>
|
||||||
|
<input
|
||||||
|
type="number"
|
||||||
|
value={settings.slow_alert_consecutive || 3}
|
||||||
|
onChange={(e) => setSettings({ ...settings, slow_alert_consecutive: parseInt(e.target.value) })}
|
||||||
|
min="1"
|
||||||
|
max="20"
|
||||||
|
/>
|
||||||
|
<small>Consecutive slow checks required before alerting — 1 = any spike, 3 = default, 5+ = only sustained slowness</small>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div className="form-group">
|
<div className="form-group">
|
||||||
<label>Incident Failure Threshold</label>
|
<label>Incident Failure Threshold</label>
|
||||||
<input
|
<input
|
||||||
@@ -579,6 +592,7 @@ function SettingsWizard() {
|
|||||||
grace_period: settings.grace_period,
|
grace_period: settings.grace_period,
|
||||||
downtime_threshold: settings.downtime_threshold,
|
downtime_threshold: settings.downtime_threshold,
|
||||||
incident_failure_threshold: settings.incident_failure_threshold,
|
incident_failure_threshold: settings.incident_failure_threshold,
|
||||||
|
slow_alert_consecutive: settings.slow_alert_consecutive,
|
||||||
})}
|
})}
|
||||||
disabled={savingSection === 'Monitoring'}
|
disabled={savingSection === 'Monitoring'}
|
||||||
>
|
>
|
||||||
|
|||||||
@@ -147,6 +147,7 @@ function buildSettingsFromDb(dbSettings = {}) {
|
|||||||
ntfy_url: dbSettings.ntfy_url || process.env.NTFY_URL || 'https://ntfy.sh',
|
ntfy_url: dbSettings.ntfy_url || process.env.NTFY_URL || 'https://ntfy.sh',
|
||||||
ntfy_topic: dbSettings.ntfy_topic || process.env.NTFY_TOPIC || '',
|
ntfy_topic: dbSettings.ntfy_topic || process.env.NTFY_TOPIC || '',
|
||||||
incident_failure_threshold: parseInt(dbSettings.incident_failure_threshold || process.env.INCIDENT_FAILURE_THRESHOLD) || 10,
|
incident_failure_threshold: parseInt(dbSettings.incident_failure_threshold || process.env.INCIDENT_FAILURE_THRESHOLD) || 10,
|
||||||
|
slow_alert_consecutive: parseInt(dbSettings.slow_alert_consecutive || process.env.SLOW_ALERT_CONSECUTIVE) || 3,
|
||||||
webhook_template: dbSettings.webhook_template || process.env.WEBHOOK_TEMPLATE || '',
|
webhook_template: dbSettings.webhook_template || process.env.WEBHOOK_TEMPLATE || '',
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -773,6 +774,7 @@ app.post('/api/settings', (req, res) => {
|
|||||||
ntfy_url: { env: 'NTFY_URL', coerce: v => v || 'https://ntfy.sh' },
|
ntfy_url: { env: 'NTFY_URL', coerce: v => v || 'https://ntfy.sh' },
|
||||||
ntfy_topic: { env: 'NTFY_TOPIC', coerce: v => v || '' },
|
ntfy_topic: { env: 'NTFY_TOPIC', coerce: v => v || '' },
|
||||||
incident_failure_threshold: { env: 'INCIDENT_FAILURE_THRESHOLD', coerce: v => String(v || 10) },
|
incident_failure_threshold: { env: 'INCIDENT_FAILURE_THRESHOLD', coerce: v => String(v || 10) },
|
||||||
|
slow_alert_consecutive: { env: 'SLOW_ALERT_CONSECUTIVE', coerce: v => String(v || 3) },
|
||||||
};
|
};
|
||||||
|
|
||||||
// Collect only the fields present in this request
|
// Collect only the fields present in this request
|
||||||
|
|||||||
+3
-2
@@ -181,8 +181,9 @@ class Scheduler {
|
|||||||
if (result.response_time > effectiveThreshold) {
|
if (result.response_time > effectiveThreshold) {
|
||||||
const consecutive = (this.slowConsecutiveCount.get(resource.id) || 0) + 1;
|
const consecutive = (this.slowConsecutiveCount.get(resource.id) || 0) + 1;
|
||||||
this.slowConsecutiveCount.set(resource.id, consecutive);
|
this.slowConsecutiveCount.set(resource.id, consecutive);
|
||||||
// Require 3 consecutive slow checks before alerting (avoids single-spike false positives)
|
const slowAlertConsecutiveSetting = db.prepare(`SELECT value FROM settings WHERE key = 'slow_alert_consecutive'`).get();
|
||||||
if (consecutive >= 3) {
|
const slowAlertConsecutive = parseInt(slowAlertConsecutiveSetting?.value) || 3;
|
||||||
|
if (consecutive >= slowAlertConsecutive) {
|
||||||
const lastSlowAlert = this.slowAlertCooldown.get(resource.id) || 0;
|
const lastSlowAlert = this.slowAlertCooldown.get(resource.id) || 0;
|
||||||
if (now - lastSlowAlert > 60 * 60 * 1000) {
|
if (now - lastSlowAlert > 60 * 60 * 1000) {
|
||||||
this.slowAlertCooldown.set(resource.id, now);
|
this.slowAlertCooldown.set(resource.id, now);
|
||||||
|
|||||||
Reference in new issue
Block a user