Add HTTP method/body, alert escalation, baseline alerts, notification grouping, rate limiting, App.js split, and API tests

- HTTP method + request body: GET/POST/PUT/PATCH/DELETE/HEAD selector and body textarea for http/health checks; stored in http_method/http_body columns; monitorService sends correct axios method+data
- Alert escalation: incidents open longer than escalation_hours get forwarded to fallback_webhook and marked escalated=1; hourly cron in scheduler
- Baseline-relative slow alerts: effectiveThreshold = response_time_threshold || (baseline * 2) — adaptive threshold without manual config
- Notification grouping: pendingAlerts collected per check cycle; ≥2 alerts flush as one grouped email+webhook; individual in-app entries still saved per resource
- Rate limiting: in-memory per-IP limiter (20/min heartbeat, 60/min status-page); no external package
- App.js split: Dashboard.js (monitors list + modals) and ResourceDetail.js extracted; App.js reduced from ~1987 to ~80 lines
- formatDuration moved to utils/timeUtils.js; imported by both new components
- DB migrations: http_method, http_body, incidents.escalated columns
- 18 passing API/unit tests (node:test, zero dependencies)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
nmemmertandClaude Sonnet 4.6 committed 2026-07-31 09:37:31 -04:00
1 parent a6cf6a9762
commit 4fea0a95f5
19 files changed
+1795 -1966

No files matched your search

+73 -30
View File
@@ -24,30 +24,30 @@ class Scheduler {
}
});
// Archive old checks daily at 2 AM
// Archive old checks + update baselines + check escalations daily at 2 AM
cron.schedule('0 2 * * *', () => {
this.archiveOldChecks();
this.updateBaselines();
});
// Escalation check runs every hour
cron.schedule('0 * * * *', async () => {
await this.checkEscalations();
});
// Also run archive on startup (with delay to let DB init)
setTimeout(() => {
this.archiveOldChecks();
this.updateBaselines();
}, 5000);
}
archiveOldChecks() {
try {
// Get global retention setting
const retentionSetting = db.prepare(`
SELECT value FROM settings WHERE key = 'retention_days'
`).get();
const retentionSetting = db.prepare(`SELECT value FROM settings WHERE key = 'retention_days'`).get();
const globalRetentionDays = retentionSetting ? parseInt(retentionSetting.value) : 30;
// Get all resources with their retention settings
const resources = db.prepare(`
SELECT id, retention_days FROM resources
`).all();
const resources = db.prepare(`SELECT id, retention_days FROM resources`).all();
const bulkArchive = db.prepare(`
INSERT INTO archived_checks (resource_id, status, response_time, status_code, error_message, details, checked_at)
@@ -73,16 +73,60 @@ class Scheduler {
}
}
updateBaselines() {
try {
const resources = db.prepare(`SELECT id, type FROM resources WHERE type != 'heartbeat'`).all();
for (const resource of resources) {
const baseline = monitorService.calculateResponseTimeBaseline(resource.id, 7);
if (baseline !== null) {
db.prepare(`UPDATE resources SET response_time_baseline = ? WHERE id = ?`).run(baseline, resource.id);
}
}
} catch (error) {
// Baseline update error handled silently
}
}
async checkEscalations() {
try {
const escalationSetting = db.prepare(`SELECT value FROM settings WHERE key = 'escalation_hours'`).get();
const escalationHours = parseInt(escalationSetting?.value || '4');
const fallbackSetting = db.prepare(`SELECT value FROM settings WHERE key = 'fallback_webhook'`).get();
const fallbackWebhook = fallbackSetting?.value || process.env.FALLBACK_WEBHOOK || '';
if (!fallbackWebhook) return;
const cutoff = new Date(Date.now() - escalationHours * 60 * 60 * 1000)
.toISOString().replace('T', ' ').split('.')[0];
const stalled = db.prepare(`
SELECT i.*, r.name AS resource_name, r.url AS resource_url, r.type AS resource_type,
r.maintenance_mode
FROM incidents i
JOIN resources r ON r.id = i.resource_id
WHERE i.resolved_at IS NULL AND i.escalated = 0 AND i.started_at < ?
AND r.maintenance_mode = 0
`).all(cutoff);
for (const incident of stalled) {
await notificationService.sendEscalationAlert(incident, fallbackWebhook);
db.prepare(`UPDATE incidents SET escalated = 1 WHERE id = ?`).run(incident.id);
}
} catch (error) {
// Escalation check error handled silently
}
}
async runChecks() {
const resources = db.prepare(`
SELECT * FROM resources WHERE enabled = 1
`).all();
const resources = db.prepare(`SELECT * FROM resources WHERE enabled = 1`).all();
// Stagger checks to avoid thundering herd
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
const jitter = (maxMs = 1500) => Math.floor(Math.random() * maxMs);
const baseSpacingMs = 250;
// Collect incident alerts to flush as a group after all checks complete
const pendingAlerts = [];
const baseSpacingMs = 250; // spread starts in 250ms increments
const tasks = resources.map((resource, index) => (async () => {
// Honour per-resource check_interval — skip if not yet due
const now = Date.now();
@@ -115,30 +159,27 @@ class Scheduler {
// Non-fatal
}
const incident = monitorService.handleIncident(
resource.id,
result.status === 'down'
);
const incident = monitorService.handleIncident(resource.id, result.status === 'down');
if (incident.type !== 'none') {
if (incident.type !== 'none' && !resource.maintenance_mode) {
const stats = monitorService.getResourceStats(resource.id, 24);
const recentChecks = db.prepare(`
SELECT response_time, status FROM checks
WHERE resource_id = ?
ORDER BY checked_at DESC
LIMIT 12
ORDER BY checked_at DESC LIMIT 12
`).all(resource.id);
stats.recentChecks = recentChecks.reverse();
if (!resource.maintenance_mode) {
await notificationService.sendAlert(resource, incident, stats);
}
pendingAlerts.push({ resource, incident, stats });
}
// Response time threshold alert (only when up, with 1-hour cooldown)
// Slow alert: prefer explicit threshold, fall back to 2× computed baseline
const effectiveThreshold = resource.response_time_threshold ||
(resource.response_time_baseline ? resource.response_time_baseline * 2 : null);
if (
result.status === 'up' &&
resource.response_time_threshold &&
result.response_time > resource.response_time_threshold &&
effectiveThreshold &&
result.response_time > effectiveThreshold &&
!resource.maintenance_mode
) {
const lastSlowAlert = this.slowAlertCooldown.get(resource.id) || 0;
@@ -147,7 +188,7 @@ class Scheduler {
await notificationService.sendAlert(resource, {
type: 'slow',
responseTime: result.response_time,
threshold: resource.response_time_threshold,
threshold: effectiveThreshold,
id: null,
});
}
@@ -159,6 +200,9 @@ class Scheduler {
await Promise.allSettled(tasks);
// Flush collected alerts — grouped if multiple fired in the same cycle
await notificationService.flushAlerts(pendingAlerts);
// Broadcast updated dashboard to all connected WebSocket clients
if (global.broadcastDashboardUpdate) {
global.broadcastDashboardUpdate();
@@ -166,7 +210,6 @@ class Scheduler {
}
stop() {
// Stop the scheduler (cron tasks will continue running but we flag isRunning)
this.isRunning = false;
}
}