Add HTTP method/body, alert escalation, baseline alerts, notification grouping, rate limiting, App.js split, and API tests

- HTTP method + request body: GET/POST/PUT/PATCH/DELETE/HEAD selector and body textarea for http/health checks; stored in http_method/http_body columns; monitorService sends correct axios method+data
- Alert escalation: incidents open longer than escalation_hours get forwarded to fallback_webhook and marked escalated=1; hourly cron in scheduler
- Baseline-relative slow alerts: effectiveThreshold = response_time_threshold || (baseline * 2) — adaptive threshold without manual config
- Notification grouping: pendingAlerts collected per check cycle; ≥2 alerts flush as one grouped email+webhook; individual in-app entries still saved per resource
- Rate limiting: in-memory per-IP limiter (20/min heartbeat, 60/min status-page); no external package
- App.js split: Dashboard.js (monitors list + modals) and ResourceDetail.js extracted; App.js reduced from ~1987 to ~80 lines
- formatDuration moved to utils/timeUtils.js; imported by both new components
- DB migrations: http_method, http_body, incidents.escalated columns
- 18 passing API/unit tests (node:test, zero dependencies)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
nmemmertandClaude Sonnet 4.6 committed 2026-07-31 09:37:31 -04:00
1 parent a6cf6a9762
commit 4fea0a95f5
19 files changed
+1795 -1966

No files matched your search

+7
View File
@@ -311,6 +311,13 @@ try {
db.prepare("INSERT OR IGNORE INTO settings (key, value) VALUES ('incident_failure_threshold', '10')").run();
} catch (err) {}
// Feature: HTTP method + request body support
try { db.prepare("ALTER TABLE resources ADD COLUMN http_method TEXT DEFAULT 'GET'").run(); } catch(e) {}
try { db.prepare("ALTER TABLE resources ADD COLUMN http_body TEXT").run(); } catch(e) {}
// Feature: alert escalation tracking
try { db.prepare("ALTER TABLE incidents ADD COLUMN escalated INTEGER DEFAULT 0").run(); } catch(e) {}
// Feature: heartbeat / cron-job monitoring
try { db.prepare("ALTER TABLE resources ADD COLUMN heartbeat_token TEXT").run(); } catch(e) {}
try { db.prepare("ALTER TABLE resources ADD COLUMN heartbeat_timeout INTEGER DEFAULT 300000").run(); } catch(e) {}
+32 -8
View File
@@ -141,6 +141,24 @@ function buildSettingsFromDb(dbSettings = {}) {
};
}
// Simple in-memory per-IP rate limiter (no external package needed)
const _rlMap = new Map();
setInterval(() => {
const now = Date.now();
for (const [k, v] of _rlMap) { if (now - v.start > 60000) _rlMap.delete(k); }
}, 5 * 60 * 1000);
function rateLimit(maxReqs, windowMs = 60000) {
return (req, res, next) => {
const key = `${req.path}:${req.ip}`;
const now = Date.now();
const e = _rlMap.get(key);
if (!e || now - e.start > windowMs) { _rlMap.set(key, { start: now, count: 1 }); return next(); }
e.count++;
if (e.count > maxReqs) return res.status(429).json({ error: 'Too many requests' });
next();
};
}
// Build dashboard payload (shared between HTTP route and WebSocket broadcast)
function getDashboardPayload() {
const resources = db.prepare('SELECT * FROM resources ORDER BY group_id, name').all();
@@ -194,6 +212,8 @@ function getDashboardPayload() {
heartbeat_timeout: resource.heartbeat_timeout,
consecutive_failures_threshold: resource.consecutive_failures_threshold,
response_time_threshold: resource.response_time_threshold,
http_method: resource.http_method || 'GET',
http_body: resource.http_body || null,
certDaysRemaining,
status: lastCheck?.status || 'unknown',
uptime: stats.uptime,
@@ -311,7 +331,7 @@ app.get('/api/resources/:id', (req, res) => {
// Create resource
app.post('/api/resources', (req, res) => {
const { name, url, type, check_interval, timeout, group_id, http_keyword, http_headers, quiet_hours_start, quiet_hours_end, cert_expiry_days, sla_target, email_to, maintenance_mode, retention_days, is_public, heartbeat_timeout, consecutive_failures_threshold, response_time_threshold } = req.body;
const { name, url, type, check_interval, timeout, group_id, http_keyword, http_headers, quiet_hours_start, quiet_hours_end, cert_expiry_days, sla_target, email_to, maintenance_mode, retention_days, is_public, heartbeat_timeout, consecutive_failures_threshold, response_time_threshold, http_method, http_body } = req.body;
const validationError = validateResourceBody(req.body);
if (validationError) return res.status(400).json({ error: validationError });
@@ -322,8 +342,8 @@ app.post('/api/resources', (req, res) => {
const effectiveUrl = isHeartbeat ? `heartbeat://${name}` : url;
const stmt = db.prepare(`
INSERT INTO resources (name, url, type, check_interval, timeout, group_id, http_keyword, http_headers, quiet_hours_start, quiet_hours_end, cert_expiry_days, sla_target, email_to, maintenance_mode, retention_days, is_public, heartbeat_token, heartbeat_timeout, consecutive_failures_threshold, response_time_threshold)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
INSERT INTO resources (name, url, type, check_interval, timeout, group_id, http_keyword, http_headers, quiet_hours_start, quiet_hours_end, cert_expiry_days, sla_target, email_to, maintenance_mode, retention_days, is_public, heartbeat_token, heartbeat_timeout, consecutive_failures_threshold, response_time_threshold, http_method, http_body)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
const result = stmt.run(
@@ -346,7 +366,9 @@ app.post('/api/resources', (req, res) => {
token,
heartbeat_timeout || 300000,
consecutive_failures_threshold || 1,
response_time_threshold || null
response_time_threshold || null,
http_method || 'GET',
http_body || null
);
// Invalidate related cache entries
@@ -367,7 +389,7 @@ app.post('/api/resources', (req, res) => {
// Update resource
app.put('/api/resources/:id', (req, res) => {
const { name, url, type, check_interval, timeout, enabled, group_id, http_keyword, http_headers, quiet_hours_start, quiet_hours_end, cert_expiry_days, sla_target, email_to, maintenance_mode, retention_days, is_public, heartbeat_timeout, consecutive_failures_threshold, response_time_threshold } = req.body;
const { name, url, type, check_interval, timeout, enabled, group_id, http_keyword, http_headers, quiet_hours_start, quiet_hours_end, cert_expiry_days, sla_target, email_to, maintenance_mode, retention_days, is_public, heartbeat_timeout, consecutive_failures_threshold, response_time_threshold, http_method, http_body } = req.body;
const validationError = validateResourceBody(req.body);
if (validationError) return res.status(400).json({ error: validationError });
@@ -378,7 +400,7 @@ app.put('/api/resources/:id', (req, res) => {
const stmt = db.prepare(`
UPDATE resources
SET name = ?, url = ?, type = ?, check_interval = ?, timeout = ?, enabled = ?, group_id = ?, http_keyword = ?, http_headers = ?, quiet_hours_start = ?, quiet_hours_end = ?, cert_expiry_days = ?, sla_target = ?, email_to = ?, maintenance_mode = ?, retention_days = ?, is_public = ?, heartbeat_timeout = ?, consecutive_failures_threshold = ?, response_time_threshold = ?
SET name = ?, url = ?, type = ?, check_interval = ?, timeout = ?, enabled = ?, group_id = ?, http_keyword = ?, http_headers = ?, quiet_hours_start = ?, quiet_hours_end = ?, cert_expiry_days = ?, sla_target = ?, email_to = ?, maintenance_mode = ?, retention_days = ?, is_public = ?, heartbeat_timeout = ?, consecutive_failures_threshold = ?, response_time_threshold = ?, http_method = ?, http_body = ?
WHERE id = ?
`);
@@ -403,6 +425,8 @@ app.put('/api/resources/:id', (req, res) => {
heartbeat_timeout || 300000,
consecutive_failures_threshold || 1,
response_time_threshold || null,
http_method || 'GET',
http_body || null,
req.params.id
);
@@ -1715,7 +1739,7 @@ app.get('/api/resources/:id/alert-rules', (req, res) => {
});
// Heartbeat ping endpoint — cron jobs POST here to signal they're alive
app.post('/api/heartbeat/:token', (req, res) => {
app.post('/api/heartbeat/:token', rateLimit(20), (req, res) => {
const resource = db.prepare('SELECT id, name FROM resources WHERE heartbeat_token = ? AND enabled = 1').get(req.params.token);
if (!resource) return res.status(404).json({ error: 'Unknown heartbeat token' });
@@ -1731,7 +1755,7 @@ app.post('/api/heartbeat/:token', (req, res) => {
});
// Public status page — only returns resources with is_public = 1
app.get('/api/status-page', (req, res) => {
app.get('/api/status-page', rateLimit(60), (req, res) => {
try {
const resources = db.prepare('SELECT * FROM resources WHERE is_public = 1 AND enabled = 1 ORDER BY name').all();
const incidents = db.prepare(`
+14 -3
View File
@@ -145,15 +145,26 @@ class MonitorService {
}
}
const isHttps = resource.url.startsWith('https');
const response = await axios.get(resource.url, {
const method = (resource.http_method || 'GET').toUpperCase();
const requestConfig = {
method,
url: resource.url,
timeout: resource.timeout,
validateStatus: () => true,
maxRedirects: 5,
headers,
httpAgent: this.httpAgent,
httpsAgent: this.httpsAgent,
});
};
if (resource.http_body && ['POST', 'PUT', 'PATCH'].includes(method)) {
try {
requestConfig.data = JSON.parse(resource.http_body);
if (!headers['Content-Type']) headers['Content-Type'] = 'application/json';
} catch {
requestConfig.data = resource.http_body;
}
}
const response = await axios(requestConfig);
const responseTime = Date.now() - startTime;
const isUp = response.status >= 200 && response.status < 400;
+93
View File
@@ -149,6 +149,99 @@ class NotificationService {
}
}
async flushAlerts(pendingAlerts) {
if (pendingAlerts.length === 0) return;
if (pendingAlerts.length === 1) {
const { resource, incident, stats } = pendingAlerts[0];
await this.sendAlert(resource, incident, stats);
return;
}
// Multiple alerts in one check cycle — save individual in-app entries but send one grouped email/webhook
for (const { resource, incident } of pendingAlerts) {
const status = incident.type === 'started' ? 'down' : 'up';
const message = `${resource.name} is ${status.toUpperCase()}. URL: ${resource.url}`;
this.saveNotification(resource, incident, status, message);
}
await this.sendGroupedEmail(pendingAlerts);
if (this.webhookEnabled) {
await this.sendGroupedWebhook(pendingAlerts);
}
}
async sendGroupedEmail(alerts) {
if (!this.transporter) return;
const targetEmail = this.config.email_to;
if (!targetEmail || !this.emailEnabled) return;
try {
const rows = alerts.map(({ resource, incident }) => {
const status = incident.type === 'started' ? 'DOWN' : 'UP';
const emoji = incident.type === 'started' ? '🔴' : '🟢';
return `<li>${emoji} <strong>${resource.name}</strong> is ${status} — ${resource.url}</li>`;
}).join('');
await this.transporter.sendMail({
from: this.config.email_from,
to: targetEmail,
subject: `⚠️ ${alerts.length} monitors changed status`,
text: alerts.map(({ resource, incident }) =>
`${resource.name} is ${incident.type === 'started' ? 'DOWN' : 'UP'} — ${resource.url}`
).join('\n'),
html: `<div style="font-family:Arial,sans-serif;max-width:600px">
<h2 style="color:#d32f2f">⚠️ ${alerts.length} Monitors Changed Status</h2>
<ul>${rows}</ul>
<p><strong>Time:</strong> ${new Date().toLocaleString()}</p>
</div>`,
});
} catch (error) {
// Grouped email error handled
}
}
async sendGroupedWebhook(alerts) {
const webhookUrl = this.config.webhook_url || process.env.WEBHOOK_URL;
if (!webhookUrl) return;
try {
await axios.post(webhookUrl, {
type: 'grouped_alert',
count: alerts.length,
resources: alerts.map(({ resource, incident }) => ({
name: resource.name,
url: resource.url,
status: incident.type === 'started' ? 'down' : 'up',
})),
timestamp: new Date().toISOString(),
});
} catch (error) {
// Grouped webhook error handled
}
}
async sendEscalationAlert(incident, fallbackWebhookUrl) {
try {
const durationMin = Math.round((Date.now() - new Date(incident.started_at).getTime()) / 60000);
const message = `⚠️ ESCALATION: ${incident.resource_name} has been DOWN for ${durationMin} minutes without resolution.`;
if (fallbackWebhookUrl) {
await axios.post(fallbackWebhookUrl, {
resource: incident.resource_name,
url: incident.resource_url,
status: 'escalated',
duration_minutes: durationMin,
incident_id: incident.id,
started_at: incident.started_at,
message,
timestamp: new Date().toISOString(),
});
}
this.saveNotification(
{ id: incident.resource_id, name: incident.resource_name, url: incident.resource_url },
{ id: incident.id },
'escalated',
message
);
} catch (error) {
// Escalation alert error handled
}
}
isQuietHours(resource) {
if (!resource.quiet_hours_start || !resource.quiet_hours_end) {
return false;
+73 -30
View File
@@ -24,30 +24,30 @@ class Scheduler {
}
});
// Archive old checks daily at 2 AM
// Archive old checks + update baselines + check escalations daily at 2 AM
cron.schedule('0 2 * * *', () => {
this.archiveOldChecks();
this.updateBaselines();
});
// Escalation check runs every hour
cron.schedule('0 * * * *', async () => {
await this.checkEscalations();
});
// Also run archive on startup (with delay to let DB init)
setTimeout(() => {
this.archiveOldChecks();
this.updateBaselines();
}, 5000);
}
archiveOldChecks() {
try {
// Get global retention setting
const retentionSetting = db.prepare(`
SELECT value FROM settings WHERE key = 'retention_days'
`).get();
const retentionSetting = db.prepare(`SELECT value FROM settings WHERE key = 'retention_days'`).get();
const globalRetentionDays = retentionSetting ? parseInt(retentionSetting.value) : 30;
// Get all resources with their retention settings
const resources = db.prepare(`
SELECT id, retention_days FROM resources
`).all();
const resources = db.prepare(`SELECT id, retention_days FROM resources`).all();
const bulkArchive = db.prepare(`
INSERT INTO archived_checks (resource_id, status, response_time, status_code, error_message, details, checked_at)
@@ -73,16 +73,60 @@ class Scheduler {
}
}
updateBaselines() {
try {
const resources = db.prepare(`SELECT id, type FROM resources WHERE type != 'heartbeat'`).all();
for (const resource of resources) {
const baseline = monitorService.calculateResponseTimeBaseline(resource.id, 7);
if (baseline !== null) {
db.prepare(`UPDATE resources SET response_time_baseline = ? WHERE id = ?`).run(baseline, resource.id);
}
}
} catch (error) {
// Baseline update error handled silently
}
}
async checkEscalations() {
try {
const escalationSetting = db.prepare(`SELECT value FROM settings WHERE key = 'escalation_hours'`).get();
const escalationHours = parseInt(escalationSetting?.value || '4');
const fallbackSetting = db.prepare(`SELECT value FROM settings WHERE key = 'fallback_webhook'`).get();
const fallbackWebhook = fallbackSetting?.value || process.env.FALLBACK_WEBHOOK || '';
if (!fallbackWebhook) return;
const cutoff = new Date(Date.now() - escalationHours * 60 * 60 * 1000)
.toISOString().replace('T', ' ').split('.')[0];
const stalled = db.prepare(`
SELECT i.*, r.name AS resource_name, r.url AS resource_url, r.type AS resource_type,
r.maintenance_mode
FROM incidents i
JOIN resources r ON r.id = i.resource_id
WHERE i.resolved_at IS NULL AND i.escalated = 0 AND i.started_at < ?
AND r.maintenance_mode = 0
`).all(cutoff);
for (const incident of stalled) {
await notificationService.sendEscalationAlert(incident, fallbackWebhook);
db.prepare(`UPDATE incidents SET escalated = 1 WHERE id = ?`).run(incident.id);
}
} catch (error) {
// Escalation check error handled silently
}
}
async runChecks() {
const resources = db.prepare(`
SELECT * FROM resources WHERE enabled = 1
`).all();
const resources = db.prepare(`SELECT * FROM resources WHERE enabled = 1`).all();
// Stagger checks to avoid thundering herd
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
const jitter = (maxMs = 1500) => Math.floor(Math.random() * maxMs);
const baseSpacingMs = 250;
// Collect incident alerts to flush as a group after all checks complete
const pendingAlerts = [];
const baseSpacingMs = 250; // spread starts in 250ms increments
const tasks = resources.map((resource, index) => (async () => {
// Honour per-resource check_interval — skip if not yet due
const now = Date.now();
@@ -115,30 +159,27 @@ class Scheduler {
// Non-fatal
}
const incident = monitorService.handleIncident(
resource.id,
result.status === 'down'
);
const incident = monitorService.handleIncident(resource.id, result.status === 'down');
if (incident.type !== 'none') {
if (incident.type !== 'none' && !resource.maintenance_mode) {
const stats = monitorService.getResourceStats(resource.id, 24);
const recentChecks = db.prepare(`
SELECT response_time, status FROM checks
WHERE resource_id = ?
ORDER BY checked_at DESC
LIMIT 12
ORDER BY checked_at DESC LIMIT 12
`).all(resource.id);
stats.recentChecks = recentChecks.reverse();
if (!resource.maintenance_mode) {
await notificationService.sendAlert(resource, incident, stats);
}
pendingAlerts.push({ resource, incident, stats });
}
// Response time threshold alert (only when up, with 1-hour cooldown)
// Slow alert: prefer explicit threshold, fall back to 2× computed baseline
const effectiveThreshold = resource.response_time_threshold ||
(resource.response_time_baseline ? resource.response_time_baseline * 2 : null);
if (
result.status === 'up' &&
resource.response_time_threshold &&
result.response_time > resource.response_time_threshold &&
effectiveThreshold &&
result.response_time > effectiveThreshold &&
!resource.maintenance_mode
) {
const lastSlowAlert = this.slowAlertCooldown.get(resource.id) || 0;
@@ -147,7 +188,7 @@ class Scheduler {
await notificationService.sendAlert(resource, {
type: 'slow',
responseTime: result.response_time,
threshold: resource.response_time_threshold,
threshold: effectiveThreshold,
id: null,
});
}
@@ -159,6 +200,9 @@ class Scheduler {
await Promise.allSettled(tasks);
// Flush collected alerts — grouped if multiple fired in the same cycle
await notificationService.flushAlerts(pendingAlerts);
// Broadcast updated dashboard to all connected WebSocket clients
if (global.broadcastDashboardUpdate) {
global.broadcastDashboardUpdate();
@@ -166,7 +210,6 @@ class Scheduler {
}
stop() {
// Stop the scheduler (cron tasks will continue running but we flag isRunning)
this.isRunning = false;
}
}
+144
View File
@@ -0,0 +1,144 @@
import { test, describe } from 'node:test';
import assert from 'node:assert/strict';
// ---------------------------------------------------------------------------
// validateResourceBody (inline copy — avoids spinning up Express)
// ---------------------------------------------------------------------------
function validateResourceBody(body, requireUrl = true) {
const { name, url, type, check_interval, timeout, sla_target, retention_days } = body;
const isHeartbeat = (type || 'http') === 'heartbeat';
if (!name) return 'Name is required';
if (!isHeartbeat && requireUrl && !url) return 'URL is required';
if (!isHeartbeat && url) {
try { new URL(url); } catch (e) { return 'Invalid URL format'; }
}
if (check_interval && (isNaN(check_interval) || check_interval < 10000)) return 'Check interval must be at least 10000ms';
if (timeout && (isNaN(timeout) || timeout < 1000)) return 'Timeout must be at least 1000ms';
if (sla_target !== undefined && sla_target !== null && (isNaN(sla_target) || sla_target < 0 || sla_target > 100)) return 'SLA target must be between 0 and 100';
if (retention_days && (isNaN(retention_days) || retention_days < 1 || retention_days > 365)) return 'Retention days must be between 1 and 365';
return null;
}
// ---------------------------------------------------------------------------
// formatDuration (inline copy)
// ---------------------------------------------------------------------------
function formatDuration(ms) {
if (!ms || ms < 0) return '0m';
const minutes = Math.floor(ms / 60000);
const hours = Math.floor(minutes / 60);
const mins = minutes % 60;
if (hours > 0) return `${hours}h ${mins}m`;
return `${mins}m`;
}
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
describe('validateResourceBody', () => {
test('rejects missing name', () => {
const result = validateResourceBody({ url: 'https://example.com' });
assert.equal(result, 'Name is required');
});
test('rejects missing URL for non-heartbeat', () => {
const result = validateResourceBody({ name: 'Test' });
assert.equal(result, 'URL is required');
});
test('rejects invalid URL', () => {
const result = validateResourceBody({ name: 'Test', url: 'not-a-url' });
assert.equal(result, 'Invalid URL format');
});
test('accepts valid http resource', () => {
const result = validateResourceBody({ name: 'Test', url: 'https://example.com' });
assert.equal(result, null);
});
test('accepts heartbeat without URL', () => {
const result = validateResourceBody({ name: 'My Job', type: 'heartbeat' });
assert.equal(result, null);
});
test('rejects check_interval below 10000', () => {
const result = validateResourceBody({ name: 'Test', url: 'https://example.com', check_interval: 5000 });
assert.equal(result, 'Check interval must be at least 10000ms');
});
test('rejects timeout below 1000', () => {
const result = validateResourceBody({ name: 'Test', url: 'https://example.com', timeout: 500 });
assert.equal(result, 'Timeout must be at least 1000ms');
});
test('rejects sla_target above 100', () => {
const result = validateResourceBody({ name: 'Test', url: 'https://example.com', sla_target: 101 });
assert.equal(result, 'SLA target must be between 0 and 100');
});
test('rejects retention_days above 365', () => {
const result = validateResourceBody({ name: 'Test', url: 'https://example.com', retention_days: 400 });
assert.equal(result, 'Retention days must be between 1 and 365');
});
});
describe('formatDuration', () => {
test('returns 0m for null input', () => {
assert.equal(formatDuration(null), '0m');
});
test('returns 0m for negative input', () => {
assert.equal(formatDuration(-100), '0m');
});
test('formats minutes only', () => {
assert.equal(formatDuration(5 * 60000), '5m');
});
test('formats hours and minutes', () => {
assert.equal(formatDuration(90 * 60000), '1h 30m');
});
test('formats exactly 1 hour', () => {
assert.equal(formatDuration(60 * 60000), '1h 0m');
});
test('formats 0 minutes as 0m', () => {
assert.equal(formatDuration(0), '0m');
});
});
describe('rate limiter logic', () => {
// Inline the rate-limit counter logic
function makeRateLimiter(maxReqs, windowMs = 60000) {
const map = new Map();
return (ip, path) => {
const key = `${path}:${ip}`;
const now = Date.now();
const e = map.get(key);
if (!e || now - e.start > windowMs) { map.set(key, { start: now, count: 1 }); return true; }
e.count++;
return e.count <= maxReqs;
};
}
test('allows requests within limit', () => {
const check = makeRateLimiter(3);
assert.ok(check('127.0.0.1', '/api/heartbeat/tok'));
assert.ok(check('127.0.0.1', '/api/heartbeat/tok'));
assert.ok(check('127.0.0.1', '/api/heartbeat/tok'));
});
test('blocks requests exceeding limit', () => {
const check = makeRateLimiter(2);
check('127.0.0.1', '/test');
check('127.0.0.1', '/test');
const allowed = check('127.0.0.1', '/test');
assert.equal(allowed, false);
});
test('different IPs are tracked separately', () => {
const check = makeRateLimiter(1);
assert.ok(check('1.1.1.1', '/test'));
assert.ok(check('2.2.2.2', '/test'));
});
});