Feature: Auto-create incidents after 10 failed checks with rich descriptions
- Add description and failed_check_count fields to incidents table - Update handleIncident() to use configurable threshold (default 10 checks) - Auto-generate incident description with: - Number of consecutive failures - Last error message/reason - First failure timestamp - Add API endpoints: GET/POST /api/settings/incident-threshold - Add UI settings field to configure incident failure threshold - Server: Get threshold from database settings (persistent across restarts) - Default threshold: 10 consecutive failed checks per resource
This commit is contained in:
1 parent
6b00530a89
commit
5330c8932f
10 files changed
+106
-29
No files matched your search
@@ -313,7 +313,10 @@ class MonitorService {
|
||||
|
||||
handleIncident(resourceId, isDown) {
|
||||
const resource = db.prepare('SELECT * FROM resources WHERE id = ?').get(resourceId);
|
||||
const threshold = resource?.consecutive_failures_threshold || 1;
|
||||
|
||||
// Get incident failure threshold from settings (default 10)
|
||||
const setting = db.prepare("SELECT value FROM settings WHERE key = 'incident_failure_threshold'").get();
|
||||
const threshold = parseInt(setting?.value || '10');
|
||||
|
||||
const activeIncident = db.prepare(`
|
||||
SELECT * FROM incidents
|
||||
@@ -323,7 +326,7 @@ class MonitorService {
|
||||
if (isDown) {
|
||||
// Check consecutive failures
|
||||
const recentChecks = db.prepare(`
|
||||
SELECT status FROM checks
|
||||
SELECT status, error_message, checked_at FROM checks
|
||||
WHERE resource_id = ?
|
||||
ORDER BY checked_at DESC
|
||||
LIMIT ?
|
||||
@@ -334,14 +337,22 @@ class MonitorService {
|
||||
|
||||
// Only trigger incident if we've hit the threshold
|
||||
if (consecutiveFailures >= threshold && !activeIncident) {
|
||||
// Build description with failure reason and first failed check timestamp
|
||||
const lastCheck = recentChecks[0];
|
||||
const firstFailedCheck = recentChecks[recentChecks.length - 1];
|
||||
|
||||
const reason = lastCheck?.error_message || 'Service returned down status';
|
||||
const description = `Resource down after ${consecutiveFailures} consecutive failed checks.\nReason: ${reason}\nFirst failure: ${firstFailedCheck?.checked_at}`;
|
||||
|
||||
const stmt = db.prepare(`
|
||||
INSERT INTO incidents (resource_id) VALUES (?)
|
||||
INSERT INTO incidents (resource_id, description, failed_check_count) VALUES (?, ?, ?)
|
||||
`);
|
||||
const result = stmt.run(resourceId);
|
||||
const result = stmt.run(resourceId, description, consecutiveFailures);
|
||||
return {
|
||||
type: 'started',
|
||||
incident: true,
|
||||
consecutiveFailures
|
||||
consecutiveFailures,
|
||||
description
|
||||
};
|
||||
}
|
||||
} else if (!isDown && activeIncident) {
|
||||
|
||||
Reference in new issue
Block a user