Feature: Auto-create incidents after 10 failed checks with rich descriptions

- Add description and failed_check_count fields to incidents table
- Update handleIncident() to use configurable threshold (default 10 checks)
- Auto-generate incident description with:
  - Number of consecutive failures
  - Last error message/reason
  - First failure timestamp
- Add API endpoints: GET/POST /api/settings/incident-threshold
- Add UI settings field to configure incident failure threshold
- Server: Get threshold from database settings (persistent across restarts)
- Default threshold: 10 consecutive failed checks per resource

server/database.js

@@ -245,9 +245,30 @@ try {
   }
 }
 
-// Settings for data retention
+// Add description column to incidents for incident summaries
+try {
+  db.prepare("ALTER TABLE incidents ADD COLUMN description TEXT").run();
+  console.log('[DB] Added description column to incidents');
+} catch (err) {
+  if (err.message.includes('duplicate')) {
+    console.log('[DB] description column already exists');
+  }
+}
+
+// Add failed_check_count to incidents to track consecutive failures at creation time
+try {
+  db.prepare("ALTER TABLE incidents ADD COLUMN failed_check_count INTEGER DEFAULT 0").run();
+  console.log('[DB] Added failed_check_count column to incidents');
+} catch (err) {
+  if (err.message.includes('duplicate')) {
+    console.log('[DB] failed_check_count column already exists');
+  }
+}
+
+// Settings for data retention and incident thresholds
 try {
   db.prepare("INSERT OR IGNORE INTO settings (key, value) VALUES ('retention_days', '30')").run();
+  db.prepare("INSERT OR IGNORE INTO settings (key, value) VALUES ('incident_failure_threshold', '10')").run();
 } catch (err) {}
 
 // Helpful indexes for time-range queries

server/index.js

@@ -521,6 +521,36 @@ app.post('/api/settings/retention', (req, res) => {
   }
 });
 
+// Get incident failure threshold
+app.get('/api/settings/incident-threshold', (req, res) => {
+  try {
+    const setting = db.prepare("SELECT value FROM settings WHERE key = 'incident_failure_threshold'").get();
+    res.json({ incident_failure_threshold: parseInt(setting?.value || '10') });
+  } catch (error) {
+    res.status(500).json({ error: 'Failed to get incident threshold' });
+  }
+});
+
+// Update incident failure threshold
+app.post('/api/settings/incident-threshold', (req, res) => {
+  const { incident_failure_threshold } = req.body;
+  
+  if (!incident_failure_threshold || incident_failure_threshold < 1 || incident_failure_threshold > 100) {
+    return res.status(400).json({ error: 'Threshold must be between 1 and 100' });
+  }
+
+  try {
+    db.prepare(`
+      INSERT INTO settings (key, value) VALUES ('incident_failure_threshold', ?)
+      ON CONFLICT(key) DO UPDATE SET value = ?
+    `).run(String(incident_failure_threshold), String(incident_failure_threshold));
+    
+    res.json({ message: 'Incident threshold updated', incident_failure_threshold });
+  } catch (error) {
+    res.status(500).json({ error: 'Failed to update incident threshold' });
+  }
+});
+
 // Get archived checks for a resource
 app.get('/api/resources/:id/archived', (req, res) => {
   const { id } = req.params;

server/monitorService.js

@@ -313,7 +313,10 @@ class MonitorService {
 
   handleIncident(resourceId, isDown) {
     const resource = db.prepare('SELECT * FROM resources WHERE id = ?').get(resourceId);
-    const threshold = resource?.consecutive_failures_threshold || 1;
+    
+    // Get incident failure threshold from settings (default 10)
+    const setting = db.prepare("SELECT value FROM settings WHERE key = 'incident_failure_threshold'").get();
+    const threshold = parseInt(setting?.value || '10');
     
     const activeIncident = db.prepare(`
       SELECT * FROM incidents 
@@ -323,7 +326,7 @@ class MonitorService {
     if (isDown) {
       // Check consecutive failures
       const recentChecks = db.prepare(`
-        SELECT status FROM checks
+        SELECT status, error_message, checked_at FROM checks
         WHERE resource_id = ?
         ORDER BY checked_at DESC
         LIMIT ?
@@ -334,14 +337,22 @@ class MonitorService {
 
       // Only trigger incident if we've hit the threshold
       if (consecutiveFailures >= threshold && !activeIncident) {
+        // Build description with failure reason and first failed check timestamp
+        const lastCheck = recentChecks[0];
+        const firstFailedCheck = recentChecks[recentChecks.length - 1];
+        
+        const reason = lastCheck?.error_message || 'Service returned down status';
+        const description = `Resource down after ${consecutiveFailures} consecutive failed checks.\nReason: ${reason}\nFirst failure: ${firstFailedCheck?.checked_at}`;
+
         const stmt = db.prepare(`
-          INSERT INTO incidents (resource_id) VALUES (?)
+          INSERT INTO incidents (resource_id, description, failed_check_count) VALUES (?, ?, ?)
         `);
-        const result = stmt.run(resourceId);
+        const result = stmt.run(resourceId, description, consecutiveFailures);
         return { 
           type: 'started', 
           incident: true,
-          consecutiveFailures
+          consecutiveFailures,
+          description
         };
       }
     } else if (!isDown && activeIncident) {