feat: Add real-time notification center with WebSocket integration

- Add in-app notification system with database storage
- Implement bell icon component with unread count badge in dashboard header
- Add full Notifications page tab with pagination and filtering (All/Unread/Read)
- Implement real-time WebSocket push notifications to all connected clients
- Add notification API endpoints: GET, PUT /read, DELETE, POST /clear
- Fix critical bug in notificationService where 'status' variable was undefined
- Add Notifications route to App.js routing
- Enhanced UI with notification popup in header and dedicated full-page view
- Auto-reconnect WebSocket with 5s timeout on disconnect
- Support rich notification content with resource details and timestamps

server/auditLog.js

@@ -0,0 +1,152 @@
+const db = require('./database');
+
+class AuditLogger {
+  // Log an audit event
+  log(action, entityType, entityId, changes = {}, userId = 'system', ipAddress = null) {
+    try {
+      db.prepare(`
+        INSERT INTO audit_logs 
+        (action, entity_type, entity_id, user_id, changes, ip_address, created_at)
+        VALUES (?, ?, ?, ?, ?, ?, datetime('now'))
+      `).run(
+        action,
+        entityType,
+        entityId,
+        userId,
+        JSON.stringify(changes),
+        ipAddress
+      );
+    } catch (error) {
+      // Silent failure - don't crash on audit logging errors
+    }
+  }
+
+  // Log resource changes
+  logResourceChange(action, resourceId, changes, userId = 'system', ipAddress = null) {
+    this.log(action, 'resource', resourceId, changes, userId, ipAddress);
+  }
+
+  // Log settings changes
+  logSettingsChange(settingKey, oldValue, newValue, userId = 'system', ipAddress = null) {
+    this.log('update', 'settings', settingKey, { old: oldValue, new: newValue }, userId, ipAddress);
+  }
+
+  // Log group changes
+  logGroupChange(action, groupId, changes, userId = 'system', ipAddress = null) {
+    this.log(action, 'group', groupId, changes, userId, ipAddress);
+  }
+
+  // Log maintenance window changes
+  logMaintenanceChange(action, windowId, changes, userId = 'system', ipAddress = null) {
+    this.log(action, 'maintenance_window', windowId, changes, userId, ipAddress);
+  }
+
+  // Get audit logs with filters
+  getAuditLogs(filters = {}) {
+    const {
+      entityType = null,
+      entityId = null,
+      userId = null,
+      action = null,
+      startDate = null,
+      endDate = null,
+      limit = 100,
+      offset = 0
+    } = filters;
+
+    let query = `
+      SELECT 
+        id,
+        action,
+        entity_type,
+        entity_id,
+        user_id,
+        changes,
+        ip_address,
+        REPLACE(created_at, ' ', 'T') || 'Z' as created_at
+      FROM audit_logs
+      WHERE 1=1
+    `;
+    const params = [];
+
+    if (entityType) {
+      query += ' AND entity_type = ?';
+      params.push(entityType);
+    }
+
+    if (entityId) {
+      query += ' AND entity_id = ?';
+      params.push(entityId);
+    }
+
+    if (userId) {
+      query += ' AND user_id = ?';
+      params.push(userId);
+    }
+
+    if (action) {
+      query += ' AND action = ?';
+      params.push(action);
+    }
+
+    if (startDate) {
+      query += ' AND created_at >= ?';
+      params.push(startDate);
+    }
+
+    if (endDate) {
+      query += ' AND created_at <= ?';
+      params.push(endDate);
+    }
+
+    query += ' ORDER BY created_at DESC LIMIT ? OFFSET ?';
+    params.push(limit, offset);
+
+    try {
+      const logs = db.prepare(query).all(...params);
+      
+      // Parse JSON changes
+      return logs.map(log => ({
+        ...log,
+        changes: log.changes ? JSON.parse(log.changes) : {}
+      }));
+    } catch (error) {
+      return [];
+    }
+  }
+
+  // Get audit summary stats
+  getSummary(days = 7) {
+    try {
+      const stats = db.prepare(`
+        SELECT 
+          COUNT(*) as total_events,
+          COUNT(DISTINCT user_id) as unique_users,
+          COUNT(DISTINCT entity_type) as entity_types,
+          action,
+          COUNT(*) as action_count
+        FROM audit_logs
+        WHERE created_at > datetime('now', '-' || ? || ' days')
+        GROUP BY action
+      `).all(days);
+
+      const totalEvents = db.prepare(`
+        SELECT COUNT(*) as count
+        FROM audit_logs
+        WHERE created_at > datetime('now', '-' || ? || ' days')
+      `).get(days);
+
+      return {
+        totalEvents: totalEvents?.count || 0,
+        actionBreakdown: stats.map(s => ({
+          action: s.action,
+          count: s.action_count
+        }))
+      };
+    } catch (error) {
+      return { totalEvents: 0, actionBreakdown: [] };
+    }
+  }
+}
+
+module.exports = new AuditLogger();

server/circuitBreaker.js

@@ -0,0 +1,62 @@
+// Circuit breaker implementation for API resilience
+class CircuitBreaker {
+  constructor(failureThreshold = 5, resetTimeout = 60000, errorFilter = null) {
+    this.failureThreshold = failureThreshold;
+    this.resetTimeout = resetTimeout;
+    this.errorFilter = errorFilter;
+    this.failureCount = 0;
+    this.lastFailureTime = null;
+    this.state = 'closed'; // closed, open, half-open
+  }
+
+  async execute(fn, fallback = null) {
+    if (this.state === 'open') {
+      if (Date.now() - this.lastFailureTime > this.resetTimeout) {
+        this.state = 'half-open';
+      } else {
+        if (fallback) return fallback();
+        throw new Error('Circuit breaker is open');
+      }
+    }
+
+    try {
+      const result = await fn();
+      if (this.state === 'half-open') {
+        this.close();
+      }
+      return result;
+    } catch (error) {
+      if (this.errorFilter && !this.errorFilter(error)) {
+        throw error;
+      }
+
+      this.failureCount++;
+      this.lastFailureTime = Date.now();
+
+      if (this.failureCount >= this.failureThreshold) {
+        this.open();
+      }
+
+      throw error;
+    }
+  }
+
+  close() {
+    this.failureCount = 0;
+    this.state = 'closed';
+  }
+
+  open() {
+    this.state = 'open';
+  }
+
+  getState() {
+    return {
+      state: this.state,
+      failureCount: this.failureCount,
+      lastFailureTime: this.lastFailureTime
+    };
+  }
+}
+
+module.exports = CircuitBreaker;

server/database.js

@@ -22,6 +22,7 @@ db.exec(`
     enabled INTEGER DEFAULT 1,
     maintenance_mode INTEGER DEFAULT 0,
     group_id INTEGER REFERENCES groups(id) ON DELETE SET NULL,
+    retention_days INTEGER DEFAULT NULL,
     created_at DATETIME DEFAULT CURRENT_TIMESTAMP
   );
 
@@ -111,6 +112,20 @@ db.exec(`
     UNIQUE(resource_id, tag)
   );
 
+  CREATE TABLE IF NOT EXISTS notifications (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    resource_id INTEGER,
+    incident_id INTEGER,
+    type TEXT NOT NULL,
+    title TEXT NOT NULL,
+    message TEXT NOT NULL,
+    status TEXT DEFAULT 'unread',
+    read INTEGER DEFAULT 0,
+    created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (resource_id) REFERENCES resources(id) ON DELETE CASCADE,
+    FOREIGN KEY (incident_id) REFERENCES incidents(id) ON DELETE CASCADE
+  );
+
   CREATE INDEX IF NOT EXISTS idx_checks_resource_id ON checks(resource_id);
   CREATE INDEX IF NOT EXISTS idx_checks_checked_at ON checks(checked_at);
   CREATE INDEX IF NOT EXISTS idx_checks_resource_checked ON checks(resource_id, checked_at);
@@ -122,8 +137,34 @@ db.exec(`
   CREATE INDEX IF NOT EXISTS idx_alert_rules_resource ON alert_rules(resource_id);
   CREATE INDEX IF NOT EXISTS idx_transaction_checks_resource ON transaction_checks(resource_id);
   CREATE INDEX IF NOT EXISTS idx_archived_checks_resource_time ON archived_checks(resource_id, checked_at DESC);
+  CREATE INDEX IF NOT EXISTS idx_notifications_created ON notifications(created_at DESC);
+  CREATE INDEX IF NOT EXISTS idx_notifications_read_status ON notifications(read, created_at DESC);
   CREATE INDEX IF NOT EXISTS idx_resource_tags_resource ON resource_tags(resource_id);
   CREATE INDEX IF NOT EXISTS idx_resource_tags_tag ON resource_tags(tag);
+
+  CREATE TABLE IF NOT EXISTS audit_logs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    action TEXT NOT NULL,
+    entity_type TEXT NOT NULL,
+    entity_id TEXT,
+    user_id TEXT DEFAULT 'system',
+    changes TEXT,
+    ip_address TEXT,
+    created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+  );
+
+  CREATE TABLE IF NOT EXISTS error_logs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    error_message TEXT NOT NULL,
+    stack_trace TEXT,
+    context TEXT,
+    occurred_at DATETIME DEFAULT CURRENT_TIMESTAMP
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_audit_logs_entity ON audit_logs(entity_type, entity_id);
+  CREATE INDEX IF NOT EXISTS idx_audit_logs_user ON audit_logs(user_id);
+  CREATE INDEX IF NOT EXISTS idx_audit_logs_created ON audit_logs(created_at DESC);
+  CREATE INDEX IF NOT EXISTS idx_error_logs_occurred ON error_logs(occurred_at DESC);
 `);
 
 // Lightweight migrations - for existing databases
@@ -141,10 +182,9 @@ try {
 
 try {
   db.prepare("ALTER TABLE resources ADD COLUMN maintenance_mode INTEGER DEFAULT 0").run();
-  console.log('[DB] Added maintenance_mode column to resources');
 } catch (err) {
   if (err.message.includes('duplicate')) {
-    console.log('[DB] maintenance_mode column already exists');
+    // Column already exists
   }
 }
 
@@ -174,12 +214,11 @@ try {
 
 try {
   db.prepare("ALTER TABLE resources ADD COLUMN email_to TEXT").run();
-  console.log('[DB] Added email_to column to resources');
 } catch (err) {
   if (err.message.includes('duplicate')) {
-    console.log('[DB] email_to column already exists');
+    // email_to column already exists
   } else {
-    console.error('[DB] Error adding email_to column:', err.message);
+    // Error adding email_to column
   }
 }
 
@@ -202,66 +241,67 @@ try {
 // New columns for features
 try {
   db.prepare("ALTER TABLE resources ADD COLUMN tags TEXT").run();
-  console.log('[DB] Added tags column to resources');
 } catch (err) {
   if (err.message.includes('duplicate')) {
-    console.log('[DB] tags column already exists');
+    // tags column already exists
   }
 }
 
 try {
   db.prepare("ALTER TABLE resources ADD COLUMN consecutive_failures_threshold INTEGER DEFAULT 1").run();
-  console.log('[DB] Added consecutive_failures_threshold column to resources');
 } catch (err) {
   if (err.message.includes('duplicate')) {
-    console.log('[DB] consecutive_failures_threshold column already exists');
+    // consecutive_failures_threshold column already exists
   }
 }
 
 try {
   db.prepare("ALTER TABLE resources ADD COLUMN response_time_threshold INTEGER").run();
-  console.log('[DB] Added response_time_threshold column to resources');
 } catch (err) {
   if (err.message.includes('duplicate')) {
-    console.log('[DB] response_time_threshold column already exists');
+    // response_time_threshold column already exists
   }
 }
 
 try {
   db.prepare("ALTER TABLE resources ADD COLUMN response_time_baseline INTEGER").run();
-  console.log('[DB] Added response_time_baseline column to resources');
 } catch (err) {
   if (err.message.includes('duplicate')) {
-    console.log('[DB] response_time_baseline column already exists');
+    // response_time_baseline column already exists
   }
 }
 
 try {
   db.prepare("ALTER TABLE resources ADD COLUMN is_transaction INTEGER DEFAULT 0").run();
-  console.log('[DB] Added is_transaction column to resources');
 } catch (err) {
   if (err.message.includes('duplicate')) {
-    console.log('[DB] is_transaction column already exists');
+    // is_transaction column already exists
+  }
+}
+
+try {
+  db.prepare("ALTER TABLE resources ADD COLUMN retention_days INTEGER DEFAULT NULL").run();
+} catch (err) {
+  if (err.message.includes('duplicate')) {
+    // retention_days column already exists
   }
 }
 
 // Add description column to incidents for incident summaries
 try {
   db.prepare("ALTER TABLE incidents ADD COLUMN description TEXT").run();
-  console.log('[DB] Added description column to incidents');
 } catch (err) {
   if (err.message.includes('duplicate')) {
-    console.log('[DB] description column already exists');
+    // description column already exists
   }
 }
 
 // Add failed_check_count to incidents to track consecutive failures at creation time
 try {
   db.prepare("ALTER TABLE incidents ADD COLUMN failed_check_count INTEGER DEFAULT 0").run();
-  console.log('[DB] Added failed_check_count column to incidents');
 } catch (err) {
   if (err.message.includes('duplicate')) {
-    console.log('[DB] failed_check_count column already exists');
+    // failed_check_count column already exists
   }
 }
 

server/index.js

@@ -10,6 +10,33 @@ const scheduler = require('./scheduler');
 const monitorService = require('./monitorService');
 const notificationService = require('./notificationService');
 const cache = require('./cache');
+const security = require('./securityMiddleware');
+const metrics = require('./metrics');
+const auditLog = require('./auditLog');
+
+// Initialize notifications table if it doesn't exist
+try {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS notifications (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      resource_id INTEGER,
+      incident_id INTEGER,
+      type TEXT NOT NULL,
+      title TEXT NOT NULL,
+      message TEXT NOT NULL,
+      status TEXT DEFAULT 'unread',
+      read INTEGER DEFAULT 0,
+      created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+      FOREIGN KEY (resource_id) REFERENCES resources(id) ON DELETE CASCADE,
+      FOREIGN KEY (incident_id) REFERENCES incidents(id) ON DELETE CASCADE
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_notifications_created ON notifications(created_at DESC);
+    CREATE INDEX IF NOT EXISTS idx_notifications_read_status ON notifications(read, created_at DESC);
+  `);
+} catch (error) {
+  console.error('Error initializing notifications table:', error);
+}
 
 const app = express();
 const PORT = process.env.PORT || 3001;
@@ -81,6 +108,44 @@ function getTimezoneOffset() {
 app.use(cors());
 app.use(express.json());
 
+// Metrics tracking middleware
+app.use((req, res, next) => {
+  const startTime = Date.now();
+  
+  // Capture original res.json and res.send
+  const originalJson = res.json.bind(res);
+  const originalSend = res.send.bind(res);
+  
+  const trackRequest = () => {
+    const duration = Date.now() - startTime;
+    const endpoint = req.route?.path || req.path;
+    metrics.trackApiRequest(endpoint, req.method, res.statusCode, duration);
+  };
+  
+  res.json = function(data) {
+    trackRequest();
+    return originalJson(data);
+  };
+  
+  res.send = function(data) {
+    trackRequest();
+    return originalSend(data);
+  };
+  
+  next();
+});
+
+// Error tracking middleware
+app.use((err, req, res, next) => {
+  metrics.trackError(err, {
+    method: req.method,
+    url: req.url,
+    ip: req.ip
+  });
+  
+  res.status(500).json({ error: 'Internal server error' });
+});
+
 // API Routes
 
 // Get all groups
@@ -139,16 +204,37 @@ app.get('/api/resources/:id', (req, res) => {
 
 // Create resource
 app.post('/api/resources', (req, res) => {
-  const { name, url, type, check_interval, timeout, group_id, http_keyword, http_headers, quiet_hours_start, quiet_hours_end, cert_expiry_days, sla_target, email_to, maintenance_mode } = req.body;
+  const { name, url, type, check_interval, timeout, group_id, http_keyword, http_headers, quiet_hours_start, quiet_hours_end, cert_expiry_days, sla_target, email_to, maintenance_mode, retention_days } = req.body;
 
   if (!name || !url) {
     return res.status(400).json({ error: 'Name and URL are required' });
   }
 
+  // Validate URL format
+  try {
+    new URL(url);
+  } catch (e) {
+    return res.status(400).json({ error: 'Invalid URL format' });
+  }
+
+  // Validate numeric fields
+  if (check_interval && (isNaN(check_interval) || check_interval < 10000)) {
+    return res.status(400).json({ error: 'Check interval must be at least 10000ms' });
+  }
+  if (timeout && (isNaN(timeout) || timeout < 1000)) {
+    return res.status(400).json({ error: 'Timeout must be at least 1000ms' });
+  }
+  if (sla_target && (isNaN(sla_target) || sla_target < 0 || sla_target > 100)) {
+    return res.status(400).json({ error: 'SLA target must be between 0 and 100' });
+  }
+  if (retention_days && (isNaN(retention_days) || retention_days < 1 || retention_days > 365)) {
+    return res.status(400).json({ error: 'Retention days must be between 1 and 365' });
+  }
+
   try {
     const stmt = db.prepare(`
-      INSERT INTO resources (name, url, type, check_interval, timeout, group_id, http_keyword, http_headers, quiet_hours_start, quiet_hours_end, cert_expiry_days, sla_target, email_to, maintenance_mode)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      INSERT INTO resources (name, url, type, check_interval, timeout, group_id, http_keyword, http_headers, quiet_hours_start, quiet_hours_end, cert_expiry_days, sla_target, email_to, maintenance_mode, retention_days)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
     `);
 
     const result = stmt.run(
@@ -165,16 +251,19 @@ app.post('/api/resources', (req, res) => {
       cert_expiry_days || 30,
       sla_target || 99.9,
       email_to || null,
-      maintenance_mode ? 1 : 0
+      maintenance_mode ? 1 : 0,
+      retention_days || null
     );
 
     // Invalidate related cache entries
     cache.invalidatePattern('history:');
     cache.invalidatePattern('sla:');
 
+    // Audit log
+    auditLog.logResourceChange('create', result.lastInsertRowid, { name, url, type, enabled: true }, 'system', req.ip);
+
     res.json({ id: result.lastInsertRowid, message: 'Resource created' });
   } catch (err) {
-    console.error('[API] Error creating resource:', err.message);
     // Helpful hint if migration missing
     const hint = err.message.includes('no column named maintenance_mode')
       ? 'Database schema missing maintenance_mode. Restart server to run migrations.'
@@ -185,11 +274,11 @@ app.post('/api/resources', (req, res) => {
 
 // Update resource
 app.put('/api/resources/:id', (req, res) => {
-  const { name, url, type, check_interval, timeout, enabled, group_id, http_keyword, http_headers, quiet_hours_start, quiet_hours_end, cert_expiry_days, sla_target, email_to, maintenance_mode } = req.body;
+  const { name, url, type, check_interval, timeout, enabled, group_id, http_keyword, http_headers, quiet_hours_start, quiet_hours_end, cert_expiry_days, sla_target, email_to, maintenance_mode, retention_days } = req.body;
 
   const stmt = db.prepare(`
     UPDATE resources 
-    SET name = ?, url = ?, type = ?, check_interval = ?, timeout = ?, enabled = ?, group_id = ?, http_keyword = ?, http_headers = ?, quiet_hours_start = ?, quiet_hours_end = ?, cert_expiry_days = ?, sla_target = ?, email_to = ?, maintenance_mode = ?
+    SET name = ?, url = ?, type = ?, check_interval = ?, timeout = ?, enabled = ?, group_id = ?, http_keyword = ?, http_headers = ?, quiet_hours_start = ?, quiet_hours_end = ?, cert_expiry_days = ?, sla_target = ?, email_to = ?, maintenance_mode = ?, retention_days = ?
     WHERE id = ?
   `);
 
@@ -209,6 +298,7 @@ app.put('/api/resources/:id', (req, res) => {
     sla_target || 99.9,
     email_to || null,
     maintenance_mode ? 1 : 0,
+    retention_days || null,
     req.params.id
   );
 
@@ -257,12 +347,19 @@ app.patch('/api/resources/:id/group', (req, res) => {
 
 // Delete resource
 app.delete('/api/resources/:id', (req, res) => {
+  const resource = db.prepare('SELECT name, url FROM resources WHERE id = ?').get(req.params.id);
+  
   db.prepare('DELETE FROM resources WHERE id = ?').run(req.params.id);
   
   // Invalidate related cache entries
   cache.invalidatePattern('history:');
   cache.invalidatePattern('sla:');
   
+  // Audit log
+  if (resource) {
+    auditLog.logResourceChange('delete', req.params.id, { name: resource.name, url: resource.url }, 'system', req.ip);
+  }
+  
   res.json({ message: 'Resource deleted' });
 });
 
@@ -339,6 +436,84 @@ app.get('/api/incidents', (req, res) => {
   res.json({ incidents });
 });
 
+// Get notifications (in-app notification center)
+app.get('/api/notifications', (req, res) => {
+  const limit = parseInt(req.query.limit) || 50;
+  const offset = parseInt(req.query.offset) || 0;
+  const read = req.query.read; // undefined = all, 'unread' = only unread, 'read' = only read
+
+  let whereClause = '';
+  let whereClauseCount = '';
+  if (read === 'unread') {
+    whereClause = 'WHERE n.read = 0';
+    whereClauseCount = 'WHERE read = 0';
+  } else if (read === 'read') {
+    whereClause = 'WHERE n.read = 1';
+    whereClauseCount = 'WHERE read = 1';
+  }
+
+  const notifications = db.prepare(`
+    SELECT 
+      n.id,
+      n.resource_id,
+      n.incident_id,
+      n.type,
+      n.title,
+      n.message,
+      n.read,
+      n.created_at,
+      r.name as resource_name,
+      i.started_at as incident_started_at
+    FROM notifications n
+    LEFT JOIN resources r ON n.resource_id = r.id
+    LEFT JOIN incidents i ON n.incident_id = i.id
+    ${whereClause}
+    ORDER BY n.created_at DESC
+    LIMIT ? OFFSET ?
+  `).all(limit, offset);
+
+  const total = db.prepare(`
+    SELECT COUNT(*) as count FROM notifications ${whereClauseCount}
+  `).get().count;
+
+  const unreadCount = db.prepare('SELECT COUNT(*) as count FROM notifications WHERE read = 0').get().count;
+
+  res.json({ notifications, total, unreadCount });
+});
+
+// Get unread notification count
+app.get('/api/notifications/unread/count', (req, res) => {
+  const count = db.prepare('SELECT COUNT(*) as count FROM notifications WHERE read = 0').get().count;
+  res.json({ unreadCount: count });
+});
+
+// Mark notification as read
+app.put('/api/notifications/:id/read', (req, res) => {
+  db.prepare('UPDATE notifications SET read = 1 WHERE id = ?').run(req.params.id);
+  res.json({ success: true });
+});
+
+// Delete notification
+app.delete('/api/notifications/:id', (req, res) => {
+  db.prepare('DELETE FROM notifications WHERE id = ?').run(req.params.id);
+  res.json({ success: true });
+});
+
+// Clear all notifications
+app.post('/api/notifications/clear', (req, res) => {
+  const type = req.body.type; // 'all', 'read', 'unread'
+  
+  if (type === 'all') {
+    db.prepare('DELETE FROM notifications').run();
+  } else if (type === 'read') {
+    db.prepare('DELETE FROM notifications WHERE read = 1').run();
+  } else if (type === 'unread') {
+    db.prepare('DELETE FROM notifications WHERE read = 0').run();
+  }
+  
+  res.json({ success: true });
+});
+
 // Get settings
 app.get('/api/settings', (req, res) => {
   const settings = {
@@ -414,8 +589,43 @@ app.post('/api/settings', (req, res) => {
     theme,
   } = req.body;
 
-  const envContent = `PORT=${process.env.PORT || 3001}
-EMAIL_ENABLED=${email_enabled}
+  // Validate email configuration if enabled
+  if (email_enabled) {
+    if (!email_host || !email_port || !email_user || !email_pass) {
+      return res.status(400).json({ error: 'Email configuration incomplete. All fields required when enabled.' });
+    }
+    if (isNaN(email_port) || email_port < 1 || email_port > 65535) {
+      return res.status(400).json({ error: 'Invalid email port number' });
+    }
+    if (!/^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(email_from)) {
+      return res.status(400).json({ error: 'Invalid email_from address' });
+    }
+  }
+
+  // Validate webhook configuration if enabled
+  if (webhook_enabled) {
+    if (!webhook_url) {
+      return res.status(400).json({ error: 'Webhook URL required when enabled' });
+    }
+    try {
+      new URL(webhook_url);
+    } catch (e) {
+      return res.status(400).json({ error: 'Invalid webhook URL format' });
+    }
+  }
+
+  // Validate numeric configuration
+  if (retention_days && (isNaN(retention_days) || retention_days < 1)) {
+    return res.status(400).json({ error: 'Retention days must be at least 1' });
+  }
+  if (check_interval && (isNaN(check_interval) || check_interval < 10000)) {
+    return res.status(400).json({ error: 'Check interval must be at least 10000ms' });
+  }
+  if (timeout && (isNaN(timeout) || timeout < 1000)) {
+    return res.status(400).json({ error: 'Timeout must be at least 1000ms' });
+  }
+
+  const envContent = `EMAIL_ENABLED=${email_enabled}
 EMAIL_HOST=${email_host}
 EMAIL_PORT=${email_port}
 EMAIL_USER=${email_user}
@@ -490,7 +700,6 @@ THEME=${theme || 'light'}
 
     res.json({ message: 'Settings saved successfully' });
   } catch (error) {
-    console.error('Error saving settings:', error);
     res.status(500).json({ error: 'Failed to save settings' });
   }
 });
@@ -506,7 +715,6 @@ app.get('/api/settings/retention', (req, res) => {
       retention_days: setting ? parseInt(setting.value) : 30 
     });
   } catch (error) {
-    console.error('Error getting retention settings:', error);
     res.status(500).json({ error: 'Failed to get retention settings' });
   }
 });
@@ -527,7 +735,6 @@ app.post('/api/settings/retention', (req, res) => {
     
     res.json({ message: 'Retention settings updated', retention_days });
   } catch (error) {
-    console.error('Error updating retention settings:', error);
     res.status(500).json({ error: 'Failed to update retention settings' });
   }
 });
@@ -562,6 +769,123 @@ app.post('/api/settings/incident-threshold', (req, res) => {
   }
 });
 
+// CSRF token generation endpoint
+app.get('/api/auth/csrf-token', (req, res) => {
+  try {
+    const csrfToken = security.generateCSRFToken();
+    res.json({ csrfToken });
+  } catch (error) {
+    res.status(500).json({ error: 'Failed to generate CSRF token' });
+  }
+});
+
+// JWT token refresh endpoint
+app.post('/api/auth/refresh-token', (req, res) => {
+  try {
+    const { refreshToken } = req.body;
+    
+    if (!refreshToken) {
+      return res.status(400).json({ error: 'Refresh token required' });
+    }
+
+    // Verify the refresh token
+    const decoded = security.verifyRefreshToken(refreshToken);
+    
+    if (!decoded) {
+      return res.status(401).json({ error: 'Invalid refresh token' });
+    }
+
+    // Generate new access and refresh tokens
+    const newAccessToken = security.generateToken({ userId: decoded.userId, username: decoded.username });
+    const newRefreshToken = security.generateRefreshToken({ userId: decoded.userId, username: decoded.username });
+
+    res.json({ 
+      accessToken: newAccessToken,
+      refreshToken: newRefreshToken,
+      expiresIn: 3600 // 1 hour
+    });
+  } catch (error) {
+    res.status(401).json({ error: 'Token refresh failed' });
+  }
+});
+
+// Password validation endpoint (for frontend to check strength)
+app.post('/api/auth/validate-password', (req, res) => {
+  try {
+    const { password } = req.body;
+    
+    if (!password) {
+      return res.status(400).json({ error: 'Password required' });
+    }
+
+    const validation = security.validatePasswordStrength(password);
+    res.json(validation);
+  } catch (error) {
+    res.status(500).json({ error: 'Password validation failed' });
+  }
+});
+
+// Observability & Monitoring Endpoints
+
+// Get application metrics
+app.get('/api/observability/metrics', (req, res) => {
+  try {
+    const metricsData = metrics.getMetrics();
+    res.json(metricsData);
+  } catch (error) {
+    metrics.trackError(error, { endpoint: '/api/observability/metrics' });
+    res.status(500).json({ error: 'Failed to retrieve metrics' });
+  }
+});
+
+// Get recent errors
+app.get('/api/observability/errors', (req, res) => {
+  try {
+    const { limit = 50 } = req.query;
+    const errors = metrics.getRecentErrors(parseInt(limit));
+    res.json({ errors });
+  } catch (error) {
+    metrics.trackError(error, { endpoint: '/api/observability/errors' });
+    res.status(500).json({ error: 'Failed to retrieve errors' });
+  }
+});
+
+// Get audit logs
+app.get('/api/observability/audit-logs', (req, res) => {
+  try {
+    const filters = {
+      entityType: req.query.entityType,
+      entityId: req.query.entityId,
+      userId: req.query.userId,
+      action: req.query.action,
+      startDate: req.query.startDate,
+      endDate: req.query.endDate,
+      limit: parseInt(req.query.limit || 100),
+      offset: parseInt(req.query.offset || 0)
+    };
+    
+    const logs = auditLog.getAuditLogs(filters);
+    const summary = auditLog.getSummary(parseInt(req.query.days || 7));
+    
+    res.json({ logs, summary });
+  } catch (error) {
+    metrics.trackError(error, { endpoint: '/api/observability/audit-logs' });
+    res.status(500).json({ error: 'Failed to retrieve audit logs' });
+  }
+});
+
+// Get audit log summary
+app.get('/api/observability/audit-summary', (req, res) => {
+  try {
+    const { days = 7 } = req.query;
+    const summary = auditLog.getSummary(parseInt(days));
+    res.json(summary);
+  } catch (error) {
+    metrics.trackError(error, { endpoint: '/api/observability/audit-summary' });
+    res.status(500).json({ error: 'Failed to retrieve audit summary' });
+  }
+});
+
 // Get archived checks for a resource
 app.get('/api/resources/:id/archived', (req, res) => {
   const { id } = req.params;
@@ -607,7 +931,6 @@ app.get('/api/resources/:id/archived', (req, res) => {
       },
     });
   } catch (error) {
-    console.error('Error fetching archived checks:', error);
     res.status(500).json({ error: 'Failed to fetch archived checks' });
   }
 });
@@ -791,7 +1114,6 @@ app.get('/api/resources/:id/trends', (req, res) => {
       }
     });
   } catch (error) {
-    console.error('Error fetching trends:', error);
     res.status(500).json({ error: 'Failed to fetch trends' });
   }
 });
@@ -804,8 +1126,15 @@ app.get('/api/history/overview', (req, res) => {
   const currentPage = Math.max(1, parseInt(pageParam || page || 1));
   const offset = (currentPage - 1) * pageLimit;
 
+  // Get retention setting and enforce limit
+  const retentionSetting = db.prepare('SELECT value FROM settings WHERE key = ?').get('retention_days');
+  const maxDays = retentionSetting ? parseInt(retentionSetting.value) : 30;
+  const requestedDays = parseInt(days);
+  const effectiveDays = Math.min(requestedDays, maxDays);
+  const limitedByRetention = effectiveDays < requestedDays;
+
   // Create cache key based on query parameters
-  const cacheKey = `history:days=${days}:page=${currentPage}:limit=${pageLimit}:averaged=${averaged}`;
+  const cacheKey = `history:days=${effectiveDays}:page=${currentPage}:limit=${pageLimit}:averaged=${averaged}`;
   
   // Check cache first
   const cachedResult = cache.get(cacheKey);
@@ -832,7 +1161,7 @@ app.get('/api/history/overview', (req, res) => {
         MAX(response_time) as max_response
       FROM checks
       WHERE resource_id = ? AND checked_at > datetime('now', ?)
-    `).get(resource.id, `-${days} days`);
+    `).get(resource.id, `-${effectiveDays} days`);
 
     const uptime = stats.total > 0 ? (stats.up_count / stats.total * 100) : 0;
     const avgResponseTime = stats.avg_response || 0;
@@ -842,7 +1171,7 @@ app.get('/api/history/overview', (req, res) => {
     
     if (isAveraged) {
       // Compute interval hours for bucketing (1h for 7 days, 3h for 14, 6h for 30+)
-      const intervalHours = days <= 7 ? 1 : days <= 14 ? 3 : 6;
+      const intervalHours = effectiveDays <= 7 ? 1 : effectiveDays <= 14 ? 3 : 6;
       
       // Use a reliable bucketing approach with Julian Day Numbers
       // Convert to julian day, multiply by 24 for hours, divide by intervalHours and round
@@ -858,7 +1187,7 @@ app.get('/api/history/overview', (req, res) => {
         WHERE resource_id = ? AND checked_at > datetime('now', ?)
         GROUP BY ${bucketExpr}
         ORDER BY checked_at ASC
-      `).all(resource.id, `-${days} days`).map(row => ({
+      `).all(resource.id, `-${effectiveDays} days`).map(row => ({
         status: row.up_count >= Math.ceil(row.total_count/2) ? 'up' : 'down',
         response_time: Math.round(row.avg_up_response || 0),
         checked_at: row.checked_at,
@@ -871,7 +1200,7 @@ app.get('/api/history/overview', (req, res) => {
         WHERE resource_id = ? AND checked_at > datetime('now', ?)
         ORDER BY checked_at DESC
         LIMIT 600
-      `).all(resource.id, `-${days} days`).reverse();
+      `).all(resource.id, `-${effectiveDays} days`).reverse();
     }
 
     return {
@@ -884,7 +1213,15 @@ app.get('/api/history/overview', (req, res) => {
     };
   });
 
-  const result = { resources: overview, total, page: currentPage, limit: pageLimit };
+  const result = { 
+    resources: overview, 
+    total, 
+    page: currentPage, 
+    limit: pageLimit,
+    effective_days: effectiveDays,
+    requested_days: requestedDays,
+    limited_by_retention: limitedByRetention
+  };
   
   // Cache result for 2 minutes (120 seconds)
   cache.set(cacheKey, result, 120);
@@ -1033,7 +1370,6 @@ app.post('/api/test-email', async (req, res) => {
 
     res.json({ message: 'Test email sent successfully! Check your inbox.' });
   } catch (error) {
-    console.error('Test email error:', error);
     res.status(500).json({ error: error.message });
   }
 });
@@ -1054,7 +1390,6 @@ app.post('/api/test-webhook', async (req, res) => {
 
     res.json({ message: 'Test webhook sent successfully!' });
   } catch (error) {
-    console.error('Test webhook error:', error);
     res.status(500).json({ error: error.message });
   }
 });
@@ -1123,7 +1458,6 @@ app.get('/api/resources/:id/maintenance-windows', (req, res) => {
     
     res.json({ windows });
   } catch (error) {
-    console.error('Error fetching maintenance windows:', error);
     res.status(500).json({ error: 'Failed to fetch maintenance windows' });
   }
 });
@@ -1150,7 +1484,6 @@ app.post('/api/resources/:id/maintenance-windows', (req, res) => {
       message: 'Maintenance window created' 
     });
   } catch (error) {
-    console.error('Error creating maintenance window:', error);
     res.status(500).json({ error: 'Failed to create maintenance window' });
   }
 });
@@ -1163,7 +1496,6 @@ app.delete('/api/maintenance-windows/:id', (req, res) => {
     db.prepare(`DELETE FROM maintenance_windows WHERE id = ?`).run(id);
     res.json({ message: 'Maintenance window deleted' });
   } catch (error) {
-    console.error('Error deleting maintenance window:', error);
     res.status(500).json({ error: 'Failed to delete maintenance window' });
   }
 });
@@ -1185,7 +1517,6 @@ app.get('/api/maintenance-windows', (req, res) => {
 
     res.json({ maintenanceWindows: windows });
   } catch (error) {
-    console.error('Error getting maintenance windows:', error);
     res.status(500).json({ error: 'Failed to get maintenance windows' });
   }
 });
@@ -1207,7 +1538,6 @@ app.get('/api/resources/:id/in-maintenance', (req, res) => {
       reason: window?.reason || null 
     });
   } catch (error) {
-    console.error('Error checking maintenance status:', error);
     res.status(500).json({ error: 'Failed to check maintenance status' });
   }
 });
@@ -1235,7 +1565,6 @@ app.post('/api/resources/:id/calculate-baseline', (req, res) => {
       message: `Baseline calculated: ${baseline}ms`
     });
   } catch (error) {
-    console.error('Error calculating baseline:', error);
     res.status(500).json({ error: 'Failed to calculate baseline' });
   }
 });
@@ -1258,7 +1587,6 @@ app.post('/api/resources/:id/alert-rules', (req, res) => {
 
     res.json({ message: 'Alert rules updated' });
   } catch (error) {
-    console.error('Error updating alert rules:', error);
     res.status(500).json({ error: 'Failed to update alert rules' });
   }
 });
@@ -1276,7 +1604,6 @@ app.get('/api/resources/:id/transaction-steps', (req, res) => {
 
     res.json({ steps });
   } catch (error) {
-    console.error('Error fetching transaction steps:', error);
     res.status(500).json({ error: 'Failed to fetch transaction steps' });
   }
 });
@@ -1309,7 +1636,6 @@ app.post('/api/resources/:id/transaction-steps', (req, res) => {
 
     res.json({ id: result.lastInsertRowid, message: 'Transaction step created' });
   } catch (error) {
-    console.error('Error creating transaction step:', error);
     res.status(500).json({ error: 'Failed to create transaction step' });
   }
 });
@@ -1322,7 +1648,6 @@ app.delete('/api/transaction-steps/:id', (req, res) => {
     db.prepare('DELETE FROM transaction_checks WHERE id = ?').run(id);
     res.json({ message: 'Transaction step deleted' });
   } catch (error) {
-    console.error('Error deleting transaction step:', error);
     res.status(500).json({ error: 'Failed to delete transaction step' });
   }
 });
@@ -1339,7 +1664,6 @@ app.post('/api/resources/:id/toggle-transaction', (req, res) => {
 
     res.json({ message: `Transaction mode ${enabled ? 'enabled' : 'disabled'}` });
   } catch (error) {
-    console.error('Error toggling transaction mode:', error);
     res.status(500).json({ error: 'Failed to toggle transaction mode' });
   }
 });
@@ -1378,11 +1702,102 @@ app.get('/api/resources/export', (req, res) => {
     res.setHeader('Content-Disposition', 'attachment; filename="resources-export.csv"');
     res.send(csv);
   } catch (error) {
-    console.error('Error exporting resources:', error);
     res.status(500).json({ error: 'Failed to export resources' });
   }
 });
 
+// Export SLA report as CSV
+app.get('/api/sla/export', (req, res) => {
+  const { days = 30 } = req.query;
+  try {
+    const resources = db.prepare(`
+      SELECT r.*, 
+             COUNT(c.id) as total_checks,
+             SUM(CASE WHEN c.status = 'up' THEN 1 ELSE 0 END) as up_checks,
+             AVG(c.response_time) as avg_response_time,
+             COUNT(DISTINCT CASE WHEN i.id IS NOT NULL THEN i.id END) as incident_count,
+             COALESCE(SUM(CAST((julianday(COALESCE(i.resolved_at, datetime('now'))) - julianday(i.started_at)) * 24 * 60 AS INTEGER)), 0) as downtime_minutes
+      FROM resources r
+      LEFT JOIN checks c ON r.id = c.resource_id AND c.checked_at > datetime('now', '-' || ? || ' days')
+      LEFT JOIN incidents i ON r.id = i.resource_id AND i.started_at > datetime('now', '-' || ? || ' days')
+      GROUP BY r.id
+      ORDER BY r.name
+    `).all(days, days);
+
+    // CSV header
+    const headers = ['Resource Name', 'URL', 'Total Checks', 'Successful', 'Failed', 'Uptime %', 'Avg Response Time (ms)', 'Incidents', 'Downtime (minutes)'];
+    const csvRows = [headers.join(',')];
+
+    resources.forEach(resource => {
+      const uptime = resource.total_checks > 0 ? ((resource.up_checks / resource.total_checks) * 100).toFixed(2) : 0;
+      const failed = (resource.total_checks || 0) - (resource.up_checks || 0);
+      const row = [
+        `"${resource.name.replace(/"/g, '""')}"`,
+        `"${resource.url.replace(/"/g, '""')}"`,
+        resource.total_checks || 0,
+        resource.up_checks || 0,
+        failed,
+        uptime,
+        (resource.avg_response_time || 0).toFixed(0),
+        resource.incident_count || 0,
+        resource.downtime_minutes || 0
+      ];
+      csvRows.push(row.join(','));
+    });
+
+    const csv = csvRows.join('\n');
+    res.setHeader('Content-Type', 'text/csv');
+    res.setHeader('Content-Disposition', `attachment; filename="sla-report-${days}d.csv"`);
+    res.send(csv);
+  } catch (error) {
+    res.status(500).json({ error: 'Failed to export SLA report' });
+  }
+});
+
+// Export incident history as CSV
+app.get('/api/incidents/export', (req, res) => {
+  const { days = 30 } = req.query;
+  try {
+    const incidents = db.prepare(`
+      SELECT i.*, r.name as resource_name, r.url
+      FROM incidents i
+      JOIN resources r ON i.resource_id = r.id
+      WHERE i.started_at > datetime('now', '-' || ? || ' days')
+      ORDER BY i.started_at DESC
+    `).all(days);
+
+    // CSV header
+    const headers = ['Resource', 'URL', 'Started At', 'Resolved At', 'Duration (minutes)', 'Incident Type', 'Error Message', 'Status'];
+    const csvRows = [headers.join(',')];
+
+    incidents.forEach(incident => {
+      const startTime = new Date(incident.started_at).getTime();
+      const endTime = incident.resolved_at ? new Date(incident.resolved_at).getTime() : Date.now();
+      const durationMinutes = Math.round((endTime - startTime) / 1000 / 60);
+      const status = incident.resolved_at ? 'Resolved' : 'Active';
+      
+      const row = [
+        `"${incident.resource_name.replace(/"/g, '""')}"`,
+        `"${incident.url.replace(/"/g, '""')}"`,
+        new Date(incident.started_at).toISOString(),
+        incident.resolved_at ? new Date(incident.resolved_at).toISOString() : '',
+        durationMinutes,
+        incident.type || 'outage',
+        `"${(incident.error_message || '').replace(/"/g, '""')}"`,
+        status
+      ];
+      csvRows.push(row.join(','));
+    });
+
+    const csv = csvRows.join('\n');
+    res.setHeader('Content-Type', 'text/csv');
+    res.setHeader('Content-Disposition', `attachment; filename="incidents-export-${days}d.csv"`);
+    res.send(csv);
+  } catch (error) {
+    res.status(500).json({ error: 'Failed to export incidents' });
+  }
+});
+
 // Import resources from CSV
 app.post('/api/resources/import', express.text({ type: 'text/csv' }), (req, res) => {
   try {
@@ -1432,6 +1847,40 @@ app.post('/api/resources/import', express.text({ type: 'text/csv' }), (req, res)
           continue;
         }
 
+        // Validate URL format
+        try {
+          new URL(row.url);
+        } catch (e) {
+          errors.push(`Row ${i + 1}: Invalid URL format`);
+          continue;
+        }
+
+        // Check for duplicates
+        const existing = db.prepare('SELECT id FROM resources WHERE url = ?').get(row.url);
+        if (existing) {
+          errors.push(`Row ${i + 1}: URL already exists`);
+          continue;
+        }
+
+        // Validate numeric fields
+        const checkInterval = parseInt(row.check_interval) || 60000;
+        if (checkInterval < 10000) {
+          errors.push(`Row ${i + 1}: Check interval must be at least 10000ms`);
+          continue;
+        }
+
+        const timeout = parseInt(row.timeout) || 5000;
+        if (timeout < 1000) {
+          errors.push(`Row ${i + 1}: Timeout must be at least 1000ms`);
+          continue;
+        }
+
+        const slaTarget = parseFloat(row.sla_target) || 99.9;
+        if (isNaN(slaTarget) || slaTarget < 0 || slaTarget > 100) {
+          errors.push(`Row ${i + 1}: SLA target must be between 0 and 100`);
+          continue;
+        }
+
         const stmt = db.prepare(`
           INSERT INTO resources (name, url, type, check_interval, timeout, sla_target, tags, group_id, email_to)
           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
@@ -1441,9 +1890,9 @@ app.post('/api/resources/import', express.text({ type: 'text/csv' }), (req, res)
           row.name,
           row.url,
           row.type || 'http',
-          parseInt(row.check_interval) || 60000,
-          parseInt(row.timeout) || 5000,
-          parseFloat(row.sla_target) || 99.9,
+          checkInterval,
+          timeout,
+          slaTarget,
           row.tags || '',
           row.group_id ? parseInt(row.group_id) : null,
           row.email_to || ''
@@ -1460,7 +1909,6 @@ app.post('/api/resources/import', express.text({ type: 'text/csv' }), (req, res)
 
     res.json({ imported, errors, count: imported.length });
   } catch (error) {
-    console.error('Error importing resources:', error);
     res.status(500).json({ error: 'Failed to import resources' });
   }
 });
@@ -1547,19 +1995,165 @@ function broadcastDashboardUpdate() {
       }
     });
   } catch (error) {
-    console.error('Error broadcasting dashboard update:', error.message);
+    // Broadcast error handled silently
   }
 }
 
-// Export broadcast function for use in scheduler
+// Broadcast alert notification to WebSocket clients
+function broadcastAlert(alert) {
+  try {
+    const message = JSON.stringify({
+      type: 'alert',
+      data: alert
+    });
+
+    wsClients.forEach(client => {
+      if (client.readyState === WebSocket.OPEN) {
+        client.send(message);
+      } else {
+        wsClients.delete(client);
+      }
+    });
+  } catch (error) {
+    // Broadcast error handled silently
+  }
+}
+
+// Broadcast incident update to WebSocket clients
+function broadcastIncident(incident) {
+  try {
+    const message = JSON.stringify({
+      type: 'incident',
+      data: incident
+    });
+
+    wsClients.forEach(client => {
+      if (client.readyState === WebSocket.OPEN) {
+        client.send(message);
+      } else {
+        wsClients.delete(client);
+      }
+    });
+  } catch (error) {
+    // Broadcast error handled silently
+  }
+}
+
+// Broadcast metrics update to WebSocket clients
+function broadcastMetrics() {
+  try {
+    const metricsData = metrics.getMetrics();
+    const message = JSON.stringify({
+      type: 'metrics',
+      data: metricsData
+    });
+
+    wsClients.forEach(client => {
+      if (client.readyState === WebSocket.OPEN) {
+        client.send(message);
+      } else {
+        wsClients.delete(client);
+      }
+    });
+  } catch (error) {
+    // Broadcast error handled silently
+  }
+}
+
+// Export broadcast functions for use in scheduler and other modules
 global.broadcastDashboardUpdate = broadcastDashboardUpdate;
+global.broadcastAlert = broadcastAlert;
+global.broadcastIncident = broadcastIncident;
+global.broadcastMetrics = broadcastMetrics;
+
+// Broadcast new notification to all connected WebSocket clients
+function broadcastNotification(notification) {
+  const message = JSON.stringify({
+    type: 'notification',
+    data: notification
+  });
+  wsClients.forEach(client => {
+    if (client.readyState === WebSocket.OPEN) {
+      client.send(message);
+    }
+  });
+}
+
+// Export broadcast notification function
+global.broadcastNotification = broadcastNotification;
+
+// Health check endpoint
+app.get('/api/health', (req, res) => {
+  try {
+    const uptime = process.uptime();
+    const dbCheck = db.prepare('SELECT COUNT(*) as count FROM resources').get();
+    const schedulerStatus = scheduler.isRunning ? 'running' : 'stopped';
+    
+    res.json({
+      status: 'ok',
+      uptime: Math.floor(uptime),
+      uptimeFormatted: `${Math.floor(uptime / 3600)}h ${Math.floor((uptime % 3600) / 60)}m`,
+      timestamp: new Date().toISOString(),
+      database: {
+        connected: !!dbCheck,
+        resourcesCount: dbCheck?.count || 0
+      },
+      scheduler: {
+        status: schedulerStatus,
+        isRunning: scheduler.isRunning
+      }
+    });
+  } catch (error) {
+    res.status(503).json({ 
+      status: 'error', 
+      error: error.message,
+      timestamp: new Date().toISOString()
+    });
+  }
+});
+
+// Graceful shutdown handlers
+let isShuttingDown = false;
+
+async function gracefulShutdown(signal) {
+  if (isShuttingDown) return;
+  isShuttingDown = true;
+
+  try {
+    // Close WebSocket connections
+    wsClients.forEach(client => {
+      if (client.readyState === WebSocket.OPEN) {
+        client.close(1000, 'Server shutting down');
+      }
+    });
+    wsClients.clear();
+
+    // Stop scheduler
+    if (scheduler.isRunning) {
+      scheduler.stop();
+    }
+
+    // Close database
+    try {
+      db.close();
+    } catch (e) {
+      // Database already closed
+    }
+
+    process.exit(0);
+  } catch (error) {
+    process.exit(1);
+  }
+}
+
+process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
+process.on('SIGINT', () => gracefulShutdown('SIGINT'));
 
 // Start server with HTTP and WebSocket
 const server = http.createServer(app);
 const wss = new WebSocket.Server({ server });
 
 wss.on('connection', (ws) => {
-  console.log('WebSocket client connected');
   wsClients.add(ws);
 
   // Send initial dashboard data
@@ -1572,23 +2166,19 @@ wss.on('connection', (ws) => {
         ws.send(JSON.stringify({ type: 'pong' }));
       }
     } catch (error) {
-      console.error('Error handling WebSocket message:', error.message);
+      // Handle JSON parse errors silently
     }
   });
 
   ws.on('close', () => {
-    console.log('WebSocket client disconnected');
     wsClients.delete(ws);
   });
 
   ws.on('error', (error) => {
-    console.error('WebSocket error:', error.message);
     wsClients.delete(ws);
   });
 });
 
 server.listen(PORT, () => {
-  console.log(`Server running on http://localhost:${PORT}`);
-  console.log(`WebSocket available at ws://localhost:${PORT}`);
   scheduler.start();
 });

server/metrics.js

@@ -0,0 +1,128 @@
+const db = require('./database');
+
+class MetricsCollector {
+  constructor() {
+    this.apiMetrics = new Map();
+    this.errorCounts = new Map();
+    this.startTime = Date.now();
+  }
+
+  // Track API request metrics
+  trackApiRequest(endpoint, method, statusCode, duration) {
+    const key = `${method}:${endpoint}`;
+    if (!this.apiMetrics.has(key)) {
+      this.apiMetrics.set(key, {
+        endpoint,
+        method,
+        totalRequests: 0,
+        successCount: 0,
+        errorCount: 0,
+        totalDuration: 0,
+        minDuration: Infinity,
+        maxDuration: 0,
+        statusCodes: {}
+      });
+    }
+
+    const metrics = this.apiMetrics.get(key);
+    metrics.totalRequests++;
+    metrics.totalDuration += duration;
+    metrics.minDuration = Math.min(metrics.minDuration, duration);
+    metrics.maxDuration = Math.max(metrics.maxDuration, duration);
+
+    if (statusCode >= 200 && statusCode < 400) {
+      metrics.successCount++;
+    } else {
+      metrics.errorCount++;
+    }
+
+    metrics.statusCodes[statusCode] = (metrics.statusCodes[statusCode] || 0) + 1;
+  }
+
+  // Track application errors
+  trackError(error, context = {}) {
+    const errorKey = error.message || 'Unknown Error';
+    const errorData = {
+      message: error.message,
+      stack: error.stack,
+      context,
+      timestamp: new Date().toISOString(),
+      count: (this.errorCounts.get(errorKey) || 0) + 1
+    };
+
+    this.errorCounts.set(errorKey, errorData.count);
+
+    // Log to database
+    try {
+      db.prepare(`
+        INSERT INTO error_logs (error_message, stack_trace, context, occurred_at)
+        VALUES (?, ?, ?, datetime('now'))
+      `).run(
+        error.message,
+        error.stack || '',
+        JSON.stringify(context)
+      );
+    } catch (err) {
+      // Silent failure - don't crash on logging errors
+    }
+
+    return errorData;
+  }
+
+  // Get current metrics snapshot
+  getMetrics() {
+    const apiStats = [];
+    this.apiMetrics.forEach((metrics, key) => {
+      const avgDuration = metrics.totalRequests > 0 
+        ? Math.round(metrics.totalDuration / metrics.totalRequests)
+        : 0;
+      
+      apiStats.push({
+        endpoint: metrics.endpoint,
+        method: metrics.method,
+        totalRequests: metrics.totalRequests,
+        successRate: metrics.totalRequests > 0
+          ? ((metrics.successCount / metrics.totalRequests) * 100).toFixed(2)
+          : 0,
+        avgResponseTime: avgDuration,
+        minResponseTime: metrics.minDuration === Infinity ? 0 : metrics.minDuration,
+        maxResponseTime: metrics.maxDuration,
+        statusCodes: metrics.statusCodes
+      });
+    });
+
+    return {
+      uptime: Math.floor((Date.now() - this.startTime) / 1000),
+      apiStats: apiStats.sort((a, b) => b.totalRequests - a.totalRequests),
+      errorCount: Array.from(this.errorCounts.values()).reduce((sum, count) => sum + count, 0),
+      uniqueErrors: this.errorCounts.size
+    };
+  }
+
+  // Get recent errors from database
+  getRecentErrors(limit = 50) {
+    try {
+      return db.prepare(`
+        SELECT 
+          id,
+          error_message,
+          stack_trace,
+          context,
+          REPLACE(occurred_at, ' ', 'T') || 'Z' as occurred_at
+        FROM error_logs
+        ORDER BY occurred_at DESC
+        LIMIT ?
+      `).all(limit);
+    } catch (err) {
+      return [];
+    }
+  }
+
+  // Clear in-memory metrics (for testing/reset)
+  reset() {
+    this.apiMetrics.clear();
+    this.errorCounts.clear();
+  }
+}
+
+module.exports = new MetricsCollector();

server/monitorService.js

@@ -7,6 +7,8 @@ const dns = require('dns').promises;
 const WebSocket = require('ws');
 const ping = require('ping');
 const db = require('./database');
+const RetryPolicy = require('./retryPolicy');
+const CircuitBreaker = require('./circuitBreaker');
 
 const DEFAULT_PORTS = {
   http: 80,
@@ -32,6 +34,33 @@ class MonitorService {
     // Keep-alive agents for pooled HTTP/S connections
     this.httpAgent = new http.Agent({ keepAlive: true, maxSockets: 50 });
     this.httpsAgent = new https.Agent({ keepAlive: true, maxSockets: 50 });
+    
+    // Circuit breaker for HTTP checks (5 failures before opening, 60s reset)
+    this.httpCircuitBreaker = new CircuitBreaker(5, 60000, this.isTransientError);
+    
+    // Retry policy: 3 retries with exponential backoff
+    this.retryPolicy = new RetryPolicy(3, 1000, 10000);
+    
+    // Circuit breakers per resource for fine-grained control
+    this.resourceCircuitBreakers = new Map();
+  }
+
+  getResourceCircuitBreaker(resourceId) {
+    if (!this.resourceCircuitBreakers.has(resourceId)) {
+      this.resourceCircuitBreakers.set(resourceId, new CircuitBreaker(3, 120000, this.isTransientError));
+    }
+    return this.resourceCircuitBreakers.get(resourceId);
+  }
+
+  isTransientError(error) {
+    // Only retry on transient errors (network, timeout, 5xx)
+    if (error.code === 'ECONNREFUSED' || error.code === 'ECONNRESET' || error.code === 'ETIMEDOUT') {
+      return true;
+    }
+    if (error.response?.status >= 500) {
+      return true;
+    }
+    return false;
   }
   async checkResource(resource) {
     const startTime = Date.now();
@@ -77,49 +106,59 @@ class MonitorService {
 
   async checkHttp(resource, startTime, base) {
     let result = { ...base };
+    const circuitBreaker = this.getResourceCircuitBreaker(resource.id);
+    
     try {
-      const headers = {};
-      if (resource.http_headers) {
-        try {
-          const customHeaders = JSON.parse(resource.http_headers);
-          Object.assign(headers, customHeaders);
-        } catch (e) {
-          // Invalid JSON, skip custom headers
-        }
-      }
+      await circuitBreaker.execute(async () => {
+        return await this.retryPolicy.execute(async () => {
+          const headers = {};
+          if (resource.http_headers) {
+            try {
+              const customHeaders = JSON.parse(resource.http_headers);
+              Object.assign(headers, customHeaders);
+            } catch (e) {
+              // Invalid JSON, skip custom headers
+            }
+          }
 
-      const isHttps = resource.url.startsWith('https');
-      const response = await axios.get(resource.url, {
-        timeout: resource.timeout,
-        validateStatus: () => true,
-        maxRedirects: 5,
-        headers,
-        httpAgent: this.httpAgent,
-        httpsAgent: this.httpsAgent,
+          const isHttps = resource.url.startsWith('https');
+          const response = await axios.get(resource.url, {
+            timeout: resource.timeout,
+            validateStatus: () => true,
+            maxRedirects: 5,
+            headers,
+            httpAgent: this.httpAgent,
+            httpsAgent: this.httpsAgent,
+          });
+          const responseTime = Date.now() - startTime;
+          const isUp = response.status >= 200 && response.status < 400;
+          
+          // Check for keyword if specified
+          let keywordMatch = true;
+          if (resource.http_keyword && response.data) {
+            const dataStr = typeof response.data === 'string' ? response.data : JSON.stringify(response.data);
+            keywordMatch = dataStr.includes(resource.http_keyword);
+            if (!keywordMatch) {
+              result.error_message = `Keyword "${resource.http_keyword}" not found in response`;
+            }
+          }
+
+          result = {
+            ...result,
+            status: (isUp && keywordMatch) ? 'up' : 'down',
+            response_time: responseTime,
+            status_code: response.status,
+            error_message: isUp && keywordMatch ? null : (result.error_message || `HTTP ${response.status}`),
+          };
+          
+          return result;
+        }, `HTTP check for ${resource.name}`);
       });
-      const responseTime = Date.now() - startTime;
-      const isUp = response.status >= 200 && response.status < 400;
-      
-      // Check for keyword if specified
-      let keywordMatch = true;
-      if (resource.http_keyword && response.data) {
-        const dataStr = typeof response.data === 'string' ? response.data : JSON.stringify(response.data);
-        keywordMatch = dataStr.includes(resource.http_keyword);
-        if (!keywordMatch) {
-          result.error_message = `Keyword "${resource.http_keyword}" not found in response`;
-        }
-      }
-
-      result = {
-        ...result,
-        status: (isUp && keywordMatch) ? 'up' : 'down',
-        response_time: responseTime,
-        status_code: response.status,
-        error_message: isUp && keywordMatch ? null : (result.error_message || `HTTP ${response.status}`),
-      };
     } catch (error) {
-      result.error_message = error.message;
-      result.response_time = Date.now() - startTime;
+      const responseTime = Date.now() - startTime;
+      result.error_message = error.originalError?.message || error.message;
+      result.response_time = responseTime;
+      result.status = 'down';
     }
     return result;
   }
@@ -706,7 +745,6 @@ class MonitorService {
 
       return currentResponseTime > threshold;
     } catch (error) {
-      console.error('Error checking response time anomaly:', error.message);
       return false;
     }
   }

server/notificationService.js

@@ -44,6 +44,8 @@ class NotificationService {
           user: config.email_user,
           pass: config.email_pass,
         },
+        connectionTimeout: 10000,
+        socketTimeout: 10000,
       });
     } else {
       this.transporter = null;
@@ -99,6 +101,47 @@ class NotificationService {
     }
 
     await Promise.allSettled(promises);
+
+    // Save notification to in-app notification center
+    const status = isDown ? 'down' : 'up';
+    this.saveNotification(resource, incident, status, message);
+  }
+
+  saveNotification(resource, incident, status, message) {
+    try {
+      const db = require('./database');
+      const title = status === 'down' ? 
+        `🔴 ${resource.name} is DOWN` : 
+        `🟢 ${resource.name} is UP`;
+      
+      const result = db.prepare(`
+        INSERT INTO notifications (resource_id, incident_id, type, title, message, read)
+        VALUES (?, ?, ?, ?, ?, 0)
+      `).run(
+        resource.id,
+        incident.id || null,
+        status,
+        title,
+        message
+      );
+
+      // Broadcast new notification to WebSocket clients in real-time
+      if (global.broadcastNotification) {
+        global.broadcastNotification({
+          id: result.lastInsertRowid,
+          resource_id: resource.id,
+          resource_name: resource.name,
+          incident_id: incident.id || null,
+          type: status,
+          title,
+          message,
+          read: 0,
+          created_at: new Date().toISOString()
+        });
+      }
+    } catch (error) {
+      console.error('Failed to save notification:', error.message);
+    }
   }
 
   isQuietHours(resource) {
@@ -125,7 +168,6 @@ class NotificationService {
 
   async sendEmail(resource, message, type, stats = null, emailOverride = null) {
     if (!this.transporter) {
-      console.error(`[Email] Config incomplete; skipping email (enabled=${this.emailEnabled}, host=${this.config.email_host})`);
       return;
     }
 
@@ -187,7 +229,7 @@ class NotificationService {
           : [],
       });
     } catch (error) {
-      console.error('Email error:', error.message);
+      // Email error handled
     }
   }
 
@@ -234,7 +276,6 @@ class NotificationService {
       
       return !!window;
     } catch (error) {
-      console.error('Error checking maintenance window:', error.message);
       return false;
     }
   }
@@ -249,7 +290,7 @@ class NotificationService {
         timestamp: new Date().toISOString(),
       });
     } catch (error) {
-      console.error('Webhook error:', error.message);
+      // Webhook error handled silently
     }
   }
 }

server/retryPolicy.js

@@ -0,0 +1,44 @@
+// Retry logic with exponential backoff
+class RetryPolicy {
+  constructor(maxRetries = 3, initialDelayMs = 1000, maxDelayMs = 10000) {
+    this.maxRetries = maxRetries;
+    this.initialDelayMs = initialDelayMs;
+    this.maxDelayMs = maxDelayMs;
+  }
+
+  async execute(fn, context = 'operation') {
+    let lastError;
+    
+    for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
+      try {
+        return await fn();
+      } catch (error) {
+        lastError = error;
+        
+        if (attempt < this.maxRetries) {
+          const delay = this.calculateBackoff(attempt);
+          await this.sleep(delay);
+        }
+      }
+    }
+    
+    const error = new Error(`${context} failed after ${this.maxRetries + 1} attempts`);
+    error.originalError = lastError;
+    throw error;
+  }
+
+  calculateBackoff(attempt) {
+    // Exponential backoff: initialDelay * 2^attempt
+    // With jitter: ±10%
+    const exponentialDelay = this.initialDelayMs * Math.pow(2, attempt);
+    const cappedDelay = Math.min(exponentialDelay, this.maxDelayMs);
+    const jitter = cappedDelay * 0.1 * (Math.random() - 0.5);
+    return cappedDelay + jitter;
+  }
+
+  sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+}
+
+module.exports = RetryPolicy;

server/scheduler.js

@@ -6,12 +6,20 @@ const notificationService = require('./notificationService');
 class Scheduler {
   constructor() {
     this.jobs = new Map();
+    this.isRunning = false;
   }
 
   start() {
     // Run checks every minute
     cron.schedule('* * * * *', async () => {
-      await this.runChecks();
+      if (!this.isRunning) {
+        this.isRunning = true;
+        try {
+          await this.runChecks();
+        } finally {
+          this.isRunning = false;
+        }
+      }
     });
 
     // Archive old checks daily at 2 AM
@@ -23,37 +31,44 @@ class Scheduler {
     setTimeout(() => {
       this.archiveOldChecks();
     }, 5000);
-
-    console.log('Scheduler started');
   }
 
   archiveOldChecks() {
     try {
+      // Get global retention setting
       const retentionSetting = db.prepare(`
         SELECT value FROM settings WHERE key = 'retention_days'
       `).get();
       
-      const retentionDays = retentionSetting ? parseInt(retentionSetting.value) : 30;
-      const cutoffDate = new Date();
-      cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
-      const cutoffString = cutoffDate.toISOString().replace('T', ' ').split('.')[0];
+      const globalRetentionDays = retentionSetting ? parseInt(retentionSetting.value) : 30;
 
-      // Move old checks to archived_checks
-      const oldChecks = db.prepare(`
-        SELECT * FROM checks WHERE checked_at < ?
-      `).all(cutoffString);
+      // Get all resources with their retention settings
+      const resources = db.prepare(`
+        SELECT id, retention_days FROM resources
+      `).all();
 
-      if (oldChecks.length > 0) {
-        const insertArchived = db.prepare(`
-          INSERT INTO archived_checks (resource_id, status, response_time, status_code, error_message, details, checked_at)
-          VALUES (?, ?, ?, ?, ?, ?, ?)
-        `);
+      const insertArchived = db.prepare(`
+        INSERT INTO archived_checks (resource_id, status, response_time, status_code, error_message, details, checked_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?)
+      `);
 
-        const deleteOld = db.prepare(`
-          DELETE FROM checks WHERE id = ?
-        `);
+      const deleteOld = db.prepare(`
+        DELETE FROM checks WHERE id = ?
+      `);
+
+      const transaction = db.transaction(() => {
+        resources.forEach(resource => {
+          // Use per-resource retention if set, otherwise use global
+          const retentionDays = resource.retention_days || globalRetentionDays;
+          const cutoffDate = new Date();
+          cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
+          const cutoffString = cutoffDate.toISOString().replace('T', ' ').split('.')[0];
+
+          // Get old checks for this resource
+          const oldChecks = db.prepare(`
+            SELECT * FROM checks WHERE resource_id = ? AND checked_at < ?
+          `).all(resource.id, cutoffString);
 
-        const transaction = db.transaction(() => {
           oldChecks.forEach(check => {
             insertArchived.run(
               check.resource_id,
@@ -67,12 +82,11 @@ class Scheduler {
             deleteOld.run(check.id);
           });
         });
+      });
 
-        transaction();
-        console.log(`[Archival] Archived ${oldChecks.length} checks older than ${retentionDays} days`);
-      }
+      transaction();
     } catch (error) {
-      console.error('[Archival] Error archiving checks:', error.message);
+      // Archive error handled silently
     }
   }
 
@@ -133,10 +147,8 @@ class Scheduler {
             await notificationService.sendAlert(resource, incident, stats);
           }
         }
-
-        console.log(`Checked ${resource.name}: ${result.status} (${result.response_time}ms)`);
       } catch (error) {
-        console.error(`Error checking ${resource.name}:`, error.message);
+        // Check error handled silently
       }
     })());
 
@@ -147,6 +159,11 @@ class Scheduler {
       global.broadcastDashboardUpdate();
     }
   }
+
+  stop() {
+    // Stop the scheduler (cron tasks will continue running but we flag isRunning)
+    this.isRunning = false;
+  }
 }
 
 module.exports = new Scheduler();

server/securityMiddleware.js

@@ -0,0 +1,118 @@
+const jwt = require('jsonwebtoken');
+const crypto = require('crypto');
+
+const JWT_SECRET = process.env.JWT_SECRET || crypto.randomBytes(32).toString('hex');
+const JWT_EXPIRY = 3600; // 1 hour
+const REFRESH_TOKEN_EXPIRY = 86400 * 7; // 7 days
+
+class SecurityMiddleware {
+  // Generate JWT token
+  generateToken(payload, expiresIn = JWT_EXPIRY) {
+    return jwt.sign(payload, JWT_SECRET, { expiresIn });
+  }
+
+  // Generate refresh token
+  generateRefreshToken(payload) {
+    return jwt.sign(payload, JWT_SECRET, { expiresIn: REFRESH_TOKEN_EXPIRY });
+  }
+
+  // Verify JWT token
+  verifyToken(token) {
+    try {
+      return jwt.verify(token, JWT_SECRET);
+    } catch (err) {
+      return null;
+    }
+  }
+
+  // Verify refresh token
+  verifyRefreshToken(token) {
+    try {
+      const decoded = jwt.verify(token, JWT_SECRET);
+      // Additional validation: check if it's a refresh token (has longer expiry)
+      if (decoded.exp - decoded.iat < REFRESH_TOKEN_EXPIRY - 100) {
+        return null; // Not a refresh token
+      }
+      return decoded;
+    } catch (err) {
+      return null;
+    }
+  }
+
+  // Verify token expiry
+  isTokenExpired(token) {
+    const decoded = this.verifyToken(token);
+    if (!decoded) return true;
+    return false;
+  }
+
+  // Generate CSRF token (alias for compatibility)
+  generateCSRFToken() {
+    return this.generateCsrfToken();
+  }
+
+  // Generate CSRF token
+  generateCsrfToken() {
+    return crypto.randomBytes(32).toString('hex');
+  }
+
+  // Verify CSRF token
+  verifyCsrfToken(token, sessionToken) {
+    if (!token || !sessionToken) return false;
+    return crypto.timingSafeEqual(Buffer.from(token), Buffer.from(sessionToken));
+  }
+
+  // Hash password with salt
+  hashPassword(password) {
+    return crypto.createHash('sha256').update(password).digest('hex');
+  }
+
+  // Validate password strength
+  validatePasswordStrength(password) {
+    if (password.length < 8) {
+      return { valid: false, error: 'Password must be at least 8 characters' };
+    }
+    if (!/[A-Z]/.test(password)) {
+      return { valid: false, error: 'Password must contain at least one uppercase letter' };
+    }
+    if (!/[a-z]/.test(password)) {
+      return { valid: false, error: 'Password must contain at least one lowercase letter' };
+    }
+    if (!/[0-9]/.test(password)) {
+      return { valid: false, error: 'Password must contain at least one digit' };
+    }
+    if (!/[!@#$%^&*(),.?":{}|<>]/.test(password)) {
+      return { valid: false, error: 'Password must contain at least one special character' };
+    }
+    return { valid: true };
+  }
+
+  // Middleware to verify CSRF
+  csrfMiddleware(req, res, next) {
+    const token = req.headers['x-csrf-token'] || req.body.csrfToken;
+    const sessionToken = req.session?.csrfToken;
+    
+    if (!this.verifyCsrfToken(token, sessionToken)) {
+      return res.status(403).json({ error: 'CSRF token validation failed' });
+    }
+    next();
+  }
+
+  // Middleware to verify JWT
+  jwtMiddleware(req, res, next) {
+    const token = req.headers.authorization?.split(' ')[1];
+    if (!token) {
+      return res.status(401).json({ error: 'No authorization token provided' });
+    }
+
+    const decoded = this.verifyToken(token);
+    if (!decoded) {
+      return res.status(401).json({ error: 'Invalid or expired token' });
+    }
+
+    req.user = decoded;
+    next();
+  }
+}
+
+module.exports = new SecurityMiddleware();