Add agent notifications + fix update script

- scheduler.js: check agents every 2 min — alert on offline/online
  transitions and CPU/memory/disk threshold breaches (90%)
- notificationService.js: sendAgentAlert() sends via email, webhook,
  ntfy, and saves to in-app notifications table
- package.json: pin better-sqlite3 to 9.6.0 (compatible with Node 20)
- update-server.sh: add React client build step; use npm install
  instead of rm -rf + npm ci for faster incremental updates

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

package.json

@@ -22,7 +22,7 @@
   "license": "MIT",
   "dependencies": {
     "axios": "^1.6.2",
-    "better-sqlite3": "^13.0.3",
+    "better-sqlite3": "9.6.0",
     "cors": "^2.8.5",
     "dotenv": "^16.3.1",
     "express": "^4.18.2",

server/notificationService.js

@@ -446,6 +446,77 @@ class NotificationService {
     }
   }
 
+  async sendAgentAlert(agent, type, message) {
+    const isOffline = type === 'offline';
+    const isOnline  = type === 'online';
+    const emoji  = isOffline ? '🔴' : isOnline ? '🟢' : '⚠️';
+    const title  = isOffline ? `Agent Offline: ${agent.name}`
+                 : isOnline  ? `Agent Online: ${agent.name}`
+                 : `Agent Alert: ${agent.name}`;
+    const notifType = isOffline ? 'down' : isOnline ? 'up' : 'warning';
+
+    // In-app notification
+    try {
+      const db = require('./database');
+      const result = db.prepare(`
+        INSERT INTO notifications (resource_id, incident_id, type, title, message)
+        VALUES (NULL, NULL, ?, ?, ?)
+      `).run(notifType, title, message);
+      if (global.broadcastNotification) {
+        global.broadcastNotification({
+          id: result.lastInsertRowid, resource_id: null, resource_name: agent.name,
+          incident_id: null, type: notifType, title, message, read: 0,
+          created_at: new Date().toISOString(),
+        });
+      }
+    } catch (_) {}
+
+    // Email
+    if (this.emailEnabled && this.transporter && this.config.email_to) {
+      try {
+        await this.transporter.sendMail({
+          from: this.config.email_from,
+          to: this.config.email_to,
+          subject: `${emoji} SkyWatch Agent: ${title}`,
+          text: [
+            message,
+            '',
+            `Agent:    ${agent.name}`,
+            `Hostname: ${agent.hostname || '—'}`,
+            `IP:       ${agent.ip_address || '—'}`,
+            `OS:       ${agent.os_info || '—'}`,
+            `Time:     ${new Date().toLocaleString()}`,
+          ].join('\n'),
+        });
+      } catch (_) {}
+    }
+
+    // Webhook
+    if (this.webhookEnabled && this.config.webhook_url) {
+      try {
+        await axios.post(this.config.webhook_url, {
+          type: 'agent', event: type,
+          agent: { id: agent.id, name: agent.name, hostname: agent.hostname, ip_address: agent.ip_address },
+          title, message, timestamp: new Date().toISOString(),
+        });
+      } catch (_) {}
+    }
+
+    // Ntfy
+    if (this.ntfyEnabled && this.config.ntfy_topic) {
+      const ntfyBase = (this.config.ntfy_url || 'https://ntfy.sh').replace(/\/$/, '');
+      try {
+        await axios.post(`${ntfyBase}/`, {
+          topic: this.config.ntfy_topic,
+          title: `${emoji} ${title}`,
+          message,
+          priority: isOffline ? 4 : isOnline ? 2 : 3,
+          tags: isOffline ? ['red_circle'] : isOnline ? ['green_circle'] : ['warning'],
+        });
+      } catch (_) {}
+    }
+  }
+
   async sendNtfy(resource, type) {
     const ntfyUrl = this.config.ntfy_url || 'https://ntfy.sh';
     const ntfyTopic = this.config.ntfy_topic;

server/scheduler.js

@@ -3,13 +3,21 @@ const db = require('./database');
 const monitorService = require('./monitorService');
 const notificationService = require('./notificationService');
 
+const AGENT_CPU_THRESHOLD  = 90;
+const AGENT_MEM_THRESHOLD  = 90;
+const AGENT_DISK_THRESHOLD = 90;
+const AGENT_OFFLINE_MS     = 3 * 60 * 1000; // matches UI isOnline() check
+
 class Scheduler {
   constructor() {
     this.jobs = new Map();
     this.isRunning = false;
     this.lastCheckedAt = new Map();
     this.slowAlertCooldown = new Map();
-    this.slowConsecutiveCount = new Map(); // tracks consecutive slow checks per resource
+    this.slowConsecutiveCount = new Map();
+    // Agent health tracking — keyed by agent id
+    this.agentOnlineState = new Map(); // id -> boolean
+    this.agentAlertState  = new Map(); // id -> { cpu, mem, disk }
   }
 
   start() {
@@ -25,6 +33,11 @@ class Scheduler {
       }
     });
 
+    // Agent health check every 2 minutes
+    cron.schedule('*/2 * * * *', async () => {
+      await this.checkAgents();
+    });
+
     // Archive old checks + update baselines + check escalations daily at 2 AM
     cron.schedule('0 2 * * *', () => {
       this.archiveOldChecks();
@@ -43,6 +56,111 @@ class Scheduler {
     }, 5000);
   }
 
+  async checkAgents() {
+    try {
+      const rows = db.prepare(`
+        SELECT
+          a.id, a.name, a.hostname, a.ip_address, a.os_info, a.last_seen_at,
+          m.cpu_percent, m.mem_percent, m.disk_data
+        FROM agents a
+        LEFT JOIN agent_metrics m ON m.id = (
+          SELECT id FROM agent_metrics WHERE agent_id = a.id ORDER BY recorded_at DESC LIMIT 1
+        )
+        ORDER BY a.id
+      `).all();
+
+      const liveIds = new Set();
+
+      for (const agent of rows) {
+        liveIds.add(agent.id);
+
+        const isOnline = agent.last_seen_at
+          ? Date.now() - new Date(agent.last_seen_at).getTime() < AGENT_OFFLINE_MS
+          : false;
+
+        const wasOnline = this.agentOnlineState.get(agent.id);
+
+        if (wasOnline === undefined) {
+          // First poll — seed state without alerting
+          this.agentOnlineState.set(agent.id, isOnline);
+          this.agentAlertState.set(agent.id, { cpu: false, mem: false, disk: false });
+          continue;
+        }
+
+        // Online/offline transitions
+        if (wasOnline && !isOnline) {
+          await notificationService.sendAgentAlert(
+            agent, 'offline',
+            `Agent "${agent.name}" (${agent.hostname || agent.ip_address || 'unknown'}) has gone offline.`
+          );
+        } else if (!wasOnline && isOnline) {
+          await notificationService.sendAgentAlert(
+            agent, 'online',
+            `Agent "${agent.name}" (${agent.hostname || agent.ip_address || 'unknown'}) is back online.`
+          );
+          this.agentAlertState.set(agent.id, { cpu: false, mem: false, disk: false });
+        }
+
+        this.agentOnlineState.set(agent.id, isOnline);
+
+        if (!isOnline || agent.cpu_percent == null) continue;
+
+        const as = this.agentAlertState.get(agent.id) || { cpu: false, mem: false, disk: false };
+
+        // CPU
+        if (agent.cpu_percent > AGENT_CPU_THRESHOLD && !as.cpu) {
+          await notificationService.sendAgentAlert(
+            agent, 'cpu',
+            `Agent "${agent.name}" CPU is at ${Number(agent.cpu_percent).toFixed(1)}% (threshold: ${AGENT_CPU_THRESHOLD}%).`
+          );
+          as.cpu = true;
+        } else if (agent.cpu_percent <= AGENT_CPU_THRESHOLD) {
+          as.cpu = false;
+        }
+
+        // Memory
+        if (agent.mem_percent > AGENT_MEM_THRESHOLD && !as.mem) {
+          await notificationService.sendAgentAlert(
+            agent, 'memory',
+            `Agent "${agent.name}" memory is at ${Number(agent.mem_percent).toFixed(1)}% (threshold: ${AGENT_MEM_THRESHOLD}%).`
+          );
+          as.mem = true;
+        } else if (agent.mem_percent <= AGENT_MEM_THRESHOLD) {
+          as.mem = false;
+        }
+
+        // Disk — check all partitions
+        if (agent.disk_data) {
+          try {
+            const disks = JSON.parse(agent.disk_data);
+            const worst = disks.reduce((m, d) => (d.percent > (m?.percent ?? 0) ? d : m), null);
+            if (worst && worst.percent > AGENT_DISK_THRESHOLD && !as.disk) {
+              await notificationService.sendAgentAlert(
+                agent, 'disk',
+                `Agent "${agent.name}" disk ${worst.path} is at ${worst.percent}% (threshold: ${AGENT_DISK_THRESHOLD}%).`
+              );
+              as.disk = true;
+            } else if (!worst || worst.percent <= AGENT_DISK_THRESHOLD) {
+              as.disk = false;
+            }
+          } catch (_) {}
+        }
+
+        this.agentAlertState.set(agent.id, as);
+      }
+
+      // Remove state for deleted agents
+      for (const id of this.agentOnlineState.keys()) {
+        if (!liveIds.has(id)) {
+          this.agentOnlineState.delete(id);
+          this.agentAlertState.delete(id);
+        }
+      }
+    } catch (err) {
+      console.error('Agent health check error:', err.message);
+    }
+  }
+
   archiveOldChecks() {
     try {
       const retentionSetting = db.prepare(`SELECT value FROM settings WHERE key = 'retention_days'`).get();

update-server.sh

@@ -1,52 +1,44 @@
 #!/bin/bash
-# Quick update script for SkyWatch on remote server
-# Run this on the server: bash update-server.sh
-
+# SkyWatch update script — run on the server to pull and deploy the latest version
+# Usage: bash update-server.sh [--skip-build]
 set -e
 
-echo "🔄 Updating SkyWatch..."
+APP_DIR="/opt/resource-monitor"
+SKIP_BUILD=false
+for arg in "$@"; do [[ "$arg" == "--skip-build" ]] && SKIP_BUILD=true; done
 
-cd /opt/resource-monitor
+echo "==> Updating SkyWatch at ${APP_DIR} ..."
+cd "$APP_DIR"
 
-echo "📥 Pulling latest code..."
+echo "==> Pulling latest code ..."
 git pull origin main
 
-echo "🔧 Ensuring build tools are present (needed for native modules)..."
-if command -v apt-get &>/dev/null; then
-	apt-get install -y --no-install-recommends make gcc g++ python3 2>&1 | tail -5
-elif command -v dnf &>/dev/null; then
-	dnf install -y make gcc gcc-c++ python3 2>&1 | tail -5
-elif command -v yum &>/dev/null; then
-	yum install -y make gcc gcc-c++ python3 2>&1 | tail -5
-fi
+echo "==> Installing/updating server dependencies ..."
+# Use npm install (not ci + rm -rf) so existing compatible packages stay in place
+npm install --omit=dev 2>&1 | tail -20
 
-echo "📦 Installing dependencies..."
-rm -rf node_modules
-
-# Prefer deterministic installs with npm ci when lockfile is present.
-if [ -f package-lock.json ]; then
-	if ! npm ci --omit=dev 2>&1 | tail -20; then
-		echo "npm ci failed, falling back to npm install..."
-		npm install --omit=dev 2>&1 | tail -20
-	fi
+if [ "$SKIP_BUILD" = false ]; then
+  echo "==> Building React client ..."
+  cd client
+  npm install 2>&1 | tail -10
+  npm run build 2>&1 | tail -20
+  cd ..
 else
-	echo "No package-lock.json found, using npm install..."
-	npm install --omit=dev 2>&1 | tail -20
+  echo "==> Skipping client build (--skip-build flag set)"
 fi
 
-echo "🔄 Restarting application..."
+echo "==> Restarting application via PM2 ..."
 pm2 restart resource-monitor
 
-echo "✅ Waiting for startup..."
-sleep 3
+echo "==> Waiting for startup ..."
+sleep 4
 
-echo "📋 Checking status..."
+echo "==> Status:"
 pm2 status resource-monitor
 
 echo ""
-echo "📊 Recent logs (checking for database migration):"
-pm2 logs resource-monitor --lines 30 --nostream
+echo "==> Health check:"
+curl -s http://localhost:3001/api/health | grep -o '"status":"[^"]*"' || echo "(no response yet)"
 
 echo ""
 echo "✅ Update complete!"
-echo "🌐 Check your dashboard at http://$(hostname -I | awk '{print $1}'):3001"