diff --git a/package.json b/package.json index 4a3093d..c0ce351 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "license": "MIT", "dependencies": { "axios": "^1.6.2", - "better-sqlite3": "^13.0.3", + "better-sqlite3": "9.6.0", "cors": "^2.8.5", "dotenv": "^16.3.1", "express": "^4.18.2", diff --git a/server/notificationService.js b/server/notificationService.js index c265a3e..4bf2e33 100644 --- a/server/notificationService.js +++ b/server/notificationService.js @@ -446,6 +446,77 @@ class NotificationService { } } + async sendAgentAlert(agent, type, message) { + const isOffline = type === 'offline'; + const isOnline = type === 'online'; + const emoji = isOffline ? '🔴' : isOnline ? '🟢' : '⚠️'; + const title = isOffline ? `Agent Offline: ${agent.name}` + : isOnline ? `Agent Online: ${agent.name}` + : `Agent Alert: ${agent.name}`; + const notifType = isOffline ? 'down' : isOnline ? 'up' : 'warning'; + + // In-app notification + try { + const db = require('./database'); + const result = db.prepare(` + INSERT INTO notifications (resource_id, incident_id, type, title, message) + VALUES (NULL, NULL, ?, ?, ?) + `).run(notifType, title, message); + if (global.broadcastNotification) { + global.broadcastNotification({ + id: result.lastInsertRowid, resource_id: null, resource_name: agent.name, + incident_id: null, type: notifType, title, message, read: 0, + created_at: new Date().toISOString(), + }); + } + } catch (_) {} + + // Email + if (this.emailEnabled && this.transporter && this.config.email_to) { + try { + await this.transporter.sendMail({ + from: this.config.email_from, + to: this.config.email_to, + subject: `${emoji} SkyWatch Agent: ${title}`, + text: [ + message, + '', + `Agent: ${agent.name}`, + `Hostname: ${agent.hostname || '—'}`, + `IP: ${agent.ip_address || '—'}`, + `OS: ${agent.os_info || '—'}`, + `Time: ${new Date().toLocaleString()}`, + ].join('\n'), + }); + } catch (_) {} + } + + // Webhook + if (this.webhookEnabled && this.config.webhook_url) { + try { + await axios.post(this.config.webhook_url, { + type: 'agent', event: type, + agent: { id: agent.id, name: agent.name, hostname: agent.hostname, ip_address: agent.ip_address }, + title, message, timestamp: new Date().toISOString(), + }); + } catch (_) {} + } + + // Ntfy + if (this.ntfyEnabled && this.config.ntfy_topic) { + const ntfyBase = (this.config.ntfy_url || 'https://ntfy.sh').replace(/\/$/, ''); + try { + await axios.post(`${ntfyBase}/`, { + topic: this.config.ntfy_topic, + title: `${emoji} ${title}`, + message, + priority: isOffline ? 4 : isOnline ? 2 : 3, + tags: isOffline ? ['red_circle'] : isOnline ? ['green_circle'] : ['warning'], + }); + } catch (_) {} + } + } + async sendNtfy(resource, type) { const ntfyUrl = this.config.ntfy_url || 'https://ntfy.sh'; const ntfyTopic = this.config.ntfy_topic; diff --git a/server/scheduler.js b/server/scheduler.js index 18ea34d..36dc1cb 100644 --- a/server/scheduler.js +++ b/server/scheduler.js @@ -3,13 +3,21 @@ const db = require('./database'); const monitorService = require('./monitorService'); const notificationService = require('./notificationService'); +const AGENT_CPU_THRESHOLD = 90; +const AGENT_MEM_THRESHOLD = 90; +const AGENT_DISK_THRESHOLD = 90; +const AGENT_OFFLINE_MS = 3 * 60 * 1000; // matches UI isOnline() check + class Scheduler { constructor() { this.jobs = new Map(); this.isRunning = false; this.lastCheckedAt = new Map(); this.slowAlertCooldown = new Map(); - this.slowConsecutiveCount = new Map(); // tracks consecutive slow checks per resource + this.slowConsecutiveCount = new Map(); + // Agent health tracking — keyed by agent id + this.agentOnlineState = new Map(); // id -> boolean + this.agentAlertState = new Map(); // id -> { cpu, mem, disk } } start() { @@ -25,6 +33,11 @@ class Scheduler { } }); + // Agent health check every 2 minutes + cron.schedule('*/2 * * * *', async () => { + await this.checkAgents(); + }); + // Archive old checks + update baselines + check escalations daily at 2 AM cron.schedule('0 2 * * *', () => { this.archiveOldChecks(); @@ -43,6 +56,111 @@ class Scheduler { }, 5000); } + async checkAgents() { + try { + const rows = db.prepare(` + SELECT + a.id, a.name, a.hostname, a.ip_address, a.os_info, a.last_seen_at, + m.cpu_percent, m.mem_percent, m.disk_data + FROM agents a + LEFT JOIN agent_metrics m ON m.id = ( + SELECT id FROM agent_metrics WHERE agent_id = a.id ORDER BY recorded_at DESC LIMIT 1 + ) + ORDER BY a.id + `).all(); + + const liveIds = new Set(); + + for (const agent of rows) { + liveIds.add(agent.id); + + const isOnline = agent.last_seen_at + ? Date.now() - new Date(agent.last_seen_at).getTime() < AGENT_OFFLINE_MS + : false; + + const wasOnline = this.agentOnlineState.get(agent.id); + + if (wasOnline === undefined) { + // First poll — seed state without alerting + this.agentOnlineState.set(agent.id, isOnline); + this.agentAlertState.set(agent.id, { cpu: false, mem: false, disk: false }); + continue; + } + + // Online/offline transitions + if (wasOnline && !isOnline) { + await notificationService.sendAgentAlert( + agent, 'offline', + `Agent "${agent.name}" (${agent.hostname || agent.ip_address || 'unknown'}) has gone offline.` + ); + } else if (!wasOnline && isOnline) { + await notificationService.sendAgentAlert( + agent, 'online', + `Agent "${agent.name}" (${agent.hostname || agent.ip_address || 'unknown'}) is back online.` + ); + this.agentAlertState.set(agent.id, { cpu: false, mem: false, disk: false }); + } + + this.agentOnlineState.set(agent.id, isOnline); + + if (!isOnline || agent.cpu_percent == null) continue; + + const as = this.agentAlertState.get(agent.id) || { cpu: false, mem: false, disk: false }; + + // CPU + if (agent.cpu_percent > AGENT_CPU_THRESHOLD && !as.cpu) { + await notificationService.sendAgentAlert( + agent, 'cpu', + `Agent "${agent.name}" CPU is at ${Number(agent.cpu_percent).toFixed(1)}% (threshold: ${AGENT_CPU_THRESHOLD}%).` + ); + as.cpu = true; + } else if (agent.cpu_percent <= AGENT_CPU_THRESHOLD) { + as.cpu = false; + } + + // Memory + if (agent.mem_percent > AGENT_MEM_THRESHOLD && !as.mem) { + await notificationService.sendAgentAlert( + agent, 'memory', + `Agent "${agent.name}" memory is at ${Number(agent.mem_percent).toFixed(1)}% (threshold: ${AGENT_MEM_THRESHOLD}%).` + ); + as.mem = true; + } else if (agent.mem_percent <= AGENT_MEM_THRESHOLD) { + as.mem = false; + } + + // Disk — check all partitions + if (agent.disk_data) { + try { + const disks = JSON.parse(agent.disk_data); + const worst = disks.reduce((m, d) => (d.percent > (m?.percent ?? 0) ? d : m), null); + if (worst && worst.percent > AGENT_DISK_THRESHOLD && !as.disk) { + await notificationService.sendAgentAlert( + agent, 'disk', + `Agent "${agent.name}" disk ${worst.path} is at ${worst.percent}% (threshold: ${AGENT_DISK_THRESHOLD}%).` + ); + as.disk = true; + } else if (!worst || worst.percent <= AGENT_DISK_THRESHOLD) { + as.disk = false; + } + } catch (_) {} + } + + this.agentAlertState.set(agent.id, as); + } + + // Remove state for deleted agents + for (const id of this.agentOnlineState.keys()) { + if (!liveIds.has(id)) { + this.agentOnlineState.delete(id); + this.agentAlertState.delete(id); + } + } + } catch (err) { + console.error('Agent health check error:', err.message); + } + } + archiveOldChecks() { try { const retentionSetting = db.prepare(`SELECT value FROM settings WHERE key = 'retention_days'`).get(); diff --git a/update-server.sh b/update-server.sh index 4b0764a..7b4b7c5 100644 --- a/update-server.sh +++ b/update-server.sh @@ -1,52 +1,44 @@ #!/bin/bash -# Quick update script for SkyWatch on remote server -# Run this on the server: bash update-server.sh - +# SkyWatch update script — run on the server to pull and deploy the latest version +# Usage: bash update-server.sh [--skip-build] set -e -echo "🔄 Updating SkyWatch..." +APP_DIR="/opt/resource-monitor" +SKIP_BUILD=false +for arg in "$@"; do [[ "$arg" == "--skip-build" ]] && SKIP_BUILD=true; done -cd /opt/resource-monitor +echo "==> Updating SkyWatch at ${APP_DIR} ..." +cd "$APP_DIR" -echo "📥 Pulling latest code..." +echo "==> Pulling latest code ..." git pull origin main -echo "🔧 Ensuring build tools are present (needed for native modules)..." -if command -v apt-get &>/dev/null; then - apt-get install -y --no-install-recommends make gcc g++ python3 2>&1 | tail -5 -elif command -v dnf &>/dev/null; then - dnf install -y make gcc gcc-c++ python3 2>&1 | tail -5 -elif command -v yum &>/dev/null; then - yum install -y make gcc gcc-c++ python3 2>&1 | tail -5 -fi +echo "==> Installing/updating server dependencies ..." +# Use npm install (not ci + rm -rf) so existing compatible packages stay in place +npm install --omit=dev 2>&1 | tail -20 -echo "📦 Installing dependencies..." -rm -rf node_modules - -# Prefer deterministic installs with npm ci when lockfile is present. -if [ -f package-lock.json ]; then - if ! npm ci --omit=dev 2>&1 | tail -20; then - echo "npm ci failed, falling back to npm install..." - npm install --omit=dev 2>&1 | tail -20 - fi +if [ "$SKIP_BUILD" = false ]; then + echo "==> Building React client ..." + cd client + npm install 2>&1 | tail -10 + npm run build 2>&1 | tail -20 + cd .. else - echo "No package-lock.json found, using npm install..." - npm install --omit=dev 2>&1 | tail -20 + echo "==> Skipping client build (--skip-build flag set)" fi -echo "🔄 Restarting application..." +echo "==> Restarting application via PM2 ..." pm2 restart resource-monitor -echo "✅ Waiting for startup..." -sleep 3 +echo "==> Waiting for startup ..." +sleep 4 -echo "📋 Checking status..." +echo "==> Status:" pm2 status resource-monitor echo "" -echo "📊 Recent logs (checking for database migration):" -pm2 logs resource-monitor --lines 30 --nostream +echo "==> Health check:" +curl -s http://localhost:3001/api/health | grep -o '"status":"[^"]*"' || echo "(no response yet)" echo "" echo "✅ Update complete!" -echo "🌐 Check your dashboard at http://$(hostname -I | awk '{print $1}'):3001"