Add full /data backup export/import for server migration

- New admin endpoints: GET /api/admin-backup/export streams the entire data directory as tar.gz (after flushing queued writes); POST /api/admin-backup/import validates the archive, snapshots current data, replaces the folder, and reloads all in-memory state. - Admin UI: "Full Data Backup" section with download and restore-from- file controls in the Analytics panel. - Fix admin TOTP secret path to honor SITEFORGE_DATA_DIR (moved base path resolution to server/paths.js to avoid a circular import), so 2FA for admin and study users survives backup/restore. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-07 12:08:23 -04:00
parent 17c9cbbc8b
commit f4fd177421
9 changed files with 371 additions and 17 deletions
@@ -29,7 +29,8 @@
        "react-router-dom": "^7.13.1",
        "react-simple-maps": "^3.0.0",
        "remark-gfm": "^4.0.1",
-        "resend": "^6.10.0"
+        "resend": "^6.10.0",
+        "tar": "^7.5.19"
      },
      "devDependencies": {
        "@eslint/js": "^9.39.4",
@@ -611,6 +612,18 @@
        "url": "https://github.com/sponsors/nzakas"
      }
    },
+    "node_modules/@isaacs/fs-minipass": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz",
+      "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==",
+      "license": "ISC",
+      "dependencies": {
+        "minipass": "^7.0.4"
+      },
+      "engines": {
+        "node": ">=18.0.0"
+      }
+    },
    "node_modules/@jridgewell/gen-mapping": {
      "version": "0.3.13",
      "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz",
@@ -2288,6 +2301,15 @@
        "pnpm": ">=8"
      }
    },
+    "node_modules/chownr": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz",
+      "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==",
+      "license": "BlueOak-1.0.0",
+      "engines": {
+        "node": ">=18"
+      }
+    },
    "node_modules/cliui": {
      "version": "8.0.1",
      "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",
@@ -5112,6 +5134,27 @@
        "node": "*"
      }
    },
+    "node_modules/minipass": {
+      "version": "7.1.3",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz",
+      "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==",
+      "license": "BlueOak-1.0.0",
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      }
+    },
+    "node_modules/minizlib": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz",
+      "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==",
+      "license": "MIT",
+      "dependencies": {
+        "minipass": "^7.1.2"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
    "node_modules/ms": {
      "version": "2.1.3",
      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
@@ -6496,6 +6539,31 @@
        "uuid": "^10.0.0"
      }
    },
+    "node_modules/tar": {
+      "version": "7.5.19",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.19.tgz",
+      "integrity": "sha512-4LeEWl96twnS2Q7Bz4MGqgazLqO+hJN63GZxXoIqh1T3VweYD997gbU1ItNsQafqqXTXd5WFyFdReLtwvRBNiw==",
+      "license": "BlueOak-1.0.0",
+      "dependencies": {
+        "@isaacs/fs-minipass": "^4.0.0",
+        "chownr": "^3.0.0",
+        "minipass": "^7.1.2",
+        "minizlib": "^3.1.0",
+        "yallist": "^5.0.0"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/tar/node_modules/yallist": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz",
+      "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==",
+      "license": "BlueOak-1.0.0",
+      "engines": {
+        "node": ">=18"
+      }
+    },
    "node_modules/tinyglobby": {
      "version": "0.2.15",
      "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz",
@@ -34,7 +34,8 @@
    "react-router-dom": "^7.13.1",
    "react-simple-maps": "^3.0.0",
    "remark-gfm": "^4.0.1",
-    "resend": "^6.10.0"
+    "resend": "^6.10.0",
+    "tar": "^7.5.19"
  },
  "devDependencies": {
    "@eslint/js": "^9.39.4",
@@ -41,6 +41,7 @@ import { recordVisitor } from './server/routes/analytics.js'
 import { register as registerAdminAuth } from './server/routes/admin-auth.js'
 import { register as registerAdminContent } from './server/routes/admin-content.js'
 import { register as registerAdminAssets } from './server/routes/admin-assets.js'
+import { register as registerAdminBackup } from './server/routes/admin-backup.js'
 import { register as registerStudyAuth } from './server/routes/study-auth.js'
 import { register as registerStudyData } from './server/routes/study-data.js'
 import { register as registerStudyAccount } from './server/routes/study-account.js'
@@ -85,6 +86,7 @@ app.use((_req, res, next) => {
 registerAdminAuth(app)
 registerAdminContent(app)
 registerAdminAssets(app)
+registerAdminBackup(app)
 registerStudyAuth(app)
 registerStudyData(app)
 registerStudyAccount(app)
@@ -1,12 +1,10 @@
 import { createHash, randomUUID, timingSafeEqual, createHmac, randomFillSync } from 'node:crypto'
 import { readFile, writeFile } from 'node:fs/promises'
 import path from 'node:path'
-import { fileURLToPath } from 'node:url'
 import { parseCookies } from './helpers.js'
+import { DATA_DIR } from './paths.js'

-const __filename = fileURLToPath(import.meta.url)
-const __dirname = path.dirname(__filename)
-const TOTP_SECRET_FILE = path.join(__dirname, '..', 'data', 'totp-secret.json')
+const TOTP_SECRET_FILE = path.join(DATA_DIR, 'totp-secret.json')

 // Pending sessions: password verified, waiting for TOTP code
 // Map<pendingToken, { expiresAt }>
@@ -1,17 +1,8 @@
 import path from 'node:path'
-import { fileURLToPath } from 'node:url'
 import { validateAdminPasswordSetup } from './auth.js'
+import { ROOT_DIR, DATA_DIR } from './paths.js'

-const __filename = fileURLToPath(import.meta.url)
-const __dirname = path.dirname(__filename)
-// server/config.js is inside server/, so the project root is one level up
-export const ROOT_DIR = path.resolve(__dirname, '..')
-
-const DEFAULT_DATA_DIR = path.join(ROOT_DIR, 'data')
-const configuredDataDir = typeof process.env.SITEFORGE_DATA_DIR === 'string' ? process.env.SITEFORGE_DATA_DIR.trim() : ''
-export const DATA_DIR = configuredDataDir
-  ? (path.isAbsolute(configuredDataDir) ? configuredDataDir : path.resolve(ROOT_DIR, configuredDataDir))
-  : DEFAULT_DATA_DIR
+export { ROOT_DIR, DATA_DIR }

 export const DATA_FILE = path.join(DATA_DIR, 'admin-content.json')
 export const DRAFT_DATA_FILE = path.join(DATA_DIR, 'admin-content-draft.json')
@@ -0,0 +1,15 @@
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+// Base directory resolution lives in its own module (no app imports) so that
+// both config.js and auth.js can use DATA_DIR without a circular dependency.
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+// server/paths.js is inside server/, so the project root is one level up
+export const ROOT_DIR = path.resolve(__dirname, '..')
+
+const DEFAULT_DATA_DIR = path.join(ROOT_DIR, 'data')
+const configuredDataDir = typeof process.env.SITEFORGE_DATA_DIR === 'string' ? process.env.SITEFORGE_DATA_DIR.trim() : ''
+export const DATA_DIR = configuredDataDir
+  ? (path.isAbsolute(configuredDataDir) ? configuredDataDir : path.resolve(ROOT_DIR, configuredDataDir))
+  : DEFAULT_DATA_DIR
@@ -0,0 +1,193 @@
+import { mkdir, mkdtemp, readdir, rm, cp, writeFile } from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import express from 'express'
+import * as tar from 'tar'
+import { requireAdminAuth } from '../auth.js'
+import { DATA_DIR } from '../config.js'
+import { state } from '../state.js'
+import {
+  loadHitStatsFromDisk,
+  loadVisitorStatsFromDisk,
+  loadContactSubmissionsFromDisk,
+  loadReplyTemplatesFromDisk,
+  loadReplyHistoryFromDisk,
+  loadQuestionsFromDisk,
+  loadDraftQuestionsFromDisk,
+  loadStudyUsersFromDisk,
+  loadStudyCommunityFromDisk,
+  loadStudyRemindersFromDisk,
+  loadStudyCommentsFromDisk,
+  loadStudyCertificatesFromDisk,
+  loadEpisodeScriptsFromDisk,
+  migrateStudyNotesIfNeeded,
+  loadDownloadCountsFromDisk,
+  loadQrCodesFromDisk,
+  loadEpisodePlaysFromDisk,
+  loadPodcastChecklistFromDisk,
+  loadAnalyticsEventsFromDisk,
+  createBackupSnapshot,
+  refreshContentCaches,
+} from '../data.js'
+
+// Files that identify an archive as a Siteforge data backup. At least one
+// must be present at the top level of an uploaded archive before we restore.
+const KNOWN_DATA_FILES = [
+  'admin-content.json',
+  'admin-content-draft.json',
+  'study-users.json',
+  'hit-stats.json',
+  'visitor-stats.json',
+  'questions.json',
+]
+
+// Wait for every queued disk write so the archive reflects current state.
+async function flushPendingWrites() {
+  await Promise.allSettled([
+    state.hitStatsWritePromise,
+    state.visitorStatsWritePromise,
+    state.contactSubmissionsWritePromise,
+    state.questionsWritePromise,
+    state.draftQuestionsWritePromise,
+    state.replyTemplatesWritePromise,
+    state.replyHistoryWritePromise,
+    state.podcastChecklistWritePromise,
+    state.studyUsersWritePromise,
+    state.studyCommunityWritePromise,
+    state.studyRemindersWritePromise,
+    state.studyCommentsWritePromise,
+    state.studyCertificatesWritePromise,
+    state.episodeScriptsWritePromise,
+    state.downloadCountsWritePromise,
+    state.episodePlaysWritePromise,
+    state.analyticsEventsWritePromise,
+    state.qrCodesWritePromise,
+    ...state.studyNotesWriteQueues.values(),
+    ...state.studyProgressWriteQueues.values(),
+  ])
+}
+
+// Rebuild all in-memory state from whatever is now on disk (mirrors startup).
+async function reloadStateFromDisk() {
+  state.studyNotesCache.clear()
+  state.studyNotesWriteQueues.clear()
+  state.studyProgressCache.clear()
+  state.studyProgressWriteQueues.clear()
+  // Restored study users may not match current sessions — force re-login.
+  state.studySessions.clear()
+
+  await Promise.all([
+    loadHitStatsFromDisk(),
+    loadVisitorStatsFromDisk(),
+    loadContactSubmissionsFromDisk(),
+    loadReplyTemplatesFromDisk(),
+    loadReplyHistoryFromDisk(),
+    loadQuestionsFromDisk(),
+    loadDraftQuestionsFromDisk(),
+    loadStudyUsersFromDisk(),
+    loadStudyCommunityFromDisk(),
+    loadStudyRemindersFromDisk(),
+    loadStudyCommentsFromDisk(),
+    loadStudyCertificatesFromDisk(),
+    loadEpisodeScriptsFromDisk(),
+    migrateStudyNotesIfNeeded(),
+    loadDownloadCountsFromDisk(),
+    loadQrCodesFromDisk(),
+    loadEpisodePlaysFromDisk(),
+    loadPodcastChecklistFromDisk(),
+    loadAnalyticsEventsFromDisk(),
+    refreshContentCaches(),
+  ])
+}
+
+export function register(app) {
+  // Download the entire data directory as a tar.gz (excluding the automatic
+  // snapshot folder, which is derived from the other files).
+  app.get('/api/admin-backup/export', requireAdminAuth, async (_req, res) => {
+    try {
+      await flushPendingWrites()
+      const entries = (await readdir(DATA_DIR)).filter(name => name !== 'backups')
+      if (entries.length === 0) {
+        res.status(500).json({ message: 'Data directory is empty — nothing to export.' })
+        return
+      }
+
+      const stamp = new Date().toISOString().replace(/[:.]/g, '-')
+      res.setHeader('Content-Type', 'application/gzip')
+      res.setHeader('Content-Disposition', `attachment; filename="siteforge-data-${stamp}.tar.gz"`)
+
+      const archive = tar.create({ gzip: true, cwd: DATA_DIR, portable: true }, entries)
+      archive.on('error', err => {
+        console.error('[admin-backup] export stream failed:', err)
+        res.destroy(err)
+      })
+      archive.pipe(res)
+    } catch (err) {
+      console.error('[admin-backup] export failed:', err)
+      if (!res.headersSent) res.status(500).json({ message: 'Full backup export failed.' })
+    }
+  })
+
+  // Restore the entire data directory from an uploaded tar.gz produced by the
+  // export endpoint, then reload all in-memory state from the restored files.
+  app.post(
+    '/api/admin-backup/import',
+    requireAdminAuth,
+    express.raw({ type: () => true, limit: '500mb' }),
+    async (req, res) => {
+      let workDir = null
+      try {
+        const body = req.body
+        if (!Buffer.isBuffer(body) || body.length === 0) {
+          res.status(400).json({ message: 'Upload the .tar.gz file produced by the full backup export.' })
+          return
+        }
+        if (body[0] !== 0x1f || body[1] !== 0x8b) {
+          res.status(400).json({ message: 'File is not a gzip archive (.tar.gz expected).' })
+          return
+        }
+
+        workDir = await mkdtemp(path.join(os.tmpdir(), 'siteforge-import-'))
+        const archivePath = path.join(workDir, 'import.tar.gz')
+        await writeFile(archivePath, body)
+
+        const extractDir = path.join(workDir, 'extracted')
+        await mkdir(extractDir)
+        // node-tar strips absolute paths and rejects entries that escape cwd.
+        await tar.extract({ file: archivePath, cwd: extractDir })
+
+        const extractedEntries = await readdir(extractDir)
+        if (!extractedEntries.some(name => KNOWN_DATA_FILES.includes(name))) {
+          res.status(400).json({ message: 'Archive does not look like a Siteforge data backup.' })
+          return
+        }
+
+        // Settle queued writes so nothing overwrites the restored files, and
+        // keep a snapshot of the pre-import state in the backups folder.
+        await flushPendingWrites()
+        await createBackupSnapshot('pre-import')
+
+        // Replace current data with the archive contents. The snapshot folder
+        // is preserved unless the archive itself contains one.
+        const currentEntries = await readdir(DATA_DIR)
+        for (const name of currentEntries) {
+          if (name === 'backups' && !extractedEntries.includes('backups')) continue
+          await rm(path.join(DATA_DIR, name), { recursive: true, force: true })
+        }
+        for (const name of extractedEntries) {
+          await cp(path.join(extractDir, name), path.join(DATA_DIR, name), { recursive: true })
+        }
+
+        await reloadStateFromDisk()
+        await createBackupSnapshot('post-import')
+
+        res.json({ ok: true, restoredEntries: extractedEntries.length })
+      } catch (err) {
+        console.error('[admin-backup] import failed:', err)
+        res.status(500).json({ message: 'Full data restore failed. Check the server logs — data may need manual attention.' })
+      } finally {
+        if (workDir) await rm(workDir, { recursive: true, force: true }).catch(() => {})
+      }
+    },
+  )
+}
@@ -1066,6 +1066,7 @@ export default function AdminPage({ content, onSave, onLogout }: Props) {
  const [backupFiles, setBackupFiles] = useState<BackupPreview[]>([])
  const [selectedBackup, setSelectedBackup] = useState('')
  const [selectedBackupPreview, setSelectedBackupPreview] = useState<BackupPreview | null>(null)
+  const [fullBackupBusy, setFullBackupBusy] = useState(false)

  // TOTP management state
  const [totpEnabled, setTotpEnabled] = useState<boolean | null>(null)
@@ -2068,6 +2069,52 @@ export default function AdminPage({ content, onSave, onLogout }: Props) {
    }
  }

+  async function handleDownloadFullBackup() {
+    setFullBackupBusy(true)
+    setMaintenanceMsg('Preparing full backup archive…')
+    try {
+      const r = await fetch('/api/admin-backup/export')
+      if (!r.ok) throw new Error('Export failed')
+      const blob = await r.blob()
+      const disposition = r.headers.get('Content-Disposition') ?? ''
+      const filename = /filename="([^"]+)"/.exec(disposition)?.[1] ?? 'siteforge-data-backup.tar.gz'
+      const url = URL.createObjectURL(blob)
+      const link = document.createElement('a')
+      link.href = url
+      link.download = filename
+      link.click()
+      URL.revokeObjectURL(url)
+      setMaintenanceMsg(`Full backup downloaded (${filename}). Keep it somewhere safe — it contains all site data.`)
+    } catch {
+      setMaintenanceMsg('Full backup download failed.')
+    } finally {
+      setFullBackupBusy(false)
+    }
+  }
+
+  async function handleImportFullBackup(file: File) {
+    if (!confirm(`Restore ALL site data from ${file.name}?\n\nThis replaces content, uploads, study accounts, notes, questions, and analytics with the archive contents. A pre-import snapshot is saved first, and study users will need to sign in again.`)) return
+    setFullBackupBusy(true)
+    setMaintenanceMsg('Restoring full backup — do not close this page…')
+    try {
+      const r = await fetch('/api/admin-backup/import', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/gzip' },
+        body: file,
+      })
+      const data = await r.json().catch(() => null)
+      if (!r.ok) throw new Error(data?.message ?? 'Restore failed')
+      await reloadContentFromServer()
+      await reloadStats()
+      await reloadBackups()
+      setMaintenanceMsg(`Full data restore complete — ${data?.restoredEntries ?? 'all'} items restored from ${file.name}.`)
+    } catch (err) {
+      setMaintenanceMsg(err instanceof Error ? err.message : 'Full data restore failed.')
+    } finally {
+      setFullBackupBusy(false)
+    }
+  }
+
  async function handleRestoreBackup() {
    if (!selectedBackup) {
      setMaintenanceMsg('Select a backup first.')
@@ -5025,6 +5072,9 @@ export default function AdminPage({ content, onSave, onLogout }: Props) {
              onRestore={handleRestoreBackup}
              onExport={handleExport}
              onBackupNow={handleBackupNow}
+              onDownloadFullBackup={handleDownloadFullBackup}
+              onImportFullBackup={handleImportFullBackup}
+              fullBackupBusy={fullBackupBusy}
              onPrune={handlePrune}
              onClear={handleClear}
              onPurgeCache={handlePurgeCache}
@@ -43,6 +43,9 @@ interface Props {
  onRestore: () => void
  onExport: () => void
  onBackupNow: () => void
+  onDownloadFullBackup: () => void
+  onImportFullBackup: (file: File) => void
+  fullBackupBusy: boolean
  onPrune: () => void
  onClear: () => void
  onPurgeCache: () => void
@@ -66,6 +69,9 @@ export function AnalyticsPanel({
  onRestore,
  onExport,
  onBackupNow,
+  onDownloadFullBackup,
+  onImportFullBackup,
+  fullBackupBusy,
  onPrune,
  onClear,
  onPurgeCache,
@@ -74,6 +80,7 @@ export function AnalyticsPanel({
 }: Props) {
  const [chartTab, setChartTab] = useState<'overview' | 'breakdown' | 'geographic' | 'referrers'>('overview')
  const [expandedVisitor, setExpandedVisitor] = useState<string | null>(null)
+  const [fullBackupFile, setFullBackupFile] = useState<File | null>(null)

  if (statsStatus === 'loading') return <p className="admin-stats-note">Loading analytics…</p>
  if (statsStatus === 'error') return <p className="admin-stats-note">Failed to load analytics.</p>
@@ -807,6 +814,35 @@ export function AnalyticsPanel({
          <p><strong>Total Visits:</strong> {selectedBackupPreview.totalVisits.toLocaleString()}</p>
        </div>
      )}
+
+      {/* Full Data Backup (entire /data folder — server migration) */}
+      <div className="admin-stats-head admin-stats-head--visitors">
+        <h2>Full Data Backup</h2>
+        <p>Download every persisted file — site content, uploads, study accounts, notes, progress, questions, analytics — as one archive, or restore that archive on this or a new server.</p>
+      </div>
+      <div className="admin-actions admin-actions--maintenance">
+        <button type="button" className="btn-admin-reset" onClick={onDownloadFullBackup} disabled={fullBackupBusy}>
+          {fullBackupBusy ? 'Working…' : 'Download Full Backup (.tar.gz)'}
+        </button>
+      </div>
+      <div className="admin-restore-row">
+        <label htmlFor="full-backup-file">Restore from File</label>
+        <input
+          id="full-backup-file"
+          type="file"
+          accept=".tar.gz,.tgz,application/gzip,application/x-gzip"
+          onChange={e => setFullBackupFile(e.target.files?.[0] ?? null)}
+          disabled={fullBackupBusy}
+        />
+        <button
+          type="button"
+          className="btn-admin-remove"
+          onClick={() => { if (fullBackupFile) onImportFullBackup(fullBackupFile) }}
+          disabled={!fullBackupFile || fullBackupBusy}
+        >
+          {fullBackupBusy ? 'Working…' : 'Restore Full Backup'}
+        </button>
+      </div>
      {maintenanceMsg && <p className="admin-stats-note">{maintenanceMsg}</p>}
    </section>
  )