Add full /data backup export/import for server migration

- New admin endpoints: GET /api/admin-backup/export streams the entire data directory as tar.gz (after flushing queued writes); POST /api/admin-backup/import validates the archive, snapshots current data, replaces the folder, and reloads all in-memory state. - Admin UI: "Full Data Backup" section with download and restore-from- file controls in the Analytics panel. - Fix admin TOTP secret path to honor SITEFORGE_DATA_DIR (moved base path resolution to server/paths.js to avoid a circular import), so 2FA for admin and study users survives backup/restore. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-07 12:08:23 -04:00
parent 17c9cbbc8b
commit f4fd177421
9 changed files with 371 additions and 17 deletions
@@ -1,12 +1,10 @@
 import { createHash, randomUUID, timingSafeEqual, createHmac, randomFillSync } from 'node:crypto'
 import { readFile, writeFile } from 'node:fs/promises'
 import path from 'node:path'
-import { fileURLToPath } from 'node:url'
 import { parseCookies } from './helpers.js'
+import { DATA_DIR } from './paths.js'

-const __filename = fileURLToPath(import.meta.url)
-const __dirname = path.dirname(__filename)
-const TOTP_SECRET_FILE = path.join(__dirname, '..', 'data', 'totp-secret.json')
+const TOTP_SECRET_FILE = path.join(DATA_DIR, 'totp-secret.json')

 // Pending sessions: password verified, waiting for TOTP code
 // Map<pendingToken, { expiresAt }>
@@ -1,17 +1,8 @@
 import path from 'node:path'
-import { fileURLToPath } from 'node:url'
 import { validateAdminPasswordSetup } from './auth.js'
+import { ROOT_DIR, DATA_DIR } from './paths.js'

-const __filename = fileURLToPath(import.meta.url)
-const __dirname = path.dirname(__filename)
-// server/config.js is inside server/, so the project root is one level up
-export const ROOT_DIR = path.resolve(__dirname, '..')
-
-const DEFAULT_DATA_DIR = path.join(ROOT_DIR, 'data')
-const configuredDataDir = typeof process.env.SITEFORGE_DATA_DIR === 'string' ? process.env.SITEFORGE_DATA_DIR.trim() : ''
-export const DATA_DIR = configuredDataDir
-  ? (path.isAbsolute(configuredDataDir) ? configuredDataDir : path.resolve(ROOT_DIR, configuredDataDir))
-  : DEFAULT_DATA_DIR
+export { ROOT_DIR, DATA_DIR }

 export const DATA_FILE = path.join(DATA_DIR, 'admin-content.json')
 export const DRAFT_DATA_FILE = path.join(DATA_DIR, 'admin-content-draft.json')
@@ -0,0 +1,15 @@
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+// Base directory resolution lives in its own module (no app imports) so that
+// both config.js and auth.js can use DATA_DIR without a circular dependency.
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+// server/paths.js is inside server/, so the project root is one level up
+export const ROOT_DIR = path.resolve(__dirname, '..')
+
+const DEFAULT_DATA_DIR = path.join(ROOT_DIR, 'data')
+const configuredDataDir = typeof process.env.SITEFORGE_DATA_DIR === 'string' ? process.env.SITEFORGE_DATA_DIR.trim() : ''
+export const DATA_DIR = configuredDataDir
+  ? (path.isAbsolute(configuredDataDir) ? configuredDataDir : path.resolve(ROOT_DIR, configuredDataDir))
+  : DEFAULT_DATA_DIR
@@ -0,0 +1,193 @@
+import { mkdir, mkdtemp, readdir, rm, cp, writeFile } from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import express from 'express'
+import * as tar from 'tar'
+import { requireAdminAuth } from '../auth.js'
+import { DATA_DIR } from '../config.js'
+import { state } from '../state.js'
+import {
+  loadHitStatsFromDisk,
+  loadVisitorStatsFromDisk,
+  loadContactSubmissionsFromDisk,
+  loadReplyTemplatesFromDisk,
+  loadReplyHistoryFromDisk,
+  loadQuestionsFromDisk,
+  loadDraftQuestionsFromDisk,
+  loadStudyUsersFromDisk,
+  loadStudyCommunityFromDisk,
+  loadStudyRemindersFromDisk,
+  loadStudyCommentsFromDisk,
+  loadStudyCertificatesFromDisk,
+  loadEpisodeScriptsFromDisk,
+  migrateStudyNotesIfNeeded,
+  loadDownloadCountsFromDisk,
+  loadQrCodesFromDisk,
+  loadEpisodePlaysFromDisk,
+  loadPodcastChecklistFromDisk,
+  loadAnalyticsEventsFromDisk,
+  createBackupSnapshot,
+  refreshContentCaches,
+} from '../data.js'
+
+// Files that identify an archive as a Siteforge data backup. At least one
+// must be present at the top level of an uploaded archive before we restore.
+const KNOWN_DATA_FILES = [
+  'admin-content.json',
+  'admin-content-draft.json',
+  'study-users.json',
+  'hit-stats.json',
+  'visitor-stats.json',
+  'questions.json',
+]
+
+// Wait for every queued disk write so the archive reflects current state.
+async function flushPendingWrites() {
+  await Promise.allSettled([
+    state.hitStatsWritePromise,
+    state.visitorStatsWritePromise,
+    state.contactSubmissionsWritePromise,
+    state.questionsWritePromise,
+    state.draftQuestionsWritePromise,
+    state.replyTemplatesWritePromise,
+    state.replyHistoryWritePromise,
+    state.podcastChecklistWritePromise,
+    state.studyUsersWritePromise,
+    state.studyCommunityWritePromise,
+    state.studyRemindersWritePromise,
+    state.studyCommentsWritePromise,
+    state.studyCertificatesWritePromise,
+    state.episodeScriptsWritePromise,
+    state.downloadCountsWritePromise,
+    state.episodePlaysWritePromise,
+    state.analyticsEventsWritePromise,
+    state.qrCodesWritePromise,
+    ...state.studyNotesWriteQueues.values(),
+    ...state.studyProgressWriteQueues.values(),
+  ])
+}
+
+// Rebuild all in-memory state from whatever is now on disk (mirrors startup).
+async function reloadStateFromDisk() {
+  state.studyNotesCache.clear()
+  state.studyNotesWriteQueues.clear()
+  state.studyProgressCache.clear()
+  state.studyProgressWriteQueues.clear()
+  // Restored study users may not match current sessions — force re-login.
+  state.studySessions.clear()
+
+  await Promise.all([
+    loadHitStatsFromDisk(),
+    loadVisitorStatsFromDisk(),
+    loadContactSubmissionsFromDisk(),
+    loadReplyTemplatesFromDisk(),
+    loadReplyHistoryFromDisk(),
+    loadQuestionsFromDisk(),
+    loadDraftQuestionsFromDisk(),
+    loadStudyUsersFromDisk(),
+    loadStudyCommunityFromDisk(),
+    loadStudyRemindersFromDisk(),
+    loadStudyCommentsFromDisk(),
+    loadStudyCertificatesFromDisk(),
+    loadEpisodeScriptsFromDisk(),
+    migrateStudyNotesIfNeeded(),
+    loadDownloadCountsFromDisk(),
+    loadQrCodesFromDisk(),
+    loadEpisodePlaysFromDisk(),
+    loadPodcastChecklistFromDisk(),
+    loadAnalyticsEventsFromDisk(),
+    refreshContentCaches(),
+  ])
+}
+
+export function register(app) {
+  // Download the entire data directory as a tar.gz (excluding the automatic
+  // snapshot folder, which is derived from the other files).
+  app.get('/api/admin-backup/export', requireAdminAuth, async (_req, res) => {
+    try {
+      await flushPendingWrites()
+      const entries = (await readdir(DATA_DIR)).filter(name => name !== 'backups')
+      if (entries.length === 0) {
+        res.status(500).json({ message: 'Data directory is empty — nothing to export.' })
+        return
+      }
+
+      const stamp = new Date().toISOString().replace(/[:.]/g, '-')
+      res.setHeader('Content-Type', 'application/gzip')
+      res.setHeader('Content-Disposition', `attachment; filename="siteforge-data-${stamp}.tar.gz"`)
+
+      const archive = tar.create({ gzip: true, cwd: DATA_DIR, portable: true }, entries)
+      archive.on('error', err => {
+        console.error('[admin-backup] export stream failed:', err)
+        res.destroy(err)
+      })
+      archive.pipe(res)
+    } catch (err) {
+      console.error('[admin-backup] export failed:', err)
+      if (!res.headersSent) res.status(500).json({ message: 'Full backup export failed.' })
+    }
+  })
+
+  // Restore the entire data directory from an uploaded tar.gz produced by the
+  // export endpoint, then reload all in-memory state from the restored files.
+  app.post(
+    '/api/admin-backup/import',
+    requireAdminAuth,
+    express.raw({ type: () => true, limit: '500mb' }),
+    async (req, res) => {
+      let workDir = null
+      try {
+        const body = req.body
+        if (!Buffer.isBuffer(body) || body.length === 0) {
+          res.status(400).json({ message: 'Upload the .tar.gz file produced by the full backup export.' })
+          return
+        }
+        if (body[0] !== 0x1f || body[1] !== 0x8b) {
+          res.status(400).json({ message: 'File is not a gzip archive (.tar.gz expected).' })
+          return
+        }
+
+        workDir = await mkdtemp(path.join(os.tmpdir(), 'siteforge-import-'))
+        const archivePath = path.join(workDir, 'import.tar.gz')
+        await writeFile(archivePath, body)
+
+        const extractDir = path.join(workDir, 'extracted')
+        await mkdir(extractDir)
+        // node-tar strips absolute paths and rejects entries that escape cwd.
+        await tar.extract({ file: archivePath, cwd: extractDir })
+
+        const extractedEntries = await readdir(extractDir)
+        if (!extractedEntries.some(name => KNOWN_DATA_FILES.includes(name))) {
+          res.status(400).json({ message: 'Archive does not look like a Siteforge data backup.' })
+          return
+        }
+
+        // Settle queued writes so nothing overwrites the restored files, and
+        // keep a snapshot of the pre-import state in the backups folder.
+        await flushPendingWrites()
+        await createBackupSnapshot('pre-import')
+
+        // Replace current data with the archive contents. The snapshot folder
+        // is preserved unless the archive itself contains one.
+        const currentEntries = await readdir(DATA_DIR)
+        for (const name of currentEntries) {
+          if (name === 'backups' && !extractedEntries.includes('backups')) continue
+          await rm(path.join(DATA_DIR, name), { recursive: true, force: true })
+        }
+        for (const name of extractedEntries) {
+          await cp(path.join(extractDir, name), path.join(DATA_DIR, name), { recursive: true })
+        }
+
+        await reloadStateFromDisk()
+        await createBackupSnapshot('post-import')
+
+        res.json({ ok: true, restoredEntries: extractedEntries.length })
+      } catch (err) {
+        console.error('[admin-backup] import failed:', err)
+        res.status(500).json({ message: 'Full data restore failed. Check the server logs — data may need manual attention.' })
+      } finally {
+        if (workDir) await rm(workDir, { recursive: true, force: true }).catch(() => {})
+      }
+    },
+  )
+}