From f4fd1774213712a55af353ceea19276ff6b2a9c0 Mon Sep 17 00:00:00 2001 From: nmemmert Date: Tue, 7 Jul 2026 12:08:23 -0400 Subject: [PATCH] Add full /data backup export/import for server migration - New admin endpoints: GET /api/admin-backup/export streams the entire data directory as tar.gz (after flushing queued writes); POST /api/admin-backup/import validates the archive, snapshots current data, replaces the folder, and reloads all in-memory state. - Admin UI: "Full Data Backup" section with download and restore-from- file controls in the Analytics panel. - Fix admin TOTP secret path to honor SITEFORGE_DATA_DIR (moved base path resolution to server/paths.js to avoid a circular import), so 2FA for admin and study users survives backup/restore. Co-Authored-By: Claude Fable 5 --- package-lock.json | 70 ++++++++++- package.json | 3 +- server.js | 2 + server/auth.js | 6 +- server/config.js | 13 +- server/paths.js | 15 +++ server/routes/admin-backup.js | 193 ++++++++++++++++++++++++++++++ src/AdminPage.tsx | 50 ++++++++ src/components/AnalyticsPanel.tsx | 36 ++++++ 9 files changed, 371 insertions(+), 17 deletions(-) create mode 100644 server/paths.js create mode 100644 server/routes/admin-backup.js diff --git a/package-lock.json b/package-lock.json index d4683a4..d869316 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,7 +29,8 @@ "react-router-dom": "^7.13.1", "react-simple-maps": "^3.0.0", "remark-gfm": "^4.0.1", - "resend": "^6.10.0" + "resend": "^6.10.0", + "tar": "^7.5.19" }, "devDependencies": { "@eslint/js": "^9.39.4", @@ -611,6 +612,18 @@ "url": "https://github.com/sponsors/nzakas" } }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "license": "ISC", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -2288,6 +2301,15 @@ "pnpm": ">=8" } }, + "node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -5112,6 +5134,27 @@ "node": "*" } }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/minizlib": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", + "license": "MIT", + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">= 18" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -6496,6 +6539,31 @@ "uuid": "^10.0.0" } }, + "node_modules/tar": { + "version": "7.5.19", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.19.tgz", + "integrity": "sha512-4LeEWl96twnS2Q7Bz4MGqgazLqO+hJN63GZxXoIqh1T3VweYD997gbU1ItNsQafqqXTXd5WFyFdReLtwvRBNiw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/tar/node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/tinyglobby": { "version": "0.2.15", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", diff --git a/package.json b/package.json index f6964e3..2594b4f 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,8 @@ "react-router-dom": "^7.13.1", "react-simple-maps": "^3.0.0", "remark-gfm": "^4.0.1", - "resend": "^6.10.0" + "resend": "^6.10.0", + "tar": "^7.5.19" }, "devDependencies": { "@eslint/js": "^9.39.4", diff --git a/server.js b/server.js index 97851fe..c08ad34 100644 --- a/server.js +++ b/server.js @@ -41,6 +41,7 @@ import { recordVisitor } from './server/routes/analytics.js' import { register as registerAdminAuth } from './server/routes/admin-auth.js' import { register as registerAdminContent } from './server/routes/admin-content.js' import { register as registerAdminAssets } from './server/routes/admin-assets.js' +import { register as registerAdminBackup } from './server/routes/admin-backup.js' import { register as registerStudyAuth } from './server/routes/study-auth.js' import { register as registerStudyData } from './server/routes/study-data.js' import { register as registerStudyAccount } from './server/routes/study-account.js' @@ -85,6 +86,7 @@ app.use((_req, res, next) => { registerAdminAuth(app) registerAdminContent(app) registerAdminAssets(app) +registerAdminBackup(app) registerStudyAuth(app) registerStudyData(app) registerStudyAccount(app) diff --git a/server/auth.js b/server/auth.js index cad3561..48c1243 100644 --- a/server/auth.js +++ b/server/auth.js @@ -1,12 +1,10 @@ import { createHash, randomUUID, timingSafeEqual, createHmac, randomFillSync } from 'node:crypto' import { readFile, writeFile } from 'node:fs/promises' import path from 'node:path' -import { fileURLToPath } from 'node:url' import { parseCookies } from './helpers.js' +import { DATA_DIR } from './paths.js' -const __filename = fileURLToPath(import.meta.url) -const __dirname = path.dirname(__filename) -const TOTP_SECRET_FILE = path.join(__dirname, '..', 'data', 'totp-secret.json') +const TOTP_SECRET_FILE = path.join(DATA_DIR, 'totp-secret.json') // Pending sessions: password verified, waiting for TOTP code // Map diff --git a/server/config.js b/server/config.js index 20c1e24..23fcb75 100644 --- a/server/config.js +++ b/server/config.js @@ -1,17 +1,8 @@ import path from 'node:path' -import { fileURLToPath } from 'node:url' import { validateAdminPasswordSetup } from './auth.js' +import { ROOT_DIR, DATA_DIR } from './paths.js' -const __filename = fileURLToPath(import.meta.url) -const __dirname = path.dirname(__filename) -// server/config.js is inside server/, so the project root is one level up -export const ROOT_DIR = path.resolve(__dirname, '..') - -const DEFAULT_DATA_DIR = path.join(ROOT_DIR, 'data') -const configuredDataDir = typeof process.env.SITEFORGE_DATA_DIR === 'string' ? process.env.SITEFORGE_DATA_DIR.trim() : '' -export const DATA_DIR = configuredDataDir - ? (path.isAbsolute(configuredDataDir) ? configuredDataDir : path.resolve(ROOT_DIR, configuredDataDir)) - : DEFAULT_DATA_DIR +export { ROOT_DIR, DATA_DIR } export const DATA_FILE = path.join(DATA_DIR, 'admin-content.json') export const DRAFT_DATA_FILE = path.join(DATA_DIR, 'admin-content-draft.json') diff --git a/server/paths.js b/server/paths.js new file mode 100644 index 0000000..f4336a5 --- /dev/null +++ b/server/paths.js @@ -0,0 +1,15 @@ +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +// Base directory resolution lives in its own module (no app imports) so that +// both config.js and auth.js can use DATA_DIR without a circular dependency. +const __filename = fileURLToPath(import.meta.url) +const __dirname = path.dirname(__filename) +// server/paths.js is inside server/, so the project root is one level up +export const ROOT_DIR = path.resolve(__dirname, '..') + +const DEFAULT_DATA_DIR = path.join(ROOT_DIR, 'data') +const configuredDataDir = typeof process.env.SITEFORGE_DATA_DIR === 'string' ? process.env.SITEFORGE_DATA_DIR.trim() : '' +export const DATA_DIR = configuredDataDir + ? (path.isAbsolute(configuredDataDir) ? configuredDataDir : path.resolve(ROOT_DIR, configuredDataDir)) + : DEFAULT_DATA_DIR diff --git a/server/routes/admin-backup.js b/server/routes/admin-backup.js new file mode 100644 index 0000000..d23c8b2 --- /dev/null +++ b/server/routes/admin-backup.js @@ -0,0 +1,193 @@ +import { mkdir, mkdtemp, readdir, rm, cp, writeFile } from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import express from 'express' +import * as tar from 'tar' +import { requireAdminAuth } from '../auth.js' +import { DATA_DIR } from '../config.js' +import { state } from '../state.js' +import { + loadHitStatsFromDisk, + loadVisitorStatsFromDisk, + loadContactSubmissionsFromDisk, + loadReplyTemplatesFromDisk, + loadReplyHistoryFromDisk, + loadQuestionsFromDisk, + loadDraftQuestionsFromDisk, + loadStudyUsersFromDisk, + loadStudyCommunityFromDisk, + loadStudyRemindersFromDisk, + loadStudyCommentsFromDisk, + loadStudyCertificatesFromDisk, + loadEpisodeScriptsFromDisk, + migrateStudyNotesIfNeeded, + loadDownloadCountsFromDisk, + loadQrCodesFromDisk, + loadEpisodePlaysFromDisk, + loadPodcastChecklistFromDisk, + loadAnalyticsEventsFromDisk, + createBackupSnapshot, + refreshContentCaches, +} from '../data.js' + +// Files that identify an archive as a Siteforge data backup. At least one +// must be present at the top level of an uploaded archive before we restore. +const KNOWN_DATA_FILES = [ + 'admin-content.json', + 'admin-content-draft.json', + 'study-users.json', + 'hit-stats.json', + 'visitor-stats.json', + 'questions.json', +] + +// Wait for every queued disk write so the archive reflects current state. +async function flushPendingWrites() { + await Promise.allSettled([ + state.hitStatsWritePromise, + state.visitorStatsWritePromise, + state.contactSubmissionsWritePromise, + state.questionsWritePromise, + state.draftQuestionsWritePromise, + state.replyTemplatesWritePromise, + state.replyHistoryWritePromise, + state.podcastChecklistWritePromise, + state.studyUsersWritePromise, + state.studyCommunityWritePromise, + state.studyRemindersWritePromise, + state.studyCommentsWritePromise, + state.studyCertificatesWritePromise, + state.episodeScriptsWritePromise, + state.downloadCountsWritePromise, + state.episodePlaysWritePromise, + state.analyticsEventsWritePromise, + state.qrCodesWritePromise, + ...state.studyNotesWriteQueues.values(), + ...state.studyProgressWriteQueues.values(), + ]) +} + +// Rebuild all in-memory state from whatever is now on disk (mirrors startup). +async function reloadStateFromDisk() { + state.studyNotesCache.clear() + state.studyNotesWriteQueues.clear() + state.studyProgressCache.clear() + state.studyProgressWriteQueues.clear() + // Restored study users may not match current sessions — force re-login. + state.studySessions.clear() + + await Promise.all([ + loadHitStatsFromDisk(), + loadVisitorStatsFromDisk(), + loadContactSubmissionsFromDisk(), + loadReplyTemplatesFromDisk(), + loadReplyHistoryFromDisk(), + loadQuestionsFromDisk(), + loadDraftQuestionsFromDisk(), + loadStudyUsersFromDisk(), + loadStudyCommunityFromDisk(), + loadStudyRemindersFromDisk(), + loadStudyCommentsFromDisk(), + loadStudyCertificatesFromDisk(), + loadEpisodeScriptsFromDisk(), + migrateStudyNotesIfNeeded(), + loadDownloadCountsFromDisk(), + loadQrCodesFromDisk(), + loadEpisodePlaysFromDisk(), + loadPodcastChecklistFromDisk(), + loadAnalyticsEventsFromDisk(), + refreshContentCaches(), + ]) +} + +export function register(app) { + // Download the entire data directory as a tar.gz (excluding the automatic + // snapshot folder, which is derived from the other files). + app.get('/api/admin-backup/export', requireAdminAuth, async (_req, res) => { + try { + await flushPendingWrites() + const entries = (await readdir(DATA_DIR)).filter(name => name !== 'backups') + if (entries.length === 0) { + res.status(500).json({ message: 'Data directory is empty — nothing to export.' }) + return + } + + const stamp = new Date().toISOString().replace(/[:.]/g, '-') + res.setHeader('Content-Type', 'application/gzip') + res.setHeader('Content-Disposition', `attachment; filename="siteforge-data-${stamp}.tar.gz"`) + + const archive = tar.create({ gzip: true, cwd: DATA_DIR, portable: true }, entries) + archive.on('error', err => { + console.error('[admin-backup] export stream failed:', err) + res.destroy(err) + }) + archive.pipe(res) + } catch (err) { + console.error('[admin-backup] export failed:', err) + if (!res.headersSent) res.status(500).json({ message: 'Full backup export failed.' }) + } + }) + + // Restore the entire data directory from an uploaded tar.gz produced by the + // export endpoint, then reload all in-memory state from the restored files. + app.post( + '/api/admin-backup/import', + requireAdminAuth, + express.raw({ type: () => true, limit: '500mb' }), + async (req, res) => { + let workDir = null + try { + const body = req.body + if (!Buffer.isBuffer(body) || body.length === 0) { + res.status(400).json({ message: 'Upload the .tar.gz file produced by the full backup export.' }) + return + } + if (body[0] !== 0x1f || body[1] !== 0x8b) { + res.status(400).json({ message: 'File is not a gzip archive (.tar.gz expected).' }) + return + } + + workDir = await mkdtemp(path.join(os.tmpdir(), 'siteforge-import-')) + const archivePath = path.join(workDir, 'import.tar.gz') + await writeFile(archivePath, body) + + const extractDir = path.join(workDir, 'extracted') + await mkdir(extractDir) + // node-tar strips absolute paths and rejects entries that escape cwd. + await tar.extract({ file: archivePath, cwd: extractDir }) + + const extractedEntries = await readdir(extractDir) + if (!extractedEntries.some(name => KNOWN_DATA_FILES.includes(name))) { + res.status(400).json({ message: 'Archive does not look like a Siteforge data backup.' }) + return + } + + // Settle queued writes so nothing overwrites the restored files, and + // keep a snapshot of the pre-import state in the backups folder. + await flushPendingWrites() + await createBackupSnapshot('pre-import') + + // Replace current data with the archive contents. The snapshot folder + // is preserved unless the archive itself contains one. + const currentEntries = await readdir(DATA_DIR) + for (const name of currentEntries) { + if (name === 'backups' && !extractedEntries.includes('backups')) continue + await rm(path.join(DATA_DIR, name), { recursive: true, force: true }) + } + for (const name of extractedEntries) { + await cp(path.join(extractDir, name), path.join(DATA_DIR, name), { recursive: true }) + } + + await reloadStateFromDisk() + await createBackupSnapshot('post-import') + + res.json({ ok: true, restoredEntries: extractedEntries.length }) + } catch (err) { + console.error('[admin-backup] import failed:', err) + res.status(500).json({ message: 'Full data restore failed. Check the server logs — data may need manual attention.' }) + } finally { + if (workDir) await rm(workDir, { recursive: true, force: true }).catch(() => {}) + } + }, + ) +} diff --git a/src/AdminPage.tsx b/src/AdminPage.tsx index ce6bf2e..7e3f4c1 100644 --- a/src/AdminPage.tsx +++ b/src/AdminPage.tsx @@ -1066,6 +1066,7 @@ export default function AdminPage({ content, onSave, onLogout }: Props) { const [backupFiles, setBackupFiles] = useState([]) const [selectedBackup, setSelectedBackup] = useState('') const [selectedBackupPreview, setSelectedBackupPreview] = useState(null) + const [fullBackupBusy, setFullBackupBusy] = useState(false) // TOTP management state const [totpEnabled, setTotpEnabled] = useState(null) @@ -2068,6 +2069,52 @@ export default function AdminPage({ content, onSave, onLogout }: Props) { } } + async function handleDownloadFullBackup() { + setFullBackupBusy(true) + setMaintenanceMsg('Preparing full backup archive…') + try { + const r = await fetch('/api/admin-backup/export') + if (!r.ok) throw new Error('Export failed') + const blob = await r.blob() + const disposition = r.headers.get('Content-Disposition') ?? '' + const filename = /filename="([^"]+)"/.exec(disposition)?.[1] ?? 'siteforge-data-backup.tar.gz' + const url = URL.createObjectURL(blob) + const link = document.createElement('a') + link.href = url + link.download = filename + link.click() + URL.revokeObjectURL(url) + setMaintenanceMsg(`Full backup downloaded (${filename}). Keep it somewhere safe — it contains all site data.`) + } catch { + setMaintenanceMsg('Full backup download failed.') + } finally { + setFullBackupBusy(false) + } + } + + async function handleImportFullBackup(file: File) { + if (!confirm(`Restore ALL site data from ${file.name}?\n\nThis replaces content, uploads, study accounts, notes, questions, and analytics with the archive contents. A pre-import snapshot is saved first, and study users will need to sign in again.`)) return + setFullBackupBusy(true) + setMaintenanceMsg('Restoring full backup — do not close this page…') + try { + const r = await fetch('/api/admin-backup/import', { + method: 'POST', + headers: { 'Content-Type': 'application/gzip' }, + body: file, + }) + const data = await r.json().catch(() => null) + if (!r.ok) throw new Error(data?.message ?? 'Restore failed') + await reloadContentFromServer() + await reloadStats() + await reloadBackups() + setMaintenanceMsg(`Full data restore complete — ${data?.restoredEntries ?? 'all'} items restored from ${file.name}.`) + } catch (err) { + setMaintenanceMsg(err instanceof Error ? err.message : 'Full data restore failed.') + } finally { + setFullBackupBusy(false) + } + } + async function handleRestoreBackup() { if (!selectedBackup) { setMaintenanceMsg('Select a backup first.') @@ -5025,6 +5072,9 @@ export default function AdminPage({ content, onSave, onLogout }: Props) { onRestore={handleRestoreBackup} onExport={handleExport} onBackupNow={handleBackupNow} + onDownloadFullBackup={handleDownloadFullBackup} + onImportFullBackup={handleImportFullBackup} + fullBackupBusy={fullBackupBusy} onPrune={handlePrune} onClear={handleClear} onPurgeCache={handlePurgeCache} diff --git a/src/components/AnalyticsPanel.tsx b/src/components/AnalyticsPanel.tsx index 782de5c..183459d 100644 --- a/src/components/AnalyticsPanel.tsx +++ b/src/components/AnalyticsPanel.tsx @@ -43,6 +43,9 @@ interface Props { onRestore: () => void onExport: () => void onBackupNow: () => void + onDownloadFullBackup: () => void + onImportFullBackup: (file: File) => void + fullBackupBusy: boolean onPrune: () => void onClear: () => void onPurgeCache: () => void @@ -66,6 +69,9 @@ export function AnalyticsPanel({ onRestore, onExport, onBackupNow, + onDownloadFullBackup, + onImportFullBackup, + fullBackupBusy, onPrune, onClear, onPurgeCache, @@ -74,6 +80,7 @@ export function AnalyticsPanel({ }: Props) { const [chartTab, setChartTab] = useState<'overview' | 'breakdown' | 'geographic' | 'referrers'>('overview') const [expandedVisitor, setExpandedVisitor] = useState(null) + const [fullBackupFile, setFullBackupFile] = useState(null) if (statsStatus === 'loading') return

Loading analytics…

if (statsStatus === 'error') return

Failed to load analytics.

@@ -807,6 +814,35 @@ export function AnalyticsPanel({

Total Visits: {selectedBackupPreview.totalVisits.toLocaleString()}

)} + + {/* Full Data Backup (entire /data folder — server migration) */} +
+

Full Data Backup

+

Download every persisted file — site content, uploads, study accounts, notes, progress, questions, analytics — as one archive, or restore that archive on this or a new server.

+
+
+ +
+
+ + setFullBackupFile(e.target.files?.[0] ?? null)} + disabled={fullBackupBusy} + /> + +
{maintenanceMsg &&

{maintenanceMsg}

} )