Backup: restore from a backup file, and a personal data export

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

backup.js

@@ -1,13 +1,61 @@
 // Admin-only backup: streams the whole data folder (passkeys, accounts, tasks, events, settings, Signal history) as a .tar.gz.
 // Treat the file like a password: it contains the session secret and encrypted sign-in tokens.
-const { spawn } = require('child_process');
-const path = require('path');
+// Restore (admin): upload that .tar.gz; it is checked, unpacked beside the data, and each top-level item is swapped in with the old one kept in .pre-restore-<date>; then the hub restarts.
+// Export (anyone): your own data as a .tar.gz, without sign-in tokens.
+const { spawn, execFile } = require('child_process');
+const fs = require('fs'), path = require('path'), { pipeline } = require('stream/promises');
+const users = require('./users');
 const audit = require('./audit');
 const DATA_DIR = path.resolve(process.env.DATA_DIR || path.join(__dirname, 'data'));
+const MAX_RESTORE = 4 * 1024 ** 3;
+const send = (res, code, o) => { res.writeHead(code, { 'content-type': 'application/json', 'cache-control': 'no-store' }); res.end(JSON.stringify(o)); };
+const run = (cmd, args) => new Promise((ok, no) => execFile(cmd, args, { maxBuffer: 64 * 1024 * 1024 }, (e, out, err) => e ? no(new Error((String(err) || e.message).trim().slice(0, 300))) : ok(String(out))));
+const rm = p => fs.rmSync(p, { recursive: true, force: true });
+/** Refuse an archive that could write outside the data folder or plant links: every name must stay inside, and only plain files and folders are allowed. */
+async function checkArchive(file) {
+  const names = (await run('tar', ['-tzf', file])).split('\n').filter(Boolean), long = (await run('tar', ['-tvzf', file])).split('\n').filter(Boolean);
+  if (!names.length) throw new Error('That backup is empty');
+  for (const n of names) if (path.isAbsolute(n) || n.split('/').includes('..')) throw new Error('That is not a backup this hub made (unsafe file names)');
+  for (const l of long) if (!/^[-d]/.test(l)) throw new Error('That is not a backup this hub made (it contains links or special files)');
+  return names;
+}
+async function restore(req, res, who) {
+  if (req.method !== 'POST') return send(res, 405, { error: 'Use POST' });
+  const stage = path.join(DATA_DIR, '.restore'), up = stage + '.tar.gz', out = path.join(stage, 'files'); rm(stage); rm(up);
+  try {
+    if ((+req.headers['content-length'] || 0) > MAX_RESTORE) return send(res, 413, { error: 'That file is too large' });
+    let n = 0; const cap = new (require('stream').Transform)({ transform(c, _e, cb) { n += c.length; cb(n > MAX_RESTORE ? Object.assign(new Error('That file is too large'), { status: 413 }) : null, c); } });
+    await pipeline(req, cap, fs.createWriteStream(up, { mode: 0o600 }));
+    await checkArchive(up); fs.mkdirSync(out, { recursive: true, mode: 0o700 }); await run('tar', ['-xzf', up, '--no-same-owner', '--no-same-permissions', '-C', out]);
+    const items = fs.readdirSync(out); if (!items.includes('secret.json') && !items.includes('users')) throw new Error('That does not look like an NE Hub backup (no accounts or secret in it)');
+    const keep = path.join(DATA_DIR, '.pre-restore-' + new Date().toISOString().replace(/[:.]/g, '-')); fs.mkdirSync(keep, { recursive: true, mode: 0o700 });
+    for (const it of items) { const dst = path.join(DATA_DIR, it); if (fs.existsSync(dst)) fs.renameSync(dst, path.join(keep, it)); fs.renameSync(path.join(out, it), dst); }
+    audit.log('backup restored', { who: who || '', items: items.length, kept: path.basename(keep) }, req);
+    send(res, 200, { ok: true, items: items.length, kept: path.basename(keep), restarting: true });
+    if (!process.env.NO_RESTART_AFTER_RESTORE) setTimeout(() => process.exit(0), 800); // the container restarts (restart: unless-stopped) and loads the restored data
+  } catch (e) { if (!res.headersSent) send(res, e.status || 400, { error: e.message }); }
+  finally { rm(stage); rm(up); }
+}
+/** Your own data as a .tar.gz: tasks, contacts, settings, Bible notes, your notes and so on. Sign-in tokens for mail, iCloud and Plex are left out. */
+async function exportMine(req, res) {
+  const me = users.current(); if (!me) return send(res, 401, { error: 'Sign in' });
+  const tmp = fs.mkdtempSync(path.join(DATA_DIR, '.export-')), SECRET = new Set(['mail-accounts.json', 'mail.json', 'icloud.json', 'plex.json', 'push.json']);
+  try {
+    fs.cpSync(users.dirOf(me.id), path.join(tmp, 'me'), { recursive: true, filter: s => !SECRET.has(path.basename(s)) });
+    try { const notes = JSON.parse(fs.readFileSync(path.join(DATA_DIR, 'notes.json'), 'utf8')), l = Array.isArray(notes) ? notes : notes.notes || []; fs.writeFileSync(path.join(tmp, 'me', 'my-notes.json'), JSON.stringify(l.filter(n => n.owner === me.id), null, 1)); } catch { /* no notes yet */ }
+    const name = 'nehub-' + String(me.name || 'me').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '') + '-' + new Date().toISOString().slice(0, 10) + '.tar.gz';
+    const tar = spawn('tar', ['-czf', '-', '--exclude=*.tmp', '-C', tmp, 'me'], { stdio: ['ignore', 'pipe', 'pipe'] });
+    res.writeHead(200, { 'content-type': 'application/gzip', 'content-disposition': `attachment; filename="${name}"`, 'cache-control': 'no-store' });
+    tar.stdout.pipe(res); res.on('close', () => { tar.kill(); rm(tmp); }); tar.on('error', () => res.destroy());
+    audit.log('personal export', { who: me.name }, req);
+  } catch (e) { rm(tmp); if (!res.headersSent) send(res, 500, { error: 'Could not make the export: ' + e.message }); }
+}
 function handle(req, res, who) {
+  if (new URL(req.url, 'http://x').pathname === '/api/backup/restore') return restore(req, res, who);
+  if (new URL(req.url, 'http://x').pathname === '/api/export') return exportMine(req, res);
   if (req.method !== 'GET') { res.writeHead(405); return res.end(); }
   const name = 'nehub-backup-' + new Date().toISOString().slice(0, 10) + '.tar.gz';
-  const tar = spawn('tar', ['-czf', '-', '--exclude=./lyrics', '--exclude=*.tmp', '-C', DATA_DIR, '.'], { stdio: ['ignore', 'pipe', 'pipe'] });
+  const tar = spawn('tar', ['-czf', '-', '--exclude=./lyrics', '--exclude=./.restore*', '--exclude=./.export-*', '--exclude=./.pre-restore-*', '--exclude=*.tmp', '-C', DATA_DIR, '.'], { stdio: ['ignore', 'pipe', 'pipe'] });
   tar.on('error', e => { if (!res.headersSent) { res.writeHead(500, { 'content-type': 'application/json' }); res.end(JSON.stringify({ error: 'Could not make the backup: ' + e.message })); } else res.destroy(); });
   tar.stderr.on('data', d => console.warn('[backup]', String(d).trim()));
   res.writeHead(200, { 'content-type': 'application/gzip', 'content-disposition': `attachment; filename="${name}"`, 'cache-control': 'no-store' });
@@ -15,4 +63,4 @@ function handle(req, res, who) {
   res.on('close', () => tar.kill());
   audit.log('backup downloaded', { who: who || '' }, req);
 }
-module.exports = { handle };
+module.exports = { handle, checkArchive };

tools/backup-test.js

@@ -0,0 +1,27 @@
+// Backup restore and personal export: a good backup is swapped in (the old data kept aside), unsafe archives are refused, and an export holds only your own data without sign-in tokens.
+const assert = require('assert'), fs = require('fs'), os = require('os'), path = require('path'), http = require('http'), { execFileSync } = require('child_process');
+const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'nehub-backup-')); process.env.DATA_DIR = dir; process.env.NO_RESTART_AFTER_RESTORE = '1';
+const users = require('../users'), backup = require('../backup');
+const a = users.create({ name: 'Ann', role: 'admin', handle: 'YQ' }), b = users.create({ name: 'Bo', role: 'user', handle: 'ZP' });
+fs.writeFileSync(path.join(users.dirOf(a.id), 'tasks.json'), '{"mine":1}'); fs.writeFileSync(path.join(users.dirOf(a.id), 'mail.json'), '{"token":"secret"}'); fs.writeFileSync(path.join(users.dirOf(b.id), 'tasks.json'), '{"bo":1}');
+const hub = http.createServer((req, res) => users.runAs(a, () => backup.handle(req, res, 'Ann'))).listen(0, async () => {
+  const base = 'http://127.0.0.1:' + hub.address().port;
+  const mk = (name, fn) => { const w = fs.mkdtempSync(path.join(os.tmpdir(), 'nehub-bk-src-')); fn(w); const f = path.join(os.tmpdir(), name); execFileSync('tar', ['-czf', f, '-C', w, '.']); return fs.readFileSync(f); };
+  try {
+    // export: only my data, no tokens
+    const ex = Buffer.from(await (await fetch(base + '/api/export')).arrayBuffer()), ef = path.join(os.tmpdir(), 'nehub-ex.tar.gz'); fs.writeFileSync(ef, ex);
+    const list = execFileSync('tar', ['-tzf', ef]).toString(); assert.match(list, /me\/tasks\.json/); assert.ok(!/mail\.json/.test(list), 'sign-in tokens are left out'); assert.ok(!list.includes(b.id), 'nobody else\'s data');
+    // a good backup is swapped in; the old data is kept
+    const good = mk('good.tar.gz', w => { fs.writeFileSync(path.join(w, 'secret.json'), '{"restored":true}'); fs.mkdirSync(path.join(w, 'users')); });
+    let r = await fetch(base + '/api/backup/restore', { method: 'POST', body: good }); let j = await r.json(); assert.equal(r.status, 200, JSON.stringify(j)); assert.ok(j.restarting);
+    assert.equal(JSON.parse(fs.readFileSync(path.join(dir, 'secret.json'), 'utf8')).restored, true); assert.ok(fs.existsSync(path.join(dir, j.kept, 'users')), 'the old accounts are kept aside');
+    assert.ok(!fs.existsSync(path.join(dir, '.restore')) && !fs.existsSync(path.join(dir, '.restore.tar.gz')), 'the staging files are cleaned up');
+    // unsafe or foreign archives are refused and change nothing
+    const evil = mk('evil.tar.gz', w => { fs.writeFileSync(path.join(w, 'secret.json'), '{"evil":true}'); fs.symlinkSync('/etc/passwd', path.join(w, 'users')); });
+    r = await fetch(base + '/api/backup/restore', { method: 'POST', body: evil }); assert.equal(r.status, 400); assert.match((await r.json()).error, /links/); assert.equal(JSON.parse(fs.readFileSync(path.join(dir, 'secret.json'), 'utf8')).restored, true, 'nothing changed');
+    const foreign = mk('foreign.tar.gz', w => fs.writeFileSync(path.join(w, 'readme.txt'), 'hi')); r = await fetch(base + '/api/backup/restore', { method: 'POST', body: foreign }); assert.equal(r.status, 400); assert.match((await r.json()).error, /NE Hub backup/);
+    r = await fetch(base + '/api/backup/restore', { method: 'POST', body: Buffer.from('not a tar') }); assert.equal(r.status, 400);
+    console.log('backup test ok');
+  } catch (e) { console.error(e); process.exitCode = 1; }
+  hub.close();
+});