// "Save to Files": copy books, free audiobook chapters and Verse By Verse lessons and handouts into the person's own Seafile. The hub does the downloading and the // uploading itself (using the Seafile sign-in already saved for that person), so nothing big passes through the phone, and a whole study keeps going in the background // while you use the app. Files that are already there are skipped, so running it again just fills in what is missing. const fs = require('fs'), crypto = require('crypto'); const settings = require('./settings'), users = require('./users'), books = require('./books'), vbv = require('./vbv'), librivox = require('./librivox'); const SEAFILE = () => (process.env.SEAFILE_URL || 'https://seafile.necloud.us').replace(/\/$/, ''); const MAX_FILE = 500 * 1024 * 1024; const TABS = { 'vbv-notes': 'podcast', 'vbv-lesson': 'podcast', 'vbv-study': 'podcast', book: 'read', 'librivox-book': 'audio', 'librivox-chapter': 'audio' }; const json = (res, code, o) => { res.writeHead(code, { 'content-type': 'application/json', 'cache-control': 'no-store' }); res.end(JSON.stringify(o)); }; const fail = (code, message) => Object.assign(new Error(message), { code }); const safe = (s, max = 100) => String(s || '').replace(/[\u0000-\u001f\\/:*?"<>|]+/g, ' ').replace(/\s+/g, ' ').replace(/^[. ]+|[. ]+$/g, '').slice(0, max).trim() || 'Untitled'; const num = (n, of) => String(n + 1).padStart(Math.max(2, String(of).length), '0'); const readBody = async req => { const c = []; let n = 0; for await (const d of req) { n += d.length; if (n > 1e5) throw fail(413, 'Too large'); c.push(d); } try { return JSON.parse(Buffer.concat(c).toString('utf8') || '{}'); } catch { throw fail(400, 'Bad request'); } }; // ---- Seafile, as the person function seafile(token) { const call = async (p, init = {}) => { const r = await fetch(SEAFILE() + p, { ...init, headers: { ...init.headers, Authorization: 'Token ' + token }, signal: AbortSignal.timeout(init.timeout || 30000) }); if (r.status === 401) throw fail(401, 'Your Files sign-in has expired. Open Files and sign in again.'); return r; }; const enc = encodeURIComponent; return { async list(repo, dir) { const r = await call(`/api2/repos/${enc(repo)}/dir/?p=${enc(dir)}`); if (r.status === 404) return null; if (!r.ok) throw fail(502, 'Files answered ' + r.status); return (await r.json()).map(x => ({ name: x.name, dir: x.type === 'dir' })); }, async mkdir(repo, dir) { const r = await call(`/api2/repos/${enc(repo)}/dir/?p=${enc(dir)}`, { method: 'POST', body: new URLSearchParams({ operation: 'mkdir' }), headers: { 'content-type': 'application/x-www-form-urlencoded' } }); if (!r.ok && r.status !== 409) throw fail(r.status === 403 ? 403 : 502, r.status === 403 ? 'You cannot add files to that library' : 'Could not create the folder (' + r.status + ')'); }, async upload(repo, dir, name, buf) { const lr = await call(`/api2/repos/${enc(repo)}/upload-link/?p=${enc(dir)}`); if (!lr.ok) throw fail(lr.status === 403 ? 403 : 502, lr.status === 403 ? 'You cannot add files to that library' : 'Files would not accept the upload (' + lr.status + ')'); const link = await lr.json(), fd = new FormData(); fd.append('parent_dir', dir); fd.append('replace', '0'); fd.append('file', new Blob([buf]), name); const up = await fetch(new URL(link, SEAFILE()), { method: 'POST', body: fd, signal: AbortSignal.timeout(15 * 60e3) }); if (!up.ok) throw fail(502, 'The upload failed (' + up.status + ')'); }, }; } async function download(url, allowed, ua) { if (!allowed(url)) throw fail(502, 'That address is not allowed'); let r = await fetch(url, { headers: { 'user-agent': ua }, redirect: 'manual', signal: AbortSignal.timeout(10 * 60e3) }); for (let i = 0; i < 4 && [301, 302, 303, 307, 308].includes(r.status); i++) { const to = new URL(r.headers.get('location') || '', url).href; if (!allowed(to)) throw fail(502, 'That address is not allowed'); url = to; r = await fetch(url, { headers: { 'user-agent': ua }, redirect: 'manual', signal: AbortSignal.timeout(10 * 60e3) }); } if (!r.ok) throw fail(502, 'The download failed (' + r.status + ')'); if ((+r.headers.get('content-length') || 0) > MAX_FILE) throw fail(413, 'That file is too large'); const chunks = []; let n = 0; for await (const c of r.body) { n += c.length; if (n > MAX_FILE) throw fail(413, 'That file is too large'); chunks.push(c); } return Buffer.concat(chunks); } // ---- what to save: a list of files ({ folder, name, load }), worked out before anything starts so the person gets an answer at once async function plan(me, b) { const kind = String(b.kind || ''), out = []; if (kind === 'book') { const f = books.fileOf(String(b.id || '')); if (!f) throw fail(404, 'Book not found'); out.push({ folder: ['Books'], name: safe(f.author ? `${f.author} - ${f.title}` : f.title, 120) + '.' + f.ext, load: async () => fs.promises.readFile(f.path) }); return { label: f.title, out }; } if (kind === 'librivox-book' || kind === 'librivox-chapter') { const bk = await librivox.book(String(b.id || '')), only = kind === 'librivox-chapter' ? +b.n : -1; bk.chapters.forEach(c => { if (only >= 0 && c.n !== only) return; out.push({ folder: ['Audiobooks', safe(bk.title)], name: `${num(c.n, bk.chapters.length)} - ${safe(c.title, 80)}.mp3`, load: () => download(c.url, librivox.allowed, 'NE-Hub/1.0') }); }); if (!out.length) throw fail(404, 'No chapters to save'); return { label: bk.title, out }; } if (kind.startsWith('vbv-')) { const slug = String(b.slug || ''), st = await vbv.studyInfo(slug), folder = ['Verse By Verse', safe(st.title)], what = b.what || 'both'; const lessons = kind === 'vbv-study' ? st.lessons : st.lessons.filter(l => l.n === +b.n); if (!lessons.length) throw fail(404, 'No such lesson'); const wantAudio = kind === 'vbv-notes' ? false : what !== 'notes', wantNotes = kind === 'vbv-notes' ? true : what !== 'audio'; for (const l of lessons) { const base = `${num(l.n, st.lessons.length)} - ${safe(l.title, 80)}`; if (wantAudio) out.push({ folder, name: base + '.mp3', load: async () => download(await vbv.audioUrl(slug, l.n), vbv.allowed, vbv.UA) }); if (wantNotes) out.push({ folder, name: base + ' - Notes.pdf', optional: true, load: async () => download(await vbv.notesUrl(slug, l.n), vbv.allowed, vbv.UA) }); } return { label: kind === 'vbv-study' ? st.title : `${st.title}, lesson ${+b.n + 1}`, out }; } throw fail(400, 'Unknown thing to save'); } // ---- jobs const jobs = new Map(); setInterval(() => { for (const [k, j] of jobs) if (Date.now() - j.t > 6 * 3600e3 && j.state !== 'running') jobs.delete(k); }, 600e3).unref(); const view = j => ({ id: j.id, state: j.state, label: j.label, total: j.total, done: j.done, skipped: j.skipped, missing: j.missing, failed: j.failed, current: j.current, error: j.error, errors: j.errors.slice(0, 5), where: j.where, repo: j.repo, dir: j.dir }); async function run(j, token, items) { const sf = seafile(token), made = new Set(); let existing = new Map(); const ensure = async folder => { let p = j.dir === '/' ? '' : j.dir; for (const part of folder) { p += '/' + part; if (made.has(p)) continue; const names = await sf.list(j.repo, p); if (names === null) await sf.mkdir(j.repo, p); made.add(p); } return (p || '/'); }; try { for (const it of items) { if (j.cancelled) { j.state = 'cancelled'; return; } j.current = it.name; let target; try { target = await ensure(it.folder); if (!existing.has(target)) existing.set(target, new Set(((await sf.list(j.repo, target)) || []).map(x => x.name.toLowerCase()))); if (existing.get(target).has(it.name.toLowerCase())) { j.skipped++; continue; } let buf; try { buf = await it.load(); } catch (e) { if (it.optional && e.code === 404) { j.missing++; continue; } throw e; } try { await sf.upload(j.repo, target, it.name, buf); } catch (e) { if (e.code === 401 || e.code === 403) throw e; await sf.upload(j.repo, target, it.name, buf); } // one more try existing.get(target).add(it.name.toLowerCase()); j.done++; } catch (e) { if (e.code === 401 || e.code === 403) { j.state = 'error'; j.error = e.message; return; } j.failed++; j.errors.push(`${it.name}: ${e.message}`); } } j.state = j.failed && !j.done && !j.skipped ? 'error' : 'done'; if (j.state === 'error') j.error = j.errors[0] || 'Nothing could be saved'; } catch (e) { j.state = 'error'; j.error = e.message; } finally { j.current = ''; } } async function handle(req, res) { const u = new URL(req.url, 'http://x'), p = u.pathname.slice('/api/save'.length) || '/', me = users.current(); try { let m; if (p === '/' && req.method === 'POST') { const b = await readBody(req), tab = TABS[b.kind]; if (!tab) throw fail(400, 'Unknown thing to save'); if (!users.can(me, tab)) throw fail(403, 'You do not have access to that'); const token = settings.get('sf_token'); if (!token) throw fail(401, 'Open Files and sign in first, then try again'); const repo = String(b.repo || ''), dir = '/' + String(b.dir || '/').split('/').filter(x => x && x !== '.' && x !== '..').join('/'); if (!/^[\w-]{8,64}$/.test(repo)) throw fail(400, 'Choose a library and folder'); if ([...jobs.values()].filter(j => j.user === me.id && j.state === 'running').length >= 2) throw fail(429, 'Two saves are already running. Wait for one to finish.'); const { label, out } = await plan(me, b); const j = { id: crypto.randomBytes(6).toString('hex'), user: me.id, t: Date.now(), state: 'running', label, total: out.length, done: 0, skipped: 0, missing: 0, failed: 0, current: '', error: '', errors: [], repo, dir, where: out[0].folder.join('/'), cancelled: false }; jobs.set(j.id, j); run(j, token, out); return json(res, 200, view(j)); } if ((m = /^\/job\/(\w+)$/.exec(p))) { const j = jobs.get(m[1]); if (!j || j.user !== me.id) throw fail(404, 'Not found'); if (req.method === 'DELETE') { j.cancelled = true; return json(res, 200, view(j)); } return json(res, 200, view(j)); } return json(res, 404, { error: 'Not found' }); } catch (e) { return json(res, e.code >= 400 && e.code < 600 ? e.code : 502, { error: e.message }); } } module.exports = { handle, safe, seafile };