Books: book store with Project Gutenberg, Standard Ebooks, Internet Archive and OPDS catalogues
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UaC9HmtHMdRP8dAbN4wAKj
This commit is contained in:
8 files changed
+341
-12
No files matched your search
@@ -0,0 +1,176 @@
|
||||
// Book store: free, DRM-free books from Project Gutenberg (Gutendex), Standard Ebooks (OPDS), the Internet Archive (public-domain EPUBs) and any OPDS catalogue an
|
||||
// admin adds (Calibre-Web, Kavita, ...). The hub server does all the talking to those sites: searching, covers and downloading a chosen book straight onto the
|
||||
// signed-in person's shelf (books.js), so the browser never contacts them and no catalogue sign-in reaches the browser.
|
||||
const fs = require('fs'), path = require('path'), crypto = require('crypto');
|
||||
const users = require('./users'), auth = require('./auth'), books = require('./books');
|
||||
|
||||
const UA = 'NE-Hub/1.0 (self-hosted e-book reader)';
|
||||
const MAX_BOOK = 300 * 1024 * 1024, MAX_COVER = 3 * 1024 * 1024, MAX_TEXT = 4 * 1024 * 1024;
|
||||
const env = (k, d) => (process.env[k] || d).replace(/\/+$/, '');
|
||||
const BUILTIN = {
|
||||
gutenberg: { name: 'Project Gutenberg', base: () => env('GUTENDEX_URL', 'https://gutendex.com'), hosts: ['gutendex.com', 'gutenberg.org'] },
|
||||
standardebooks: { name: 'Standard Ebooks', base: () => env('STANDARD_EBOOKS_URL', 'https://standardebooks.org/feeds/opds'), hosts: ['standardebooks.org'], opds: true },
|
||||
archive: { name: 'Internet Archive', base: () => env('ARCHIVE_URL', 'https://archive.org'), hosts: ['archive.org'], needsQuery: true },
|
||||
};
|
||||
const FORMAT_OF = { 'application/epub+zip': 'epub', 'application/pdf': 'pdf', 'application/x-mobipocket-ebook': 'mobi', 'application/vnd.amazon.mobi8-ebook': 'azw3', 'application/vnd.amazon.ebook': 'azw' };
|
||||
const PREFER = ['epub', 'azw3', 'mobi', 'azw', 'pdf'];
|
||||
|
||||
const json = (res, code, o) => { res.writeHead(code, { 'content-type': 'application/json', 'cache-control': 'no-store' }); res.end(JSON.stringify(o)); };
|
||||
const fail = (code, message) => Object.assign(new Error(message), { code });
|
||||
const readBody = async (req, max = 1e5) => { const c = []; let n = 0; for await (const d of req) { n += d.length; if (n > max) throw fail(413, 'Too large'); c.push(d); } try { return JSON.parse(Buffer.concat(c).toString('utf8') || '{}'); } catch { throw fail(400, 'Bad request'); } };
|
||||
|
||||
// ---- catalogues an admin added (shared by everyone who may use Books). Sign-ins are encrypted at rest with a key from the server secret.
|
||||
const DATA_DIR = process.env.DATA_DIR || path.join(__dirname, 'data'), SRC_FILE = () => path.join(DATA_DIR, 'store-sources.json');
|
||||
let _key; const key = () => _key || (_key = crypto.scryptSync(auth.secret, 'nehub-store', 32));
|
||||
const seal = v => { const iv = crypto.randomBytes(12), c = crypto.createCipheriv('aes-256-gcm', key(), iv), d = Buffer.concat([c.update(JSON.stringify(v), 'utf8'), c.final()]); return Buffer.concat([iv, c.getAuthTag(), d]).toString('base64'); };
|
||||
const unseal = s => { try { const b = Buffer.from(s, 'base64'), d = crypto.createDecipheriv('aes-256-gcm', key(), b.subarray(0, 12)); d.setAuthTag(b.subarray(12, 28)); return JSON.parse(Buffer.concat([d.update(b.subarray(28)), d.final()]).toString('utf8')); } catch { return null; } };
|
||||
const customs = () => { try { const l = JSON.parse(fs.readFileSync(SRC_FILE(), 'utf8')); return Array.isArray(l) ? l : []; } catch { return []; } };
|
||||
const saveCustoms = l => { const f = SRC_FILE(), tmp = f + '.tmp'; fs.mkdirSync(DATA_DIR, { recursive: true }); fs.writeFileSync(tmp, JSON.stringify(l), { mode: 0o600 }); fs.renameSync(tmp, f); };
|
||||
const sourceOf = id => {
|
||||
if (BUILTIN[id]) { const b = BUILTIN[id]; return { id, name: b.name, base: b.base(), builtin: true, opds: !!b.opds, needsQuery: !!b.needsQuery, hosts: [...b.hosts, new URL(b.base()).hostname] }; }
|
||||
const c = customs().find(x => x.id === id); if (!c) throw fail(404, 'Unknown catalogue');
|
||||
const cred = c.auth ? unseal(c.auth) : null; return { id, name: c.name, base: c.url, builtin: false, opds: true, cred };
|
||||
};
|
||||
|
||||
// ---- talking to a site: only its own addresses are ever fetched (redirects included), a size limit applies, and a catalogue's sign-in goes nowhere else
|
||||
const sameSite = (src, u) => { const h = u.hostname; return src.builtin ? src.hosts.some(x => h === x || h.endsWith('.' + x)) : u.origin === new URL(src.base).origin; };
|
||||
async function get(src, url, { accept = '*/*', timeout = 10000 } = {}) {
|
||||
let u = new URL(url);
|
||||
for (let hop = 0; hop < 6; hop++) {
|
||||
if (!/^https?:$/.test(u.protocol) || !sameSite(src, u)) throw fail(502, 'That address is not part of ' + src.name);
|
||||
const headers = { 'user-agent': UA, accept }; if (src.cred && u.origin === new URL(src.base).origin) headers.authorization = 'Basic ' + Buffer.from(src.cred.user + ':' + src.cred.pass).toString('base64');
|
||||
const r = await fetch(u, { headers, redirect: 'manual', signal: AbortSignal.timeout(timeout) });
|
||||
if ([301, 302, 303, 307, 308].includes(r.status) && r.headers.get('location')) { u = new URL(r.headers.get('location'), u); try { await r.body?.cancel(); } catch {} continue; }
|
||||
if (r.status === 401 || r.status === 403) throw fail(502, src.name + (src.cred ? ' refused the saved sign-in' : ' needs a sign-in'));
|
||||
if (!r.ok) throw fail(502, src.name + ' answered ' + r.status);
|
||||
return r;
|
||||
}
|
||||
throw fail(502, 'Too many redirects from ' + src.name);
|
||||
}
|
||||
async function limited(r, max) {
|
||||
const len = +r.headers.get('content-length') || 0; if (len > max) throw fail(413, 'That file is too large');
|
||||
const chunks = []; let n = 0; for await (const c of r.body) { n += c.length; if (n > max) throw fail(413, 'That file is too large'); chunks.push(c); } return Buffer.concat(chunks);
|
||||
}
|
||||
const getText = async (src, url, accept) => (await limited(await get(src, url, { accept }), MAX_TEXT)).toString('utf8');
|
||||
const getJson = async (src, url) => { try { return JSON.parse(await getText(src, url, 'application/json')); } catch (e) { if (e.code) throw e; throw fail(502, src.name + ' sent something unexpected'); } };
|
||||
|
||||
// ---- OPDS (Atom) feeds, read with a small parser (no XML library in the hub)
|
||||
const unent = s => String(s || '').replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1').replace(/&(#x[0-9a-f]+|#\d+|amp|lt|gt|quot|apos);/gi, (m, e) => ({ amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }[e.toLowerCase()] || (e[1] === 'x' || e[1] === 'X' ? String.fromCodePoint(parseInt(e.slice(2), 16)) : e[0] === '#' ? String.fromCodePoint(+e.slice(1)) : m)));
|
||||
const tag = (x, t) => { const m = new RegExp('<' + t + '(?:\\s[^>]*)?>([\\s\\S]*?)</' + t + '>', 'i').exec(x); return m ? unent(m[1]).replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim() : ''; };
|
||||
const attrs = s => { const o = {}; for (const m of s.matchAll(/([\w:-]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g)) o[m[1].toLowerCase()] = unent(m[2] != null ? m[2] : m[3]); return o; };
|
||||
const linksOf = (x, base) => [...x.matchAll(/<link\b([^>]*?)\/?>/gi)].map(m => attrs(m[1])).filter(a => a.href).map(a => ({ ...a, href: new URL(a.href, base).href }));
|
||||
function parseFeed(xml, base) {
|
||||
const head = xml.replace(/<entry[\s>][\s\S]*?<\/entry>/gi, ''), feed = { links: linksOf(head, base), entries: [] };
|
||||
for (const m of xml.matchAll(/<entry[\s>][\s\S]*?<\/entry>/gi)) {
|
||||
const e = m[0], links = linksOf(e, base), acq = links.filter(l => /opds-spec\.org\/acquisition/.test(l.rel || '') && FORMAT_OF[(l.type || '').split(';')[0].trim()]).map(l => ({ href: l.href, format: FORMAT_OF[l.type.split(';')[0].trim()] })).sort((a, b) => PREFER.indexOf(a.format) - PREFER.indexOf(b.format));
|
||||
const img = links.find(l => /opds-spec\.org\/image$/.test(l.rel || '')) || links.find(l => /opds-spec\.org\/image/.test(l.rel || ''));
|
||||
const nav = links.find(l => /atom\+xml/.test(l.type || '') && !/opds-spec\.org\/acquisition/.test(l.rel || ''));
|
||||
feed.entries.push({ title: tag(e, 'title'), author: tag((/<author[\s>][\s\S]*?<\/author>/i.exec(e) || [''])[0], 'name'), summary: tag(e, 'summary') || tag(e, 'content'), acq: acq[0] || null, cover: img ? img.href : '', nav: nav ? nav.href : '' });
|
||||
}
|
||||
return feed;
|
||||
}
|
||||
const unbrace = u => u.replace(/%7B/gi, '{').replace(/%7D/gi, '}');
|
||||
const rootCache = new Map();
|
||||
async function opdsRoot(src) { const c = rootCache.get(src.id + src.base); if (c && Date.now() - c.t < 600000) return c.v; const v = parseFeed(await getText(src, src.base, 'application/atom+xml, application/xml;q=0.9, */*;q=0.5'), src.base); rootCache.set(src.id + src.base, { t: Date.now(), v }); return v; }
|
||||
async function opdsSearchUrl(src, q) {
|
||||
const root = await opdsRoot(src), enc = encodeURIComponent(q), fill = t => t.replace(/\{searchTerms\??\}/g, enc).replace(/\{[^}]*\?\}/g, '').replace(/\{[^}]*\}/g, '');
|
||||
const atom = root.links.find(l => l.rel === 'search' && /atom\+xml/.test(l.type || '') && /\{searchTerms/.test(unbrace(l.href)));
|
||||
if (atom) return fill(unbrace(atom.href));
|
||||
const osd = root.links.find(l => l.rel === 'search' && /opensearchdescription/.test(l.type || ''));
|
||||
if (osd) { const x = await getText(src, osd.href, 'application/xml'); for (const m of x.matchAll(/<Url\b([^>]*?)\/?>/gi)) { const a = attrs(m[1]); if (/atom\+xml/.test(a.type || '') && a.template) return fill(unbrace(new URL(a.template.replace(/\{/g, '%7B').replace(/\}/g, '%7D'), osd.href).href)); } }
|
||||
if (src.id === 'standardebooks') return src.base + '/all?query=' + enc;
|
||||
throw fail(502, src.name + ' does not offer a search');
|
||||
}
|
||||
async function opdsSearch(src, q, page) {
|
||||
let url;
|
||||
if (q) url = await opdsSearchUrl(src, q);
|
||||
else { const root = await opdsRoot(src); const nav = root.entries.filter(e => e.nav); url = root.entries.some(e => e.acq) ? src.base : (nav.find(e => /new|recent|latest|popular/i.test(e.title)) || nav[0] || {}).nav || src.base; }
|
||||
let feed = parseFeed(await getText(src, url, 'application/atom+xml'), url);
|
||||
for (let i = 1; i < page; i++) { const next = feed.links.find(l => l.rel === 'next'); if (!next) return { items: [], more: false }; url = next.href; feed = parseFeed(await getText(src, url, 'application/atom+xml'), url); }
|
||||
const items = feed.entries.filter(e => e.acq).map(e => ({ ref: e.acq.href, format: e.acq.format, title: e.title, author: e.author, summary: e.summary.slice(0, 600), cover: e.cover }));
|
||||
return { items, more: !!feed.links.find(l => l.rel === 'next') };
|
||||
}
|
||||
|
||||
// ---- Project Gutenberg through Gutendex, and the Internet Archive
|
||||
const tidyName = n => { const m = /^([^,]+),\s*(.+)$/.exec(n || ''); return m ? (m[2] + ' ' + m[1]).replace(/\s*\(.*?\)/g, '').trim() : String(n || ''); };
|
||||
async function gutenbergSearch(src, q, page) {
|
||||
const j = await getJson(src, `${src.base}/books/?${q ? 'search=' + encodeURIComponent(q) + '&' : ''}page=${page}`);
|
||||
const items = (j.results || []).filter(b => (b.formats || {})['application/epub+zip']).map(b => ({ ref: String(b.id), format: 'epub', title: b.title, author: (b.authors || []).map(a => tidyName(a.name)).join(', '), summary: (b.subjects || []).slice(0, 3).join(' · '), cover: (b.formats || {})['image/jpeg'] || '' }));
|
||||
return { items, more: !!j.next };
|
||||
}
|
||||
async function gutenbergResolve(src, ref) {
|
||||
if (!/^\d{1,9}$/.test(ref)) throw fail(400, 'Bad book');
|
||||
const b = await getJson(src, `${src.base}/books/${ref}`), url = (b.formats || {})['application/epub+zip']; if (!url) throw fail(404, 'No EPUB for that book');
|
||||
return { url, format: 'epub' };
|
||||
}
|
||||
async function archiveSearch(src, q, page) {
|
||||
const query = `(${q}) AND mediatype:texts AND format:EPUB AND -access-restricted-item:true`;
|
||||
const j = await getJson(src, `${src.base}/advancedsearch.php?q=${encodeURIComponent(query)}&fl[]=identifier&fl[]=title&fl[]=creator&rows=20&page=${page}&output=json`);
|
||||
const docs = (j.response && j.response.docs) || [], total = (j.response && j.response.numFound) || 0;
|
||||
return { items: docs.filter(d => /^[\w.-]+$/.test(d.identifier || '')).map(d => ({ ref: d.identifier, format: 'epub', title: [].concat(d.title || '')[0] || d.identifier, author: [].concat(d.creator || []).join(', '), summary: '', cover: `${src.base}/services/img/${d.identifier}` })), more: page * 20 < total };
|
||||
}
|
||||
async function archiveResolve(src, ref) {
|
||||
if (!/^[\w.-]+$/.test(ref)) throw fail(400, 'Bad book');
|
||||
const m = await getJson(src, `${src.base}/metadata/${ref}`);
|
||||
if (m.metadata && String(m.metadata['access-restricted-item']) === 'true') throw fail(403, 'That book is borrow-only (it has DRM) and cannot be added');
|
||||
const files = (m.files || []).filter(f => /\.epub$/i.test(f.name || '')); const f = files.find(x => /^EPUB$/i.test(x.format)) || files[0]; if (!f) throw fail(404, 'No EPUB file for that item');
|
||||
return { url: `${src.base}/download/${ref}/${f.name.split('/').map(encodeURIComponent).join('/')}`, format: 'epub' };
|
||||
}
|
||||
|
||||
async function searchOne(src, q, page) {
|
||||
if (src.id === 'gutenberg') return gutenbergSearch(src, q, page);
|
||||
if (src.id === 'archive') return q ? archiveSearch(src, q, page) : { items: [], more: false };
|
||||
return opdsSearch(src, q, page);
|
||||
}
|
||||
const stamp = (src, it) => ({ ...it, source: src.id, sourceName: src.name, cover: it.cover ? `/api/read/store/cover?s=${encodeURIComponent(src.id)}&u=${encodeURIComponent(it.cover)}` : '', coverUrl: it.cover || '' });
|
||||
const listSources = () => [...Object.keys(BUILTIN), ...customs().map(c => c.id)].map(id => { const s = sourceOf(id); return { id, name: s.name, builtin: s.builtin, needsQuery: !!s.needsQuery }; });
|
||||
|
||||
async function handle(req, res, p) {
|
||||
const me = users.current();
|
||||
try {
|
||||
if (p === '/sources' && req.method === 'GET') return json(res, 200, listSources());
|
||||
if (p === '/sources' && req.method === 'POST') { // add an OPDS catalogue (admin)
|
||||
if (!me || me.role !== 'admin') throw fail(403, 'Only an admin can add a catalogue');
|
||||
const b = await readBody(req), name = String(b.name || '').trim().slice(0, 60), url = String(b.url || '').trim();
|
||||
if (!name || !/^https?:\/\/\S+$/i.test(url)) throw fail(400, 'Give the catalogue a name and its address (starting with http)');
|
||||
const id = 'c' + crypto.randomBytes(5).toString('hex'), entry = { id, name, url, auth: b.user ? seal({ user: String(b.user), pass: String(b.pass || '') }) : '' };
|
||||
const l = customs(); l.push(entry); saveCustoms(l);
|
||||
try { const src = sourceOf(id), xml = await getText(src, url, 'application/atom+xml'); if (!/<feed[\s>]/i.test(xml)) throw fail(502, 'That does not look like an OPDS catalogue'); }
|
||||
catch (e) { saveCustoms(customs().filter(x => x.id !== id)); throw fail(e.code || 502, e.code ? e.message : 'Could not read that catalogue: ' + e.message); }
|
||||
return json(res, 200, { id, name });
|
||||
}
|
||||
let m;
|
||||
if ((m = /^\/sources\/(\w+)$/.exec(p)) && req.method === 'DELETE') {
|
||||
if (!me || me.role !== 'admin') throw fail(403, 'Only an admin can remove a catalogue');
|
||||
saveCustoms(customs().filter(x => x.id !== m[1])); return json(res, 200, { removed: true });
|
||||
}
|
||||
if (p === '/search' && req.method === 'GET') {
|
||||
const u = new URL(req.url, 'http://x'), q = (u.searchParams.get('q') || '').trim().slice(0, 120), page = Math.min(20, Math.max(1, +u.searchParams.get('page') || 1)), want = u.searchParams.get('source') || 'all';
|
||||
const ids = want === 'all' ? listSources().map(s => s.id) : [want], lists = [], errors = {}; let more = false;
|
||||
await Promise.all(ids.map(async id => { let src; try { src = sourceOf(id); const r = await searchOne(src, q, page); lists.push([id, r.items.map(it => stamp(src, it))]); more = more || r.more; } catch (e) { const nm = src ? src.name : id, msg = e.name === 'TimeoutError' ? 'did not answer in time' : e.message; errors[id] = msg.startsWith(nm) ? msg : nm + ': ' + msg; } }));
|
||||
lists.sort((a, b) => ids.indexOf(a[0]) - ids.indexOf(b[0]));
|
||||
const items = []; for (let i = 0; lists.some(l => l[1][i]); i++) for (const l of lists) if (l[1][i]) items.push(l[1][i]); // one from each catalogue in turn
|
||||
return json(res, 200, { items, errors, more });
|
||||
}
|
||||
if (p === '/cover' && req.method === 'GET') {
|
||||
const u = new URL(req.url, 'http://x'), src = sourceOf(u.searchParams.get('s') || ''), r = await get(src, u.searchParams.get('u') || '', { accept: 'image/*' }), type = (r.headers.get('content-type') || '').split(';')[0].trim().toLowerCase();
|
||||
if (!/^image\/(jpeg|png|gif|webp)$/.test(type)) throw fail(502, 'Not a picture');
|
||||
const buf = await limited(r, MAX_COVER); res.writeHead(200, { 'content-type': type, 'cache-control': 'private, max-age=604800' }); return res.end(buf);
|
||||
}
|
||||
if (p === '/add' && req.method === 'POST') {
|
||||
const b = await readBody(req), src = sourceOf(String(b.source || '')), ref = String(b.ref || '');
|
||||
let target;
|
||||
if (src.id === 'gutenberg') target = await gutenbergResolve(src, ref);
|
||||
else if (src.id === 'archive') target = await archiveResolve(src, ref);
|
||||
else { if (!sameSite(src, new URL(ref, src.base))) throw fail(400, 'Bad book address'); target = { url: ref, format: String(b.format || 'epub') }; }
|
||||
const ext = books.TYPES[target.format] ? target.format : 'epub';
|
||||
const buf = await limited(await get(src, target.url, { accept: 'application/epub+zip, application/octet-stream, */*;q=0.5', timeout: 120000 }), MAX_BOOK);
|
||||
let cover; const cu = String(b.cover || ''); if (cu) { try { const r = await get(src, cu, { accept: 'image/*' }), type = (r.headers.get('content-type') || '').split(';')[0].trim().toLowerCase(); cover = { type, data: await limited(r, MAX_COVER) }; } catch {} }
|
||||
let book; try { book = books.addBook(buf, ext, { title: String(b.title || '').slice(0, 200), author: String(b.author || '').slice(0, 200) }, cover); } catch (e) { throw fail(400, e.message); }
|
||||
return json(res, 200, book);
|
||||
}
|
||||
return json(res, 404, { error: 'Not found' });
|
||||
} catch (e) { return json(res, e.code && e.code >= 400 && e.code < 600 ? e.code : 502, { error: e.name === 'TimeoutError' ? 'The site did not answer in time' : e.message }); }
|
||||
}
|
||||
|
||||
module.exports = { handle, parseFeed };
|
||||
Reference in new issue
Block a user