// Book store: free, DRM-free books from Project Gutenberg (its own catalogue feed), Standard Ebooks (its public pages), the Internet Archive (public-domain EPUBs) and any OPDS catalogue an
// admin adds (Calibre-Web, Kavita, ...). The hub server does all the talking to those sites: searching, covers and downloading a chosen book straight onto the
// signed-in person's shelf (books.js), so the browser never contacts them and no catalogue sign-in reaches the browser.
const fs = require('fs'), path = require('path'), crypto = require('crypto');
const users = require('./users'), auth = require('./auth'), books = require('./books');

const UA = 'NE-Hub/1.0 (self-hosted e-book reader)';
const MAX_BOOK = 300 * 1024 * 1024, MAX_COVER = 3 * 1024 * 1024, MAX_TEXT = 4 * 1024 * 1024;
const env = (k, d) => (process.env[k] || d).replace(/\/+$/, '');
const BUILTIN = {
  gutenberg: { name: 'Project Gutenberg', base: () => env('GUTENBERG_URL', 'https://www.gutenberg.org'), hosts: ['gutenberg.org'] }, // its own catalogue feed (fast and official)
  standardebooks: { name: 'Standard Ebooks', base: () => env('STANDARD_EBOOKS_URL', 'https://standardebooks.org'), hosts: ['standardebooks.org'] }, // its public pages: the OPDS feed is for paying members only
  archive: { name: 'Internet Archive', base: () => env('ARCHIVE_URL', 'https://archive.org'), hosts: ['archive.org'], needsQuery: true },
};
const FORMAT_OF = { 'application/epub+zip': 'epub', 'application/pdf': 'pdf', 'application/x-mobipocket-ebook': 'mobi', 'application/vnd.amazon.mobi8-ebook': 'azw3', 'application/vnd.amazon.ebook': 'azw' };
const PREFER = ['epub', 'azw3', 'mobi', 'azw', 'pdf'];

const json = (res, code, o) => { res.writeHead(code, { 'content-type': 'application/json', 'cache-control': 'no-store' }); res.end(JSON.stringify(o)); };
const fail = (code, message) => Object.assign(new Error(message), { code });
const readBody = async (req, max = 1e5) => { const c = []; let n = 0; for await (const d of req) { n += d.length; if (n > max) throw fail(413, 'Too large'); c.push(d); } try { return JSON.parse(Buffer.concat(c).toString('utf8') || '{}'); } catch { throw fail(400, 'Bad request'); } };

// ---- catalogues an admin added (shared by everyone who may use Books). Sign-ins are encrypted at rest with a key from the server secret.
const DATA_DIR = process.env.DATA_DIR || path.join(__dirname, 'data'), SRC_FILE = () => path.join(DATA_DIR, 'store-sources.json');
let _key; const key = () => _key || (_key = crypto.scryptSync(auth.secret, 'nehub-store', 32));
const seal = v => { const iv = crypto.randomBytes(12), c = crypto.createCipheriv('aes-256-gcm', key(), iv), d = Buffer.concat([c.update(JSON.stringify(v), 'utf8'), c.final()]); return Buffer.concat([iv, c.getAuthTag(), d]).toString('base64'); };
const unseal = s => { try { const b = Buffer.from(s, 'base64'), d = crypto.createDecipheriv('aes-256-gcm', key(), b.subarray(0, 12)); d.setAuthTag(b.subarray(12, 28)); return JSON.parse(Buffer.concat([d.update(b.subarray(28)), d.final()]).toString('utf8')); } catch { return null; } };
const customs = () => { try { const l = JSON.parse(fs.readFileSync(SRC_FILE(), 'utf8')); return Array.isArray(l) ? l : []; } catch { return []; } };
const saveCustoms = l => { const f = SRC_FILE(), tmp = f + '.tmp'; fs.mkdirSync(DATA_DIR, { recursive: true }); fs.writeFileSync(tmp, JSON.stringify(l), { mode: 0o600 }); fs.renameSync(tmp, f); };
const sourceOf = id => {
  if (BUILTIN[id]) { const b = BUILTIN[id]; return { id, name: b.name, base: b.base(), builtin: true, opds: !!b.opds, needsQuery: !!b.needsQuery, hosts: [...b.hosts, new URL(b.base()).hostname] }; }
  const c = customs().find(x => x.id === id); if (!c) throw fail(404, 'Unknown catalogue');
  const cred = c.auth ? unseal(c.auth) : null; return { id, name: c.name, base: c.url, builtin: false, opds: true, cred };
};

// ---- talking to a site: only its own addresses are ever fetched (redirects included), a size limit applies, and a catalogue's sign-in goes nowhere else
const sameSite = (src, u) => { const h = u.hostname; return src.builtin ? src.hosts.some(x => h === x || h.endsWith('.' + x)) : u.origin === new URL(src.base).origin; };
async function get(src, url, { accept = '*/*', timeout = 15000 } = {}) {
  let u = new URL(url);
  for (let hop = 0; hop < 6; hop++) {
    if (!/^https?:$/.test(u.protocol) || !sameSite(src, u)) throw fail(502, 'That address is not part of ' + src.name);
    const headers = { 'user-agent': UA, accept }; if (src.cred && u.origin === new URL(src.base).origin) headers.authorization = 'Basic ' + Buffer.from(src.cred.user + ':' + src.cred.pass).toString('base64');
    const r = await fetch(u, { headers, redirect: 'manual', signal: AbortSignal.timeout(timeout) });
    if ([301, 302, 303, 307, 308].includes(r.status) && r.headers.get('location')) { u = new URL(r.headers.get('location'), u); try { await r.body?.cancel(); } catch {} continue; }
    if (r.status === 401 || r.status === 403) throw fail(502, src.name + (src.cred ? ' refused the saved sign-in' : ' needs a sign-in'));
    if (!r.ok) throw fail(502, src.name + ' answered ' + r.status);
    return r;
  }
  throw fail(502, 'Too many redirects from ' + src.name);
}
async function limited(r, max) {
  const len = +r.headers.get('content-length') || 0; if (len > max) throw fail(413, 'That file is too large');
  const chunks = []; let n = 0; for await (const c of r.body) { n += c.length; if (n > max) throw fail(413, 'That file is too large'); chunks.push(c); } return Buffer.concat(chunks);
}
const getText = async (src, url, accept) => (await limited(await get(src, url, { accept }), MAX_TEXT)).toString('utf8');
const getJson = async (src, url) => { try { return JSON.parse(await getText(src, url, 'application/json')); } catch (e) { if (e.code) throw e; throw fail(502, src.name + ' sent something unexpected'); } };

// ---- OPDS (Atom) feeds, read with a small parser (no XML library in the hub)
const unent = s => String(s || '').replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1').replace(/&(#x[0-9a-f]+|#\d+|amp|lt|gt|quot|apos);/gi, (m, e) => ({ amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }[e.toLowerCase()] || (e[1] === 'x' || e[1] === 'X' ? String.fromCodePoint(parseInt(e.slice(2), 16)) : e[0] === '#' ? String.fromCodePoint(+e.slice(1)) : m)));
const tag = (x, t) => { const m = new RegExp('<' + t + '(?:\\s[^>]*)?>([\\s\\S]*?)</' + t + '>', 'i').exec(x); return m ? unent(m[1]).replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim() : ''; };
const plain = s => unent(String(s || '').replace(/<[^>]+>/g, ' ')).replace(/\s+/g, ' ').trim();
const attrs = s => { const o = {}; for (const m of s.matchAll(/([\w:-]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g)) o[m[1].toLowerCase()] = unent(m[2] != null ? m[2] : m[3]); return o; };
const linksOf = (x, base) => [...x.matchAll(/<link\b([^>]*?)\/?>/gi)].map(m => attrs(m[1])).filter(a => a.href).map(a => ({ ...a, href: new URL(a.href, base).href }));
function parseFeed(xml, base) {
  const head = xml.replace(/<entry[\s>][\s\S]*?<\/entry>/gi, ''), feed = { links: linksOf(head, base), entries: [] };
  for (const m of xml.matchAll(/<entry[\s>][\s\S]*?<\/entry>/gi)) {
    const e = m[0], links = linksOf(e, base), acq = links.filter(l => /opds-spec\.org\/acquisition/.test(l.rel || '') && FORMAT_OF[(l.type || '').split(';')[0].trim()]).map(l => ({ href: l.href, format: FORMAT_OF[l.type.split(';')[0].trim()] })).sort((a, b) => PREFER.indexOf(a.format) - PREFER.indexOf(b.format));
    const img = links.find(l => /opds-spec\.org\/image$/.test(l.rel || '')) || links.find(l => /opds-spec\.org\/image/.test(l.rel || ''));
    const nav = links.find(l => /atom\+xml/.test(l.type || '') && !/opds-spec\.org\/acquisition/.test(l.rel || ''));
    feed.entries.push({ title: tag(e, 'title'), author: tag((/<author[\s>][\s\S]*?<\/author>/i.exec(e) || [''])[0], 'name'), summary: tag(e, 'summary') || tag(e, 'content'), acq: acq[0] || null, cover: img ? img.href : '', nav: nav ? nav.href : '' });
  }
  return feed;
}
const unbrace = u => u.replace(/%7B/gi, '{').replace(/%7D/gi, '}');
const rootCache = new Map();
async function opdsRoot(src) { const c = rootCache.get(src.id + src.base); if (c && Date.now() - c.t < 600000) return c.v; const v = parseFeed(await getText(src, src.base, 'application/atom+xml, application/xml;q=0.9, */*;q=0.5'), src.base); rootCache.set(src.id + src.base, { t: Date.now(), v }); return v; }
async function opdsSearchUrl(src, q) {
  const root = await opdsRoot(src), enc = encodeURIComponent(q), fill = t => t.replace(/\{searchTerms\??\}/g, enc).replace(/\{[^}]*\?\}/g, '').replace(/\{[^}]*\}/g, '');
  const atom = root.links.find(l => l.rel === 'search' && /atom\+xml/.test(l.type || '') && /\{searchTerms/.test(unbrace(l.href)));
  if (atom) return fill(unbrace(atom.href));
  const osd = root.links.find(l => l.rel === 'search' && /opensearchdescription/.test(l.type || ''));
  if (osd) { const x = await getText(src, osd.href, 'application/xml'); for (const m of x.matchAll(/<Url\b([^>]*?)\/?>/gi)) { const a = attrs(m[1]); if (/atom\+xml/.test(a.type || '') && a.template) return fill(unbrace(new URL(a.template.replace(/\{/g, '%7B').replace(/\}/g, '%7D'), osd.href).href)); } }
  throw fail(502, src.name + ' does not offer a search');
}
async function opdsSearch(src, q, page) {
  let url;
  if (q) url = await opdsSearchUrl(src, q);
  else { const root = await opdsRoot(src); const nav = root.entries.filter(e => e.nav); url = root.entries.some(e => e.acq) ? src.base : (nav.find(e => /new|recent|latest|popular/i.test(e.title)) || nav[0] || {}).nav || src.base; }
  let feed = parseFeed(await getText(src, url, 'application/atom+xml'), url);
  for (let i = 1; i < page; i++) { const next = feed.links.find(l => l.rel === 'next'); if (!next) return { items: [], more: false }; url = next.href; feed = parseFeed(await getText(src, url, 'application/atom+xml'), url); }
  const items = feed.entries.filter(e => e.acq).map(e => ({ ref: e.acq.href, format: e.acq.format, title: e.title, author: e.author, summary: e.summary.slice(0, 600), cover: e.cover }));
  return { items, more: !!feed.links.find(l => l.rel === 'next') };
}

// ---- Project Gutenberg (its own OPDS catalogue), Standard Ebooks (its public pages), and the Internet Archive
const entriesOf = xml => [...xml.matchAll(/<entry[\s>][\s\S]*?<\/entry>/gi)].map(m => m[0]);
async function gutenbergSearch(src, q, page) {
  const url = `${src.base}/ebooks/search.opds/?${q ? 'query=' + encodeURIComponent(q) : 'sort_order=downloads'}${page > 1 ? '&start_index=' + ((page - 1) * 25 + 1) : ''}`;
  const xml = await getText(src, url, 'application/atom+xml'), items = [], seen = new Set();
  for (const e of entriesOf(xml)) { const id = (/\/ebooks\/(\d+)\.opds/.exec(tag0(e, 'id')) || [])[1]; if (!id || seen.has(id)) continue; seen.add(id); items.push({ ref: id, format: 'epub', title: tag(e, 'title'), author: tag(e, 'content'), summary: '', cover: `${src.base}/cache/epub/${id}/pg${id}.cover.medium.jpg` }); }
  return { items, more: /<link[^>]+rel="next"/i.test(xml.replace(/<entry[\s>][\s\S]*?<\/entry>/gi, '')) };
}
const tag0 = (x, t) => (new RegExp('<' + t + '>([^<]*)</' + t + '>', 'i').exec(x) || [])[1] || '';
async function gutenbergResolve(src, ref) {
  if (!/^\d{1,9}$/.test(ref)) throw fail(400, 'Bad book');
  return { url: `${src.base}/ebooks/${ref}.epub3.images`, format: 'epub' }; // (the address Project Gutenberg itself gives for the EPUB with pictures)
}
async function standardSearch(src, q, page) {
  const html = await getText(src, `${src.base}/ebooks?${q ? 'query=' + encodeURIComponent(q) + '&' : ''}page=${page}&per-page=24`, 'text/html, application/xhtml+xml'), items = [];
  for (const m of html.matchAll(/<li typeof="schema:Book" about="(\/ebooks\/[^"]+)">([\s\S]*?)<\/li>/g)) {
    const card = m[2], title = plain((/<p><a [^>]*property="schema:url"><span property="schema:name">([\s\S]*?)<\/span>/.exec(card) || [])[1]);
    const who = [...(/class="author"[\s\S]*?<\/p>/.exec(card) || [''])[0].matchAll(/<span property="schema:name">([\s\S]*?)<\/span>/g)].map(x => plain(x[1])).join(', '), img = (/<img src="([^"]+)"/.exec(card) || [])[1] || '';
    if (title) items.push({ ref: m[1], format: 'epub', title, author: who, summary: '', cover: img ? new URL(unent(img), src.base).href : '' });
  }
  return { items, more: new RegExp('href="[^"]*[?&;]page=' + (page + 1) + '[^0-9]').test(html) };
}
async function standardResolve(src, ref) {
  if (!/^\/ebooks\/[\w\/.-]+$/.test(ref) || ref.includes('..')) throw fail(400, 'Bad book');
  const html = await getText(src, src.base + ref, 'text/html'), links = [...html.matchAll(/href="([^"]*\/downloads\/[^"]+\.epub)"/g)].map(m => unent(m[1])).filter(h => !/kepub|_advanced/.test(h));
  const href = links[0] || `${ref}/downloads/${ref.replace(/^\/ebooks\//, '').split('/').join('_')}.epub`;
  return { url: new URL(href, src.base).href + '?source=download', format: 'epub' }; // (without this the site answers with a "your download has started" page)
}
async function archiveSearch(src, q, page) {
  const query = `(${q}) AND mediatype:texts AND format:EPUB AND -access-restricted-item:true`;
  const j = await getJson(src, `${src.base}/advancedsearch.php?q=${encodeURIComponent(query)}&fl[]=identifier&fl[]=title&fl[]=creator&rows=20&page=${page}&output=json`);
  const docs = (j.response && j.response.docs) || [], total = (j.response && j.response.numFound) || 0;
  return { items: docs.filter(d => /^[\w.-]+$/.test(d.identifier || '')).map(d => ({ ref: d.identifier, format: 'epub', title: [].concat(d.title || '')[0] || d.identifier, author: [].concat(d.creator || []).join(', '), summary: '', cover: `${src.base}/services/img/${d.identifier}` })), more: page * 20 < total };
}
async function archiveResolve(src, ref) {
  if (!/^[\w.-]+$/.test(ref)) throw fail(400, 'Bad book');
  const m = await getJson(src, `${src.base}/metadata/${ref}`);
  if (m.metadata && String(m.metadata['access-restricted-item']) === 'true') throw fail(403, 'That book is borrow-only (it has DRM) and cannot be added');
  const files = (m.files || []).filter(f => /\.epub$/i.test(f.name || '')); const f = files.find(x => /^EPUB$/i.test(x.format)) || files[0]; if (!f) throw fail(404, 'No EPUB file for that item');
  return { url: `${src.base}/download/${ref}/${f.name.split('/').map(encodeURIComponent).join('/')}`, format: 'epub' };
}

async function searchOne(src, q, page) {
  if (src.id === 'gutenberg') return gutenbergSearch(src, q, page);
  if (src.id === 'standardebooks') return standardSearch(src, q, page);
  if (src.id === 'archive') return q ? archiveSearch(src, q, page) : { items: [], more: false };
  return opdsSearch(src, q, page);
}
const stamp = (src, it) => ({ ...it, source: src.id, sourceName: src.name, cover: it.cover ? `/api/read/store/cover?s=${encodeURIComponent(src.id)}&u=${encodeURIComponent(it.cover)}` : '', coverUrl: it.cover || '' });
const listSources = () => [...Object.keys(BUILTIN), ...customs().map(c => c.id)].map(id => { const s = sourceOf(id); return { id, name: s.name, builtin: s.builtin, needsQuery: !!s.needsQuery }; });

async function handle(req, res, p) {
  const me = users.current();
  try {
    if (p === '/sources' && req.method === 'GET') return json(res, 200, listSources());
    if (p === '/sources' && req.method === 'POST') { // add an OPDS catalogue (admin)
      if (!me || me.role !== 'admin') throw fail(403, 'Only an admin can add a catalogue');
      const b = await readBody(req), name = String(b.name || '').trim().slice(0, 60), url = String(b.url || '').trim();
      if (!name || !/^https?:\/\/\S+$/i.test(url)) throw fail(400, 'Give the catalogue a name and its address (starting with http)');
      const id = 'c' + crypto.randomBytes(5).toString('hex'), entry = { id, name, url, auth: b.user ? seal({ user: String(b.user), pass: String(b.pass || '') }) : '' };
      const l = customs(); l.push(entry); saveCustoms(l);
      try { const src = sourceOf(id), xml = await getText(src, url, 'application/atom+xml'); if (!/<feed[\s>]/i.test(xml)) throw fail(502, 'That does not look like an OPDS catalogue'); }
      catch (e) { saveCustoms(customs().filter(x => x.id !== id)); throw fail(e.code || 502, e.code ? e.message : 'Could not read that catalogue: ' + e.message); }
      return json(res, 200, { id, name });
    }
    let m;
    if ((m = /^\/sources\/(\w+)$/.exec(p)) && req.method === 'DELETE') {
      if (!me || me.role !== 'admin') throw fail(403, 'Only an admin can remove a catalogue');
      saveCustoms(customs().filter(x => x.id !== m[1])); return json(res, 200, { removed: true });
    }
    if (p === '/search' && req.method === 'GET') {
      const u = new URL(req.url, 'http://x'), q = (u.searchParams.get('q') || '').trim().slice(0, 120), page = Math.min(20, Math.max(1, +u.searchParams.get('page') || 1)), want = u.searchParams.get('source') || 'all';
      const ids = want === 'all' ? listSources().map(s => s.id) : [want], lists = [], errors = {}; let more = false;
      await Promise.all(ids.map(async id => { let src; try { src = sourceOf(id); const r = await searchOne(src, q, page); lists.push([id, r.items.map(it => stamp(src, it))]); more = more || r.more; } catch (e) { const nm = src ? src.name : id, msg = e.name === 'TimeoutError' ? 'did not answer in time' : e.message; errors[id] = msg.startsWith(nm) ? msg : nm + ': ' + msg; } }));
      lists.sort((a, b) => ids.indexOf(a[0]) - ids.indexOf(b[0]));
      const items = []; for (let i = 0; lists.some(l => l[1][i]); i++) for (const l of lists) if (l[1][i]) items.push(l[1][i]); // one from each catalogue in turn
      return json(res, 200, { items, errors, more });
    }
    if (p === '/cover' && req.method === 'GET') {
      const u = new URL(req.url, 'http://x'), src = sourceOf(u.searchParams.get('s') || ''), r = await get(src, u.searchParams.get('u') || '', { accept: 'image/*' }), type = (r.headers.get('content-type') || '').split(';')[0].trim().toLowerCase();
      if (!/^image\/(jpeg|png|gif|webp)$/.test(type)) throw fail(502, 'Not a picture');
      const buf = await limited(r, MAX_COVER); res.writeHead(200, { 'content-type': type, 'cache-control': 'private, max-age=604800' }); return res.end(buf);
    }
    if (p === '/add' && req.method === 'POST') {
      const b = await readBody(req), src = sourceOf(String(b.source || '')), ref = String(b.ref || '');
      let target;
      if (src.id === 'gutenberg') target = await gutenbergResolve(src, ref);
      else if (src.id === 'standardebooks') target = await standardResolve(src, ref);
      else if (src.id === 'archive') target = await archiveResolve(src, ref);
      else { if (!sameSite(src, new URL(ref, src.base))) throw fail(400, 'Bad book address'); target = { url: ref, format: String(b.format || 'epub') }; }
      const ext = books.TYPES[target.format] ? target.format : 'epub';
      const buf = await limited(await get(src, target.url, { accept: 'application/epub+zip, application/octet-stream, */*;q=0.5', timeout: 120000 }), MAX_BOOK);
      let cover; const cu = String(b.cover || ''); if (cu) { try { const r = await get(src, cu, { accept: 'image/*' }), type = (r.headers.get('content-type') || '').split(';')[0].trim().toLowerCase(); cover = { type, data: await limited(r, MAX_COVER) }; } catch {} }
      let book; try { book = books.addBook(buf, ext, { title: String(b.title || '').slice(0, 200), author: String(b.author || '').slice(0, 200) }, cover); } catch (e) { throw fail(400, e.message); }
      return json(res, 200, book);
    }
    return json(res, 404, { error: 'Not found' });
  } catch (e) { return json(res, e.code && e.code >= 400 && e.code < 600 ? e.code : 502, { error: e.name === 'TimeoutError' ? 'The site did not answer in time' : e.message }); }
}

module.exports = { handle, parseFeed };