Book store: Project Gutenberg through its own catalogue feed (Gutendex timed out), Standard Ebooks through its public pages (its OPDS feed needs a paid sign-in)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UaC9HmtHMdRP8dAbN4wAKj

store.js

@@ -1,4 +1,4 @@
-// Book store: free, DRM-free books from Project Gutenberg (Gutendex), Standard Ebooks (OPDS), the Internet Archive (public-domain EPUBs) and any OPDS catalogue an
+// Book store: free, DRM-free books from Project Gutenberg (its own catalogue feed), Standard Ebooks (its public pages), the Internet Archive (public-domain EPUBs) and any OPDS catalogue an
 // admin adds (Calibre-Web, Kavita, ...). The hub server does all the talking to those sites: searching, covers and downloading a chosen book straight onto the
 // signed-in person's shelf (books.js), so the browser never contacts them and no catalogue sign-in reaches the browser.
 const fs = require('fs'), path = require('path'), crypto = require('crypto');
@@ -8,8 +8,8 @@ const UA = 'NE-Hub/1.0 (self-hosted e-book reader)';
 const MAX_BOOK = 300 * 1024 * 1024, MAX_COVER = 3 * 1024 * 1024, MAX_TEXT = 4 * 1024 * 1024;
 const env = (k, d) => (process.env[k] || d).replace(/\/+$/, '');
 const BUILTIN = {
-  gutenberg: { name: 'Project Gutenberg', base: () => env('GUTENDEX_URL', 'https://gutendex.com'), hosts: ['gutendex.com', 'gutenberg.org'] },
-  standardebooks: { name: 'Standard Ebooks', base: () => env('STANDARD_EBOOKS_URL', 'https://standardebooks.org/feeds/opds'), hosts: ['standardebooks.org'], opds: true },
+  gutenberg: { name: 'Project Gutenberg', base: () => env('GUTENBERG_URL', 'https://www.gutenberg.org'), hosts: ['gutenberg.org'] }, // its own catalogue feed (fast and official)
+  standardebooks: { name: 'Standard Ebooks', base: () => env('STANDARD_EBOOKS_URL', 'https://standardebooks.org'), hosts: ['standardebooks.org'] }, // its public pages: the OPDS feed is for paying members only
   archive: { name: 'Internet Archive', base: () => env('ARCHIVE_URL', 'https://archive.org'), hosts: ['archive.org'], needsQuery: true },
 };
 const FORMAT_OF = { 'application/epub+zip': 'epub', 'application/pdf': 'pdf', 'application/x-mobipocket-ebook': 'mobi', 'application/vnd.amazon.mobi8-ebook': 'azw3', 'application/vnd.amazon.ebook': 'azw' };
@@ -34,7 +34,7 @@ const sourceOf = id => {
 
 // ---- talking to a site: only its own addresses are ever fetched (redirects included), a size limit applies, and a catalogue's sign-in goes nowhere else
 const sameSite = (src, u) => { const h = u.hostname; return src.builtin ? src.hosts.some(x => h === x || h.endsWith('.' + x)) : u.origin === new URL(src.base).origin; };
-async function get(src, url, { accept = '*/*', timeout = 10000 } = {}) {
+async function get(src, url, { accept = '*/*', timeout = 15000 } = {}) {
   let u = new URL(url);
   for (let hop = 0; hop < 6; hop++) {
     if (!/^https?:$/.test(u.protocol) || !sameSite(src, u)) throw fail(502, 'That address is not part of ' + src.name);
@@ -57,6 +57,7 @@ const getJson = async (src, url) => { try { return JSON.parse(await getText(src,
 // ---- OPDS (Atom) feeds, read with a small parser (no XML library in the hub)
 const unent = s => String(s || '').replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1').replace(/&(#x[0-9a-f]+|#\d+|amp|lt|gt|quot|apos);/gi, (m, e) => ({ amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }[e.toLowerCase()] || (e[1] === 'x' || e[1] === 'X' ? String.fromCodePoint(parseInt(e.slice(2), 16)) : e[0] === '#' ? String.fromCodePoint(+e.slice(1)) : m)));
 const tag = (x, t) => { const m = new RegExp('<' + t + '(?:\\s[^>]*)?>([\\s\\S]*?)</' + t + '>', 'i').exec(x); return m ? unent(m[1]).replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim() : ''; };
+const plain = s => unent(String(s || '').replace(/<[^>]+>/g, ' ')).replace(/\s+/g, ' ').trim();
 const attrs = s => { const o = {}; for (const m of s.matchAll(/([\w:-]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g)) o[m[1].toLowerCase()] = unent(m[2] != null ? m[2] : m[3]); return o; };
 const linksOf = (x, base) => [...x.matchAll(/<link\b([^>]*?)\/?>/gi)].map(m => attrs(m[1])).filter(a => a.href).map(a => ({ ...a, href: new URL(a.href, base).href }));
 function parseFeed(xml, base) {
@@ -78,7 +79,6 @@ async function opdsSearchUrl(src, q) {
   if (atom) return fill(unbrace(atom.href));
   const osd = root.links.find(l => l.rel === 'search' && /opensearchdescription/.test(l.type || ''));
   if (osd) { const x = await getText(src, osd.href, 'application/xml'); for (const m of x.matchAll(/<Url\b([^>]*?)\/?>/gi)) { const a = attrs(m[1]); if (/atom\+xml/.test(a.type || '') && a.template) return fill(unbrace(new URL(a.template.replace(/\{/g, '%7B').replace(/\}/g, '%7D'), osd.href).href)); } }
-  if (src.id === 'standardebooks') return src.base + '/all?query=' + enc;
   throw fail(502, src.name + ' does not offer a search');
 }
 async function opdsSearch(src, q, page) {
@@ -91,17 +91,33 @@ async function opdsSearch(src, q, page) {
   return { items, more: !!feed.links.find(l => l.rel === 'next') };
 }
 
-// ---- Project Gutenberg through Gutendex, and the Internet Archive
-const tidyName = n => { const m = /^([^,]+),\s*(.+)$/.exec(n || ''); return m ? (m[2] + ' ' + m[1]).replace(/\s*\(.*?\)/g, '').trim() : String(n || ''); };
+// ---- Project Gutenberg (its own OPDS catalogue), Standard Ebooks (its public pages), and the Internet Archive
+const entriesOf = xml => [...xml.matchAll(/<entry[\s>][\s\S]*?<\/entry>/gi)].map(m => m[0]);
 async function gutenbergSearch(src, q, page) {
-  const j = await getJson(src, `${src.base}/books/?${q ? 'search=' + encodeURIComponent(q) + '&' : ''}page=${page}`);
-  const items = (j.results || []).filter(b => (b.formats || {})['application/epub+zip']).map(b => ({ ref: String(b.id), format: 'epub', title: b.title, author: (b.authors || []).map(a => tidyName(a.name)).join(', '), summary: (b.subjects || []).slice(0, 3).join(' · '), cover: (b.formats || {})['image/jpeg'] || '' }));
-  return { items, more: !!j.next };
+  const url = `${src.base}/ebooks/search.opds/?${q ? 'query=' + encodeURIComponent(q) : 'sort_order=downloads'}${page > 1 ? '&start_index=' + ((page - 1) * 25 + 1) : ''}`;
+  const xml = await getText(src, url, 'application/atom+xml'), items = [], seen = new Set();
+  for (const e of entriesOf(xml)) { const id = (/\/ebooks\/(\d+)\.opds/.exec(tag0(e, 'id')) || [])[1]; if (!id || seen.has(id)) continue; seen.add(id); items.push({ ref: id, format: 'epub', title: tag(e, 'title'), author: tag(e, 'content'), summary: '', cover: `${src.base}/cache/epub/${id}/pg${id}.cover.medium.jpg` }); }
+  return { items, more: /<link[^>]+rel="next"/i.test(xml.replace(/<entry[\s>][\s\S]*?<\/entry>/gi, '')) };
 }
+const tag0 = (x, t) => (new RegExp('<' + t + '>([^<]*)</' + t + '>', 'i').exec(x) || [])[1] || '';
 async function gutenbergResolve(src, ref) {
   if (!/^\d{1,9}$/.test(ref)) throw fail(400, 'Bad book');
-  const b = await getJson(src, `${src.base}/books/${ref}`), url = (b.formats || {})['application/epub+zip']; if (!url) throw fail(404, 'No EPUB for that book');
-  return { url, format: 'epub' };
+  return { url: `${src.base}/ebooks/${ref}.epub3.images`, format: 'epub' }; // (the address Project Gutenberg itself gives for the EPUB with pictures)
+}
+async function standardSearch(src, q, page) {
+  const html = await getText(src, `${src.base}/ebooks?${q ? 'query=' + encodeURIComponent(q) + '&' : ''}page=${page}&per-page=24`, 'text/html, application/xhtml+xml'), items = [];
+  for (const m of html.matchAll(/<li typeof="schema:Book" about="(\/ebooks\/[^"]+)">([\s\S]*?)<\/li>/g)) {
+    const card = m[2], title = plain((/<p><a [^>]*property="schema:url"><span property="schema:name">([\s\S]*?)<\/span>/.exec(card) || [])[1]);
+    const who = [...(/class="author"[\s\S]*?<\/p>/.exec(card) || [''])[0].matchAll(/<span property="schema:name">([\s\S]*?)<\/span>/g)].map(x => plain(x[1])).join(', '), img = (/<img src="([^"]+)"/.exec(card) || [])[1] || '';
+    if (title) items.push({ ref: m[1], format: 'epub', title, author: who, summary: '', cover: img ? new URL(unent(img), src.base).href : '' });
+  }
+  return { items, more: new RegExp('href="[^"]*[?&;]page=' + (page + 1) + '[^0-9]').test(html) };
+}
+async function standardResolve(src, ref) {
+  if (!/^\/ebooks\/[\w\/.-]+$/.test(ref) || ref.includes('..')) throw fail(400, 'Bad book');
+  const html = await getText(src, src.base + ref, 'text/html'), links = [...html.matchAll(/href="([^"]*\/downloads\/[^"]+\.epub)"/g)].map(m => unent(m[1])).filter(h => !/kepub|_advanced/.test(h));
+  const href = links[0] || `${ref}/downloads/${ref.replace(/^\/ebooks\//, '').split('/').join('_')}.epub`;
+  return { url: new URL(href, src.base).href + '?source=download', format: 'epub' }; // (without this the site answers with a "your download has started" page)
 }
 async function archiveSearch(src, q, page) {
   const query = `(${q}) AND mediatype:texts AND format:EPUB AND -access-restricted-item:true`;
@@ -119,6 +135,7 @@ async function archiveResolve(src, ref) {
 
 async function searchOne(src, q, page) {
   if (src.id === 'gutenberg') return gutenbergSearch(src, q, page);
+  if (src.id === 'standardebooks') return standardSearch(src, q, page);
   if (src.id === 'archive') return q ? archiveSearch(src, q, page) : { items: [], more: false };
   return opdsSearch(src, q, page);
 }
@@ -161,6 +178,7 @@ async function handle(req, res, p) {
       const b = await readBody(req), src = sourceOf(String(b.source || '')), ref = String(b.ref || '');
       let target;
       if (src.id === 'gutenberg') target = await gutenbergResolve(src, ref);
+      else if (src.id === 'standardebooks') target = await standardResolve(src, ref);
       else if (src.id === 'archive') target = await archiveResolve(src, ref);
       else { if (!sameSite(src, new URL(ref, src.base))) throw fail(400, 'Bad book address'); target = { url: ref, format: String(b.format || 'epub') }; }
       const ext = books.TYPES[target.format] ? target.format : 'epub';