Store: Smashwords as a family-filtered book source; free audiobooks stream through the hub; extensions and Sites on Home

- Smashwords: free EPUBs from its shelf feed, filtered (adult/erotica flags, LGBTQ+, keywords, unlabelled romance); admin saves a Smashwords sign-in (encrypted) used to download - LibriVox chapter audio is fetched and streamed by the hub (with seeking) instead of redirecting the browser to archive.org - Home: a My apps section with installed extensions and Sites, also in the command palette Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-26 10:14:13 -04:00
1 parent 3c42d1ecd5
commit dc8265a850
@@ -0,0 +1,139 @@
+// Smashwords as a book source for the Store: the free, DRM-free EPUBs. Browsing uses Smashwords' own shelf feed (no sign-in needed) and every book is run through a
+// strict family filter first: anything Smashwords flags as adult or erotica, anything filed under LGBTQ+, and anything whose categories, tags or description say so, is
+// dropped. Downloading a book needs a (free) Smashwords account, so an admin saves its sign-in once; it is kept encrypted on the server and used only to sign in at
+// smashwords.com. Smashwords has no public search, so searching looks through a remembered set of its popular free books.
+const fs = require('fs'), path = require('path'), crypto = require('crypto');
+const atomicWrite = require('./atomic');
+const auth = require('./auth');
+
+const BASE = () => (process.env.SMASHWORDS_URL || 'https://www.smashwords.com').replace(/\/+$/, '');
+const COVERS = () => (process.env.SMASHWORDS_COVERS || 'https://dwtr67e3ikfml.cloudfront.net').replace(/\/+$/, '');
+const LANG = () => process.env.SMASHWORDS_LANG || '1'; // 1 = English
+const UA = 'Mozilla/5.0 (compatible; NE-Hub/1.0; self-hosted e-book reader)';
+const DATA_DIR = process.env.DATA_DIR || path.join(__dirname, 'data'), FILE = () => path.join(DATA_DIR, 'store-smashwords.json');
+const MAX_BOOK = 300 * 1024 * 1024;
+const fail = (code, message) => Object.assign(new Error(message), { code });
+
+// ---- the family filter
+const LGBTQ = '886'; // Smashwords' own category for it
+const BAD_WORDS = '(?:erotic\\w*|erotica|bdsm|smut\\w*|lgbt\\w*|queer|lesbian|bisexual|transgender|non-?binary|same-sex|m/m|f/f|m/f/m|sex scenes?|sexually explicit|explicit (?:sex|content|scenes?)|adult (?:content|themes?|romance|fiction)|mature (?:content|audiences?|themes?)|steamy|spicy romance|harem|omegaverse|gay (?:romance|fiction|love|couple|men|man|male|teen|characters?|protagonist)|18\\+|nsfw)';
+const TEXT_BAD = new RegExp('(?<![\\w/])' + BAD_WORDS + '(?![\\w/])', 'i');
+const TAG_BAD = new RegExp('(?<![\\w/])(?:' + BAD_WORDS + '|gay|trans (?:man|woman|girl|boy|character\\w*)|pegging|kink\\w*|fetish\\w*|alpha male)(?![\\w/])', 'i');
+const ROMANCE = '1235';
+const CLEAN = /\b(clean|sweet|wholesome|christian|inspirational|faith|cozy|family[- ]friendly|no sex|closed door|inspy|regency)\b/i;
+const CAT_BAD = /erotic|lgbt|gay\b|lesbian|queer|bdsm|adult|mature|fetish|taboo/i;
+const plain = s => String(s || '').replace(/<[^>]+>/g, ' ').replace(/&nbsp;/g, ' ').replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"').replace(/&#0?39;/g, "'").replace(/\s+/g, ' ').trim();
+/** true when a book (Smashwords' own record for it) is fine to show. */
+function keep(b) {
+  if (!b || b.adult || b.erotica || b.taboo_erotica || b.need_erotic_cert) return false;
+  for (const c of b.categories || []) {
+    if (String(c.id) === LGBTQ || CAT_BAD.test(c.name || '')) return false;
+    for (const p of c.parents || []) if (String(p.id) === LGBTQ || CAT_BAD.test(p.name || '')) return false;
+  }
+  if ((b.tags || []).some(t => TAG_BAD.test(String(t)))) return false;
+  // Romance is where adult content hides without being flagged, so a romance is only shown when it says it is clean (sweet, Christian, inspirational...)
+  const romance = (b.categories || []).some(c => String(c.id) === ROMANCE || /romance/i.test(c.name || '') || (c.parents || []).some(p => String(p.id) === ROMANCE || /romance/i.test(p.name || ''))) || (b.tags || []).some(t => /romance/i.test(String(t)));
+  if (romance && !CLEAN.test([...(b.tags || []), ...(b.categories || []).flatMap(c => [c.name, ...(c.parents || []).map(p => p.name)])].join(' | '))) return false;
+  if (TEXT_BAD.test(plain(b.title) + ' ' + plain(b.short_description) + ' ' + plain(b.long_description))) return false;
+  return true;
+}
+const authorOf = b => (b.contributors || []).filter(c => !c.role || /^A0?1$/.test(c.role)).map(c => c.account && c.account.display_name).filter(Boolean).slice(0, 3).join(', ');
+const free = b => !b.price || (b.price.prices || []).every(p => +p.amount === 0) || b.price.type === 'FREE';
+const hasEpub = b => !b.formats || b.formats.some(f => /^epub$/i.test(f));
+const item = b => ({ ref: String(b.id), format: 'epub', title: plain(b.title), author: authorOf(b), summary: plain(b.short_description).slice(0, 600), cover: b.cover_url ? COVERS() + '/' + String(b.cover_url).replace(/^\/+/, '') : '', _t: [b.title, authorOf(b), (b.tags || []).join(' '), plain(b.short_description)].join(' ').toLowerCase() });
+const clean = ({ _t, ...rest }) => rest;
+
+// ---- Smashwords' shelf feed
+async function getJson(url) {
+  let r; try { r = await fetch(url, { headers: { 'user-agent': UA, accept: 'application/json' }, signal: AbortSignal.timeout(20000) }); } catch (e) { throw fail(502, e.name === 'TimeoutError' ? 'Smashwords did not answer in time' : 'Could not reach Smashwords'); }
+  if (!r.ok) throw fail(502, 'Smashwords answered ' + r.status);
+  try { return await r.json(); } catch { throw fail(502, 'Smashwords sent something unexpected'); }
+}
+const shelf = (type, cat, page) => getJson(`${BASE()}/api/bookshelves/category/v3/shelf/${type}/${cat}/free/any/no_erotica/${LANG()}/${page}/10`);
+const PER_API_PAGE = 10, API_PAGES_PER_PAGE = 3; // a page of the Store is three of Smashwords' pages of ten, since the filter removes some
+
+/** The most downloaded free books, a page at a time. */
+async function browse(page) {
+  const first = (page - 1) * API_PAGES_PER_PAGE + 1, pages = await Promise.all([0, 1, 2].map(i => shelf('mostdownloads', 1, first + i).catch(e => { if (i === 0) throw e; return { book_list: [], total_books: 0 }; })));
+  const seen = new Set(), items = [];
+  for (const p of pages) for (const e of p.book_list || []) { const b = e.book; if (b && !seen.has(b.id) && free(b) && hasEpub(b) && keep(b)) { seen.add(b.id); items.push(clean(item(b))); } }
+  const total = pages[0].total_books || 0;
+  return { items, more: first + API_PAGES_PER_PAGE - 1 < Math.ceil(total / PER_API_PAGE) && (pages[2].book_list || []).length > 0 };
+}
+
+// Smashwords has no public search, so one is made from its three best free shelves (most downloaded, newest, highest rated), remembered for a few hours.
+let poolP = null, poolAt = 0, poolDone = false;
+async function buildPool() {
+  const seen = new Set(), out = [];
+  for (const type of ['mostdownloads', 'highlyrated', 'newest']) {
+    const first = await shelf(type, 1, 1), n = Math.min(16, Math.ceil((first.total_books || 10) / PER_API_PAGE)), pages = [first];
+    for (let i = 2; i <= n; i += 6) pages.push(...await Promise.all(Array.from({ length: Math.min(6, n - i + 1) }, (_, k) => shelf(type, 1, i + k).catch(() => ({ book_list: [] })))));
+    for (const p of pages) for (const e of p.book_list || []) { const b = e.book; if (b && !seen.has(b.id) && free(b) && hasEpub(b) && keep(b)) { seen.add(b.id); out.push(item(b)); } }
+  }
+  return out;
+}
+const pool = () => { if (!poolP || Date.now() - poolAt > 6 * 3600e3) { poolAt = Date.now(); poolDone = false; poolP = buildPool().then(l => { poolDone = true; return l; }, e => { poolP = null; throw e; }); } return poolP; };
+async function search(q, page, wait = true) { // (wait: false is for a search across every source, which must not stall for the first-time build; it starts and the next search has it)
+  const words = q.toLowerCase().split(/\s+/).filter(Boolean);
+  if (!words.length) return browse(page);
+  if (!wait && !poolDone) { pool().catch(() => {}); return { items: [], more: false }; }
+  const hits = (await pool()).filter(b => words.every(w => b._t.includes(w)));
+  return { items: hits.slice((page - 1) * 20, page * 20).map(clean), more: hits.length > page * 20 };
+}
+
+// ---- the sign-in (admin saves it once; encrypted with a key from the server secret)
+let _key; const key = () => _key || (_key = crypto.scryptSync(auth.secret, 'nehub-smashwords', 32));
+const seal = v => { const iv = crypto.randomBytes(12), c = crypto.createCipheriv('aes-256-gcm', key(), iv), d = Buffer.concat([c.update(JSON.stringify(v), 'utf8'), c.final()]); return Buffer.concat([iv, c.getAuthTag(), d]).toString('base64'); };
+const unseal = s => { try { const b = Buffer.from(s, 'base64'), d = crypto.createDecipheriv('aes-256-gcm', key(), b.subarray(0, 12)); d.setAuthTag(b.subarray(12, 28)); return JSON.parse(Buffer.concat([d.update(b.subarray(28)), d.final()]).toString('utf8')); } catch { return null; } };
+const saved = () => { try { return unseal(JSON.parse(fs.readFileSync(FILE(), 'utf8')).data); } catch { return null; } };
+const account = () => { const s = saved(); return { signedIn: !!s, email: s ? s.email : '' }; };
+let session = null; // { cookie, t }
+function signOut() { try { fs.unlinkSync(FILE()); } catch {} session = null; }
+
+/** Cookies from a response ("name=value" pairs), merged into the jar. */
+function eat(jar, r) { for (const c of (r.headers.getSetCookie ? r.headers.getSetCookie() : [])) { const [kv] = c.split(';'), i = kv.indexOf('='); if (i > 0) { const v = kv.slice(i + 1).trim(); if (v && v !== 'deleted') jar.set(kv.slice(0, i).trim(), v); else jar.delete(kv.slice(0, i).trim()); } } }
+const header = jar => [...jar].map(([k, v]) => k + '=' + v).join('; ');
+async function req(url, jar, init = {}) { // one request; redirects are followed by hand so the cookies from each step are kept, and only smashwords' own address is used
+  for (let hop = 0; hop < 6; hop++) {
+    const u = new URL(url); if (u.origin !== new URL(BASE()).origin) throw fail(502, 'That address is not part of Smashwords');
+    const r = await fetch(u, { ...init, headers: { 'user-agent': UA, ...(init.headers || {}), cookie: header(jar) }, redirect: 'manual', signal: AbortSignal.timeout(init.timeout || 30000) }); eat(jar, r);
+    if ([301, 302, 303, 307, 308].includes(r.status) && r.headers.get('location')) { url = new URL(r.headers.get('location'), u).href; if (r.status !== 307 && r.status !== 308) { const { 'content-type': _c, ...hs } = init.headers || {}; init = { ...init, method: 'GET', body: undefined, headers: hs }; } try { await r.body?.cancel(); } catch {} if (/\/(signin|signup\/login)\b/.test(url) && init.stopAtLogin) return { r: { status: 401, headers: new Headers(), body: null }, url }; continue; }
+    return { r, url };
+  }
+  throw fail(502, 'Too many redirects from Smashwords');
+}
+async function login(email, pass) {
+  const jar = new Map(), page = await req(BASE() + '/signin', jar), html = await page.r.text(), token = (/name="loginToken"\s+value="([^"]*)"/.exec(html) || [])[1];
+  if (!token) throw fail(502, 'Smashwords\' sign-in page was not what was expected');
+  const out = await req(BASE() + '/auth/index', jar, { method: 'POST', headers: { 'content-type': 'application/x-www-form-urlencoded', referer: BASE() + '/signin' }, body: new URLSearchParams({ email, password: pass, loginToken: token, secToken: '' }).toString() });
+  const home = await (await req(BASE() + '/', jar)).r.text(); // the front page says whether this cookie is a signed-in one
+  if (!/loggedIn\s*=\s*true/.test(home)) { try { await out.r.body?.cancel(); } catch {} throw fail(401, 'Smashwords did not accept that email and password'); }
+  return jar;
+}
+/** Check a sign-in and save it. */
+async function signIn(email, pass) {
+  email = String(email || '').trim(); pass = String(pass || ''); if (!/^\S+@\S+\.\S+$/.test(email) || !pass) throw fail(400, 'Enter your Smashwords email and password');
+  const jar = await login(email, pass); session = { jar, t: Date.now() };
+  fs.mkdirSync(DATA_DIR, { recursive: true }); atomicWrite(FILE(), JSON.stringify({ data: seal({ email, pass }) }), { mode: 0o600 }); return account();
+}
+async function jar() {
+  if (session && Date.now() - session.t < 3 * 3600e3) return session.jar;
+  const s = saved(); if (!s) throw fail(401, 'An admin needs to add a Smashwords sign-in first (Store, Books, Catalogues)');
+  session = { jar: await login(s.email, s.pass), t: Date.now() }; return session.jar;
+}
+/** The EPUB of a free book, as the signed-in account. */
+async function download(id) {
+  if (!/^\d{1,9}$/.test(id)) throw fail(400, 'Bad book');
+  for (let attempt = 0; attempt < 2; attempt++) {
+    const j = await jar(), { r } = await req(`${BASE()}/books/download/${id}/8/latest/0/0/book.epub`, j, { timeout: 120000, stopAtLogin: true, headers: { accept: 'application/epub+zip, application/octet-stream, */*;q=0.5' } });
+    if (r.status === 401) { session = null; continue; } // the saved cookie ran out: sign in again once
+    if (!r.ok) throw fail(r.status === 404 ? 404 : 502, 'Smashwords answered ' + r.status);
+    if ((+r.headers.get('content-length') || 0) > MAX_BOOK) throw fail(413, 'That file is too large');
+    const chunks = []; let n = 0; for await (const c of r.body) { n += c.length; if (n > MAX_BOOK) throw fail(413, 'That file is too large'); chunks.push(c); }
+    const buf = Buffer.concat(chunks); if (buf.subarray(0, 2).toString('latin1') !== 'PK') throw fail(502, 'Smashwords did not send an EPUB for that book (it may not be free, or may not have one)');
+    return buf;
+  }
+  throw fail(401, 'Smashwords would not sign in with the saved email and password');
+}
+
+module.exports = { keep, browse, search, account, signIn, signOut, download, item, COVERS };