From dc8265a85091f268ee183f44584e2c9de7eb1a07 Mon Sep 17 00:00:00 2001 From: nmemmert <67961292+nmemmert@users.noreply.github.com> Date: Mon, 26 Oct 2026 10:14:13 -0400 Subject: [PATCH] Store: Smashwords as a family-filtered book source; free audiobooks stream through the hub; extensions and Sites on Home - Smashwords: free EPUBs from its shelf feed, filtered (adult/erotica flags, LGBTQ+, keywords, unlabelled romance); admin saves a Smashwords sign-in (encrypted) used to download - LibriVox chapter audio is fetched and streamed by the hub (with seeking) instead of redirecting the browser to archive.org - Home: a My apps section with installed extensions and Sites, also in the command palette Co-Authored-By: Claude Sonnet 5.5 --- README.md | 4 ++ librivox.js | 17 ++++- public/app.js | 3 +- public/palette.js | 1 + public/reader.js | 6 +- public/store-view.js | 12 ++++ public/sw.js | 2 +- smashwords.js | 139 +++++++++++++++++++++++++++++++++++++++++ store.js | 18 ++++-- tools/librivox-test.js | 5 +- tools/store-test.js | 31 +++++++-- 11 files changed, 223 insertions(+), 15 deletions(-) create mode 100644 smashwords.js diff --git a/README.md b/README.md index 9d1d865..e6c4814 100644 --- a/README.md +++ b/README.md @@ -164,3 +164,7 @@ One **Store** tab (turn it on per person) with sections for **Books** (the free Files are kept in the data folder under `mystore/` (up to 1 GB each; extensions up to 2 MB). Only an admin can add, change or delete items. +**Smashwords** is a book source in the Store (Books). It lists Smashwords' most downloaded free EPUBs through its own shelf feed, filtered for a family: anything Smashwords flags as adult or erotica, anything in LGBTQ+, anything whose categories, tags or description say so, and any romance that does not say it is clean (sweet, Christian, inspirational...). The filter is automatic and cannot be perfect, so a book can occasionally slip through. Smashwords has no public search, so searching looks through a remembered set of about 400 of its popular free books. Downloading needs a free Smashwords account: an admin enters it once under Books, *Catalogues* (kept encrypted on the server, used only to sign in at smashwords.com). `SMASHWORDS_LANG` (default `1`, English) picks the language. + +**My apps on Home:** the extensions a person installed from My Store, and any Sites in My Store, appear under *My apps* on Home and in the command palette. + diff --git a/librivox.js b/librivox.js index 0307d89..06a9620 100644 --- a/librivox.js +++ b/librivox.js @@ -45,9 +45,22 @@ async function handle(req, res) { return json(res, 200, { items: items.map(b => ({ ...b, cover: b.cover ? `/api/ab/free/cover/${b.id}?u=${encodeURIComponent(b.cover)}` : '' })), more: lists.flat().length >= 20 }); } if ((m = /^\/book\/(\d+)$/.exec(p)) && req.method === 'GET') return json(res, 200, publicBook(await book(m[1]))); - if ((m = /^\/audio\/(\d+)\/(\d+)$/.exec(p)) && req.method === 'GET') { // the recording itself is on the Internet Archive; the browser is sent straight there (it supports seeking) + if ((m = /^\/audio\/(\d+)\/(\d+)$/.exec(p)) && req.method === 'GET') { // the recording is on the Internet Archive; the hub fetches it (redirects and all) and passes it on, with seeking const b = await book(m[1]), c = b.chapters[+m[2]]; if (!c) throw fail(404, 'No such chapter'); - res.writeHead(302, { location: c.url, 'cache-control': 'private, max-age=600', 'referrer-policy': 'no-referrer' }); return res.end(); + const ctl = new AbortController(); res.on('close', () => ctl.abort()); + let url = c.url, r; + for (let i = 0; i < 6; i++) { + if (!allowed(url)) throw fail(502, 'That audio address is not allowed'); + r = await fetch(url, { headers: { 'user-agent': 'NE-Hub/1.0', 'accept-encoding': 'identity', ...(req.headers.range ? { range: req.headers.range } : {}) }, redirect: 'manual', signal: ctl.signal }).catch(e => { throw fail(502, e.name === 'AbortError' ? 'Cancelled' : 'The Internet Archive did not answer'); }); + if ([301, 302, 303, 307, 308].includes(r.status) && r.headers.get('location')) { url = new URL(r.headers.get('location'), url).href; try { await r.body?.cancel(); } catch {} continue; } + break; + } + if (r.status === 416) { res.writeHead(416, { 'content-range': r.headers.get('content-range') || 'bytes */0' }); return res.end(); } + if (!r.ok && r.status !== 206) throw fail(502, 'The Internet Archive answered ' + r.status + ' for that chapter'); + const out = { 'content-type': /^audio\//.test(r.headers.get('content-type') || '') ? r.headers.get('content-type') : 'audio/mpeg', 'accept-ranges': 'bytes', 'cache-control': 'private, max-age=600', 'referrer-policy': 'no-referrer' }; + for (const k of ['content-length', 'content-range']) if (r.headers.get(k)) out[k] = r.headers.get(k); + res.writeHead(r.status, out); if (!r.body) return res.end(); + return require('stream').Readable.fromWeb(r.body).on('error', () => res.destroy()).pipe(res); } if ((m = /^\/cover\/(\d+)$/.exec(p)) && req.method === 'GET') { let cu = u.searchParams.get('u'); if (!cu || !allowed(cu)) { const b = await book(m[1]); cu = b.cover; } if (!cu || !allowed(cu)) throw fail(404, 'No cover'); diff --git a/public/app.js b/public/app.js index fdce390..64cb571 100644 --- a/public/app.js +++ b/public/app.js @@ -353,7 +353,8 @@ function viewHome() { ${recents.length && allowed('files') ? `

Recent files

${recents.map((r, i) => ``).join('')}
` : ''} ${evening ? '' : '
'} -
${homeApps()}
`; +
${homeApps()}
`; + if (typeof stHomeExt === 'function' && allowed('store')) stHomeExt($('#homeext')); if (typeof homeResume === 'function') homeResume($('#resume')); renderNpCard(); // the live Now Playing card (it follows the song and has the controls) $('#view').querySelectorAll('button[data-r]').forEach(b => b.onclick = () => { diff --git a/public/palette.js b/public/palette.js index 199a0a3..f04776d 100644 --- a/public/palette.js +++ b/public/palette.js @@ -54,6 +54,7 @@ function openPalette() { const out = navTabs().flatMap(t => MERGED[t[0]] ? MERGED[t[0]][1].filter(allowed).map(id => [id, tabLabel(id)]) : [t]).map(([id, label]) => ({ label: 'Go to ' + label, hint: (navGroups().find(g => g.tabs.some(t => t[0] === (leafOf(id) || id))) || {}).label || 'Page', ic: TAB_ICON[id], run: go('#/' + id), kw: label })); if (phone.matches) out.push({ label: 'Edit the bottom bar', hint: 'View', ic: 'menu', run: () => openPinsSheet(), kw: 'menu tabs pins more' }); if (allowed('tasks') && allowed('calendar')) out.push({ label: 'Go to Tasks', hint: 'Planner', ic: 'check', run: go('#/tasks'), kw: 'todo' }, { label: 'Go to Calendar', hint: 'Planner', ic: 'cal', run: go('#/calendar'), kw: 'agenda events' }); + if (typeof stItems !== 'undefined' && allowed('store')) for (const i of stItems) { if (i.kind === 'extension' && stInstalled.includes(i.id)) out.push({ label: 'Open ' + i.title, hint: 'Extension', ic: 'cog', run: go('#/store/ext/' + i.id), kw: 'app extension ' + i.title.toLowerCase() }); else if (i.kind === 'link') out.push({ label: 'Open ' + i.title, hint: 'Site', ic: 'link', run: i.embed ? go('#/store/site/' + i.id) : () => window.open(i.url, '_blank', 'noopener,noreferrer'), kw: 'app site ' + i.title.toLowerCase() }); } if (allowed('tasks')) out.push({ label: 'New task', hint: 'Create', ic: 'plus', run: () => quickAdd(), kw: 'add todo' }); if (allowed('calendar')) out.push({ label: 'New event', hint: 'Create', ic: 'plus', run: () => newEvent(ymd(new Date())), kw: 'add calendar' }); if (allowed('mail') && cfg.mail) out.push({ label: 'New email', hint: 'Create', ic: 'mail', run: () => { location.hash = '#/mail'; setTimeout(() => mailCompose(), 300); }, kw: 'compose write message' }); diff --git a/public/reader.js b/public/reader.js index 7ab1f28..a572e23 100644 --- a/public/reader.js +++ b/public/reader.js @@ -198,10 +198,14 @@ async function rdStore(host) { // host: { el, hash } when shown inside the Store }; const catalogues = () => { const el = openSheet('Catalogues', 'Anything that speaks OPDS: Calibre-Web, Kavita, Komga, Standard Ebooks and others', '
'); - const list = async () => { sources = await rdApi('/store/sources'); chips(); el.querySelector('#rdcl').innerHTML = `
${sources.map(x => `
${esc(x.name)}${x.builtin ? 'Built in' : 'Added by you'}${x.builtin ? '' : ``}
`).join('')}
+ const list = async () => { sources = await rdApi('/store/sources'); chips(); const sw = sources.find(x => x.id === 'smashwords'), acct = sw && sw.account; el.querySelector('#rdcl').innerHTML = `${sw ? `
Smashwords account${acct && acct.signedIn ? 'Signed in as ' + esc(acct.email) + '. Books are downloaded with this account.' : 'Free to make. Smashwords only lets signed-in members download, even free books. The sign-in is kept encrypted on this server.'}${acct && acct.signedIn ? '' : ''}
+ ${acct && acct.signedIn ? '' : '
'}` : ''}
${sources.map(x => `
${esc(x.name)}${x.builtin ? 'Built in' : 'Added by you'}${x.builtin ? '' : ``}
`).join('')}
`; + if (el.querySelector('#rdswout')) el.querySelector('#rdswout').onclick = async () => { try { await rdApi('/store/smashwords', { method: 'DELETE' }); toast('Smashwords sign-in removed'); await list(); } catch (e) { toast(e.message); } }; + if (el.querySelector('#rdswf')) el.querySelector('#rdswf').onsubmit = async e => { e.preventDefault(); const err = el.querySelector('#rdswerr'), btn = e.submitter; err.textContent = ''; if (btn) { btn.disabled = true; btn.textContent = 'Signing in…'; } + try { await rdApi('/store/smashwords', rdJson('POST', { email: el.querySelector('#rdswe').value, pass: el.querySelector('#rdswp').value })); toast('Signed in to Smashwords'); await list(); } catch (x) { err.textContent = x.message; if (btn) { btn.disabled = false; btn.textContent = 'Sign in to Smashwords'; } } }; el.querySelectorAll('[data-d]').forEach(b => b.onclick = async () => { try { await rdApi('/store/sources/' + b.dataset.d, { method: 'DELETE' }); if (S.source === b.dataset.d) S.source = 'all'; await list(); } catch (e) { toast(e.message); } }); el.querySelector('#rdcf').onsubmit = async e => { e.preventDefault(); const err = el.querySelector('#rdce'), btn = e.submitter; err.textContent = ''; if (btn) btn.disabled = true; try { await rdApi('/store/sources', rdJson('POST', { name: el.querySelector('#rdcn').value, url: el.querySelector('#rdcu').value, user: el.querySelector('#rdcuser').value, pass: el.querySelector('#rdcp').value })); toast('Catalogue added'); await list(); } catch (x) { err.textContent = x.message; if (btn) btn.disabled = false; } }; diff --git a/public/store-view.js b/public/store-view.js index edb56a3..fa480b3 100644 --- a/public/store-view.js +++ b/public/store-view.js @@ -219,3 +219,15 @@ async function stRunSite(el, id) {

If the page stays blank, that site does not allow being shown here. Use New tab.

`; } + +/** On Home, under the app tiles: the extensions this person installed, and the Sites that were added to My Store, one tap away. */ +async function stHomeExt(el) { + if (!el) return; + try { await stLoad(); } catch { return; } + if (!el.isConnected) return; + const mine = stItems.filter(i => (i.kind === 'extension' && stInstalled.includes(i.id)) || i.kind === 'link'); + if (!mine.length) { el.innerHTML = ''; return; } + const tileOf = i => { const href = i.kind === 'extension' ? '#/store/ext/' + i.id : i.embed ? '#/store/site/' + i.id : i.url, ext = i.kind === 'link' && !i.embed; + return `${esc(i.icon || (i.title.match(/\p{L}|\p{N}/u) || ['?'])[0].toUpperCase())}${esc(i.title)}${i.kind === 'extension' ? 'Extension' : 'Site'}`; }; + el.innerHTML = `

My apps

${mine.map(tileOf).join('')}
`; +} diff --git a/public/sw.js b/public/sw.js index f3065d9..269e7c2 100644 --- a/public/sw.js +++ b/public/sw.js @@ -1,5 +1,5 @@ // App-shell cache only; Seafile/API traffic always goes to the network. -const CACHE = 'hub-v54'; +const CACHE = 'hub-v55'; const SHARE = 'hub-share'; // files shared into the app from other apps, held until the person picks a folder const SHELL = ['/icon.svg', '/manifest.webmanifest', '/icons/icon-192.png', '/icons/apple-touch-icon.png']; diff --git a/smashwords.js b/smashwords.js new file mode 100644 index 0000000..9039759 --- /dev/null +++ b/smashwords.js @@ -0,0 +1,139 @@ +// Smashwords as a book source for the Store: the free, DRM-free EPUBs. Browsing uses Smashwords' own shelf feed (no sign-in needed) and every book is run through a +// strict family filter first: anything Smashwords flags as adult or erotica, anything filed under LGBTQ+, and anything whose categories, tags or description say so, is +// dropped. Downloading a book needs a (free) Smashwords account, so an admin saves its sign-in once; it is kept encrypted on the server and used only to sign in at +// smashwords.com. Smashwords has no public search, so searching looks through a remembered set of its popular free books. +const fs = require('fs'), path = require('path'), crypto = require('crypto'); +const atomicWrite = require('./atomic'); +const auth = require('./auth'); + +const BASE = () => (process.env.SMASHWORDS_URL || 'https://www.smashwords.com').replace(/\/+$/, ''); +const COVERS = () => (process.env.SMASHWORDS_COVERS || 'https://dwtr67e3ikfml.cloudfront.net').replace(/\/+$/, ''); +const LANG = () => process.env.SMASHWORDS_LANG || '1'; // 1 = English +const UA = 'Mozilla/5.0 (compatible; NE-Hub/1.0; self-hosted e-book reader)'; +const DATA_DIR = process.env.DATA_DIR || path.join(__dirname, 'data'), FILE = () => path.join(DATA_DIR, 'store-smashwords.json'); +const MAX_BOOK = 300 * 1024 * 1024; +const fail = (code, message) => Object.assign(new Error(message), { code }); + +// ---- the family filter +const LGBTQ = '886'; // Smashwords' own category for it +const BAD_WORDS = '(?:erotic\\w*|erotica|bdsm|smut\\w*|lgbt\\w*|queer|lesbian|bisexual|transgender|non-?binary|same-sex|m/m|f/f|m/f/m|sex scenes?|sexually explicit|explicit (?:sex|content|scenes?)|adult (?:content|themes?|romance|fiction)|mature (?:content|audiences?|themes?)|steamy|spicy romance|harem|omegaverse|gay (?:romance|fiction|love|couple|men|man|male|teen|characters?|protagonist)|18\\+|nsfw)'; +const TEXT_BAD = new RegExp('(? String(s || '').replace(/<[^>]+>/g, ' ').replace(/ /g, ' ').replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"').replace(/�?39;/g, "'").replace(/\s+/g, ' ').trim(); +/** true when a book (Smashwords' own record for it) is fine to show. */ +function keep(b) { + if (!b || b.adult || b.erotica || b.taboo_erotica || b.need_erotic_cert) return false; + for (const c of b.categories || []) { + if (String(c.id) === LGBTQ || CAT_BAD.test(c.name || '')) return false; + for (const p of c.parents || []) if (String(p.id) === LGBTQ || CAT_BAD.test(p.name || '')) return false; + } + if ((b.tags || []).some(t => TAG_BAD.test(String(t)))) return false; + // Romance is where adult content hides without being flagged, so a romance is only shown when it says it is clean (sweet, Christian, inspirational...) + const romance = (b.categories || []).some(c => String(c.id) === ROMANCE || /romance/i.test(c.name || '') || (c.parents || []).some(p => String(p.id) === ROMANCE || /romance/i.test(p.name || ''))) || (b.tags || []).some(t => /romance/i.test(String(t))); + if (romance && !CLEAN.test([...(b.tags || []), ...(b.categories || []).flatMap(c => [c.name, ...(c.parents || []).map(p => p.name)])].join(' | '))) return false; + if (TEXT_BAD.test(plain(b.title) + ' ' + plain(b.short_description) + ' ' + plain(b.long_description))) return false; + return true; +} +const authorOf = b => (b.contributors || []).filter(c => !c.role || /^A0?1$/.test(c.role)).map(c => c.account && c.account.display_name).filter(Boolean).slice(0, 3).join(', '); +const free = b => !b.price || (b.price.prices || []).every(p => +p.amount === 0) || b.price.type === 'FREE'; +const hasEpub = b => !b.formats || b.formats.some(f => /^epub$/i.test(f)); +const item = b => ({ ref: String(b.id), format: 'epub', title: plain(b.title), author: authorOf(b), summary: plain(b.short_description).slice(0, 600), cover: b.cover_url ? COVERS() + '/' + String(b.cover_url).replace(/^\/+/, '') : '', _t: [b.title, authorOf(b), (b.tags || []).join(' '), plain(b.short_description)].join(' ').toLowerCase() }); +const clean = ({ _t, ...rest }) => rest; + +// ---- Smashwords' shelf feed +async function getJson(url) { + let r; try { r = await fetch(url, { headers: { 'user-agent': UA, accept: 'application/json' }, signal: AbortSignal.timeout(20000) }); } catch (e) { throw fail(502, e.name === 'TimeoutError' ? 'Smashwords did not answer in time' : 'Could not reach Smashwords'); } + if (!r.ok) throw fail(502, 'Smashwords answered ' + r.status); + try { return await r.json(); } catch { throw fail(502, 'Smashwords sent something unexpected'); } +} +const shelf = (type, cat, page) => getJson(`${BASE()}/api/bookshelves/category/v3/shelf/${type}/${cat}/free/any/no_erotica/${LANG()}/${page}/10`); +const PER_API_PAGE = 10, API_PAGES_PER_PAGE = 3; // a page of the Store is three of Smashwords' pages of ten, since the filter removes some + +/** The most downloaded free books, a page at a time. */ +async function browse(page) { + const first = (page - 1) * API_PAGES_PER_PAGE + 1, pages = await Promise.all([0, 1, 2].map(i => shelf('mostdownloads', 1, first + i).catch(e => { if (i === 0) throw e; return { book_list: [], total_books: 0 }; }))); + const seen = new Set(), items = []; + for (const p of pages) for (const e of p.book_list || []) { const b = e.book; if (b && !seen.has(b.id) && free(b) && hasEpub(b) && keep(b)) { seen.add(b.id); items.push(clean(item(b))); } } + const total = pages[0].total_books || 0; + return { items, more: first + API_PAGES_PER_PAGE - 1 < Math.ceil(total / PER_API_PAGE) && (pages[2].book_list || []).length > 0 }; +} + +// Smashwords has no public search, so one is made from its three best free shelves (most downloaded, newest, highest rated), remembered for a few hours. +let poolP = null, poolAt = 0, poolDone = false; +async function buildPool() { + const seen = new Set(), out = []; + for (const type of ['mostdownloads', 'highlyrated', 'newest']) { + const first = await shelf(type, 1, 1), n = Math.min(16, Math.ceil((first.total_books || 10) / PER_API_PAGE)), pages = [first]; + for (let i = 2; i <= n; i += 6) pages.push(...await Promise.all(Array.from({ length: Math.min(6, n - i + 1) }, (_, k) => shelf(type, 1, i + k).catch(() => ({ book_list: [] }))))); + for (const p of pages) for (const e of p.book_list || []) { const b = e.book; if (b && !seen.has(b.id) && free(b) && hasEpub(b) && keep(b)) { seen.add(b.id); out.push(item(b)); } } + } + return out; +} +const pool = () => { if (!poolP || Date.now() - poolAt > 6 * 3600e3) { poolAt = Date.now(); poolDone = false; poolP = buildPool().then(l => { poolDone = true; return l; }, e => { poolP = null; throw e; }); } return poolP; }; +async function search(q, page, wait = true) { // (wait: false is for a search across every source, which must not stall for the first-time build; it starts and the next search has it) + const words = q.toLowerCase().split(/\s+/).filter(Boolean); + if (!words.length) return browse(page); + if (!wait && !poolDone) { pool().catch(() => {}); return { items: [], more: false }; } + const hits = (await pool()).filter(b => words.every(w => b._t.includes(w))); + return { items: hits.slice((page - 1) * 20, page * 20).map(clean), more: hits.length > page * 20 }; +} + +// ---- the sign-in (admin saves it once; encrypted with a key from the server secret) +let _key; const key = () => _key || (_key = crypto.scryptSync(auth.secret, 'nehub-smashwords', 32)); +const seal = v => { const iv = crypto.randomBytes(12), c = crypto.createCipheriv('aes-256-gcm', key(), iv), d = Buffer.concat([c.update(JSON.stringify(v), 'utf8'), c.final()]); return Buffer.concat([iv, c.getAuthTag(), d]).toString('base64'); }; +const unseal = s => { try { const b = Buffer.from(s, 'base64'), d = crypto.createDecipheriv('aes-256-gcm', key(), b.subarray(0, 12)); d.setAuthTag(b.subarray(12, 28)); return JSON.parse(Buffer.concat([d.update(b.subarray(28)), d.final()]).toString('utf8')); } catch { return null; } }; +const saved = () => { try { return unseal(JSON.parse(fs.readFileSync(FILE(), 'utf8')).data); } catch { return null; } }; +const account = () => { const s = saved(); return { signedIn: !!s, email: s ? s.email : '' }; }; +let session = null; // { cookie, t } +function signOut() { try { fs.unlinkSync(FILE()); } catch {} session = null; } + +/** Cookies from a response ("name=value" pairs), merged into the jar. */ +function eat(jar, r) { for (const c of (r.headers.getSetCookie ? r.headers.getSetCookie() : [])) { const [kv] = c.split(';'), i = kv.indexOf('='); if (i > 0) { const v = kv.slice(i + 1).trim(); if (v && v !== 'deleted') jar.set(kv.slice(0, i).trim(), v); else jar.delete(kv.slice(0, i).trim()); } } } +const header = jar => [...jar].map(([k, v]) => k + '=' + v).join('; '); +async function req(url, jar, init = {}) { // one request; redirects are followed by hand so the cookies from each step are kept, and only smashwords' own address is used + for (let hop = 0; hop < 6; hop++) { + const u = new URL(url); if (u.origin !== new URL(BASE()).origin) throw fail(502, 'That address is not part of Smashwords'); + const r = await fetch(u, { ...init, headers: { 'user-agent': UA, ...(init.headers || {}), cookie: header(jar) }, redirect: 'manual', signal: AbortSignal.timeout(init.timeout || 30000) }); eat(jar, r); + if ([301, 302, 303, 307, 308].includes(r.status) && r.headers.get('location')) { url = new URL(r.headers.get('location'), u).href; if (r.status !== 307 && r.status !== 308) { const { 'content-type': _c, ...hs } = init.headers || {}; init = { ...init, method: 'GET', body: undefined, headers: hs }; } try { await r.body?.cancel(); } catch {} if (/\/(signin|signup\/login)\b/.test(url) && init.stopAtLogin) return { r: { status: 401, headers: new Headers(), body: null }, url }; continue; } + return { r, url }; + } + throw fail(502, 'Too many redirects from Smashwords'); +} +async function login(email, pass) { + const jar = new Map(), page = await req(BASE() + '/signin', jar), html = await page.r.text(), token = (/name="loginToken"\s+value="([^"]*)"/.exec(html) || [])[1]; + if (!token) throw fail(502, 'Smashwords\' sign-in page was not what was expected'); + const out = await req(BASE() + '/auth/index', jar, { method: 'POST', headers: { 'content-type': 'application/x-www-form-urlencoded', referer: BASE() + '/signin' }, body: new URLSearchParams({ email, password: pass, loginToken: token, secToken: '' }).toString() }); + const home = await (await req(BASE() + '/', jar)).r.text(); // the front page says whether this cookie is a signed-in one + if (!/loggedIn\s*=\s*true/.test(home)) { try { await out.r.body?.cancel(); } catch {} throw fail(401, 'Smashwords did not accept that email and password'); } + return jar; +} +/** Check a sign-in and save it. */ +async function signIn(email, pass) { + email = String(email || '').trim(); pass = String(pass || ''); if (!/^\S+@\S+\.\S+$/.test(email) || !pass) throw fail(400, 'Enter your Smashwords email and password'); + const jar = await login(email, pass); session = { jar, t: Date.now() }; + fs.mkdirSync(DATA_DIR, { recursive: true }); atomicWrite(FILE(), JSON.stringify({ data: seal({ email, pass }) }), { mode: 0o600 }); return account(); +} +async function jar() { + if (session && Date.now() - session.t < 3 * 3600e3) return session.jar; + const s = saved(); if (!s) throw fail(401, 'An admin needs to add a Smashwords sign-in first (Store, Books, Catalogues)'); + session = { jar: await login(s.email, s.pass), t: Date.now() }; return session.jar; +} +/** The EPUB of a free book, as the signed-in account. */ +async function download(id) { + if (!/^\d{1,9}$/.test(id)) throw fail(400, 'Bad book'); + for (let attempt = 0; attempt < 2; attempt++) { + const j = await jar(), { r } = await req(`${BASE()}/books/download/${id}/8/latest/0/0/book.epub`, j, { timeout: 120000, stopAtLogin: true, headers: { accept: 'application/epub+zip, application/octet-stream, */*;q=0.5' } }); + if (r.status === 401) { session = null; continue; } // the saved cookie ran out: sign in again once + if (!r.ok) throw fail(r.status === 404 ? 404 : 502, 'Smashwords answered ' + r.status); + if ((+r.headers.get('content-length') || 0) > MAX_BOOK) throw fail(413, 'That file is too large'); + const chunks = []; let n = 0; for await (const c of r.body) { n += c.length; if (n > MAX_BOOK) throw fail(413, 'That file is too large'); chunks.push(c); } + const buf = Buffer.concat(chunks); if (buf.subarray(0, 2).toString('latin1') !== 'PK') throw fail(502, 'Smashwords did not send an EPUB for that book (it may not be free, or may not have one)'); + return buf; + } + throw fail(401, 'Smashwords would not sign in with the saved email and password'); +} + +module.exports = { keep, browse, search, account, signIn, signOut, download, item, COVERS }; diff --git a/store.js b/store.js index 61cc98f..ab7c85c 100644 --- a/store.js +++ b/store.js @@ -2,7 +2,7 @@ // admin adds (Calibre-Web, Kavita, ...). The hub server does all the talking to those sites: searching, covers and downloading a chosen book straight onto the // signed-in person's shelf (books.js), so the browser never contacts them and no catalogue sign-in reaches the browser. const fs = require('fs'), path = require('path'), crypto = require('crypto'); -const users = require('./users'), auth = require('./auth'), books = require('./books'); +const users = require('./users'), auth = require('./auth'), books = require('./books'), smashwords = require('./smashwords'); const UA = 'NE-Hub/1.0 (self-hosted e-book reader)'; const MAX_BOOK = 300 * 1024 * 1024, MAX_COVER = 3 * 1024 * 1024, MAX_TEXT = 4 * 1024 * 1024; @@ -11,6 +11,7 @@ const BUILTIN = { gutenberg: { name: 'Project Gutenberg', base: () => env('GUTENBERG_URL', 'https://www.gutenberg.org'), hosts: ['gutenberg.org'] }, // its own catalogue feed (fast and official) standardebooks: { name: 'Standard Ebooks', base: () => env('STANDARD_EBOOKS_URL', 'https://standardebooks.org'), hosts: ['standardebooks.org'] }, // its public pages: the OPDS feed is for paying members only archive: { name: 'Internet Archive', base: () => env('ARCHIVE_URL', 'https://archive.org'), hosts: ['archive.org'], needsQuery: true }, + smashwords: { name: 'Smashwords', base: () => env('SMASHWORDS_URL', 'https://www.smashwords.com'), hosts: ['smashwords.com', 'dwtr67e3ikfml.cloudfront.net'] }, // free EPUBs, with a family filter; downloading needs an account (see smashwords.js) openlibrary: { name: 'Open Library', base: () => env('OPENLIBRARY_URL', 'https://openlibrary.org'), hosts: ['openlibrary.org'], needsQuery: true }, // search here; the book itself comes from the Internet Archive }; const FORMAT_OF = { 'application/epub+zip': 'epub', 'application/pdf': 'pdf', 'application/x-mobipocket-ebook': 'mobi', 'application/vnd.amazon.mobi8-ebook': 'azw3', 'application/vnd.amazon.ebook': 'azw' }; @@ -144,15 +145,16 @@ async function openLibrarySearch(src, q, page) { return { items, more: page * 20 < (j.numFound || 0) }; } -async function searchOne(src, q, page) { +async function searchOne(src, q, page, everywhere) { if (src.id === 'gutenberg') return gutenbergSearch(src, q, page); if (src.id === 'standardebooks') return standardSearch(src, q, page); if (src.id === 'archive') return q ? archiveSearch(src, q, page) : { items: [], more: false }; + if (src.id === 'smashwords') return smashwords.search(q, page, !everywhere); if (src.id === 'openlibrary') return q ? openLibrarySearch(src, q, page) : { items: [], more: false }; return opdsSearch(src, q, page); } const stamp = (src, it) => ({ ...it, source: src.id, sourceName: src.name, cover: it.cover ? `/api/read/store/cover?s=${encodeURIComponent(src.id)}&u=${encodeURIComponent(it.cover)}` : '', coverUrl: it.cover || '' }); -const listSources = () => [...Object.keys(BUILTIN), ...customs().map(c => c.id)].map(id => { const s = sourceOf(id); return { id, name: s.name, builtin: s.builtin, needsQuery: !!s.needsQuery }; }); +const listSources = () => [...Object.keys(BUILTIN), ...customs().map(c => c.id)].map(id => { const s = sourceOf(id); return { id, name: s.name, builtin: s.builtin, needsQuery: !!s.needsQuery, ...(id === 'smashwords' ? { account: smashwords.account() } : {}) }; }); async function handle(req, res, p) { const me = users.current(); @@ -168,6 +170,11 @@ async function handle(req, res, p) { catch (e) { saveCustoms(customs().filter(x => x.id !== id)); throw fail(e.code || 502, e.code ? e.message : 'Could not read that catalogue: ' + e.message); } return json(res, 200, { id, name }); } + if (p === '/smashwords' && req.method === 'POST') { // the Smashwords sign-in (admin): checked by signing in, then kept encrypted + if (!me || me.role !== 'admin') throw fail(403, 'Only an admin can add the Smashwords sign-in'); + const b = await readBody(req); return json(res, 200, await smashwords.signIn(b.email, b.pass)); + } + if (p === '/smashwords' && req.method === 'DELETE') { if (!me || me.role !== 'admin') throw fail(403, 'Only an admin can remove the Smashwords sign-in'); smashwords.signOut(); return json(res, 200, smashwords.account()); } let m; if ((m = /^\/sources\/(\w+)$/.exec(p)) && req.method === 'DELETE') { if (!me || me.role !== 'admin') throw fail(403, 'Only an admin can remove a catalogue'); @@ -176,7 +183,7 @@ async function handle(req, res, p) { if (p === '/search' && req.method === 'GET') { const u = new URL(req.url, 'http://x'), q = (u.searchParams.get('q') || '').trim().slice(0, 120), page = Math.min(20, Math.max(1, +u.searchParams.get('page') || 1)), want = u.searchParams.get('source') || 'all'; const ids = want === 'all' ? listSources().map(s => s.id) : [want], lists = [], errors = {}; let more = false; - await Promise.all(ids.map(async id => { let src; try { src = sourceOf(id); const r = await searchOne(src, q, page); lists.push([id, r.items.map(it => stamp(src, it))]); more = more || r.more; } catch (e) { const nm = src ? src.name : id, msg = e.name === 'TimeoutError' ? 'did not answer in time' : e.message; errors[id] = msg.startsWith(nm) ? msg : nm + ': ' + msg; } })); + await Promise.all(ids.map(async id => { let src; try { src = sourceOf(id); const r = await searchOne(src, q, page, want === 'all'); lists.push([id, r.items.map(it => stamp(src, it))]); more = more || r.more; } catch (e) { const nm = src ? src.name : id, msg = e.name === 'TimeoutError' ? 'did not answer in time' : e.message; errors[id] = msg.startsWith(nm) ? msg : nm + ': ' + msg; } })); lists.sort((a, b) => ids.indexOf(a[0]) - ids.indexOf(b[0])); const items = []; for (let i = 0; lists.some(l => l[1][i]); i++) for (const l of lists) if (l[1][i]) items.push(l[1][i]); // one from each catalogue in turn return json(res, 200, { items, errors, more }); @@ -192,10 +199,11 @@ async function handle(req, res, p) { if (src.id === 'gutenberg') target = await gutenbergResolve(src, ref); else if (src.id === 'standardebooks') target = await standardResolve(src, ref); else if (src.id === 'archive') target = await archiveResolve(src, ref); + else if (src.id === 'smashwords') target = { format: 'epub' }; else if (src.id === 'openlibrary') { dl = sourceOf('archive'); target = await archiveResolve(dl, ref); } // (Open Library's free editions are Internet Archive items) else { if (!sameSite(src, new URL(ref, src.base))) throw fail(400, 'Bad book address'); target = { url: ref, format: String(b.format || 'epub') }; } const ext = books.TYPES[target.format] ? target.format : 'epub'; - const buf = await limited(await get(dl, target.url, { accept: 'application/epub+zip, application/octet-stream, */*;q=0.5', timeout: 120000 }), MAX_BOOK); + const buf = src.id === 'smashwords' ? await smashwords.download(ref) : await limited(await get(dl, target.url, { accept: 'application/epub+zip, application/octet-stream, */*;q=0.5', timeout: 120000 }), MAX_BOOK); let cover; const cu = String(b.cover || ''); if (cu) { try { const r = await get(src, cu, { accept: 'image/*' }), type = (r.headers.get('content-type') || '').split(';')[0].trim().toLowerCase(); cover = { type, data: await limited(r, MAX_COVER) }; } catch {} } let book; try { book = books.addBook(buf, ext, { title: String(b.title || '').slice(0, 200), author: String(b.author || '').slice(0, 200) }, cover); } catch (e) { throw fail(400, e.message); } return json(res, 200, book); diff --git a/tools/librivox-test.js b/tools/librivox-test.js index c662fcd..3a0c20a 100644 --- a/tools/librivox-test.js +++ b/tools/librivox-test.js @@ -12,6 +12,8 @@ const fake = http.createServer((req, res) => { if (u.searchParams.get('title') === 'prejudice' || u.searchParams.get('author') === 'austen') return J({ books: [book] }); return J({ error: 'Audiobooks could not be found' }, 404); } + if (p === '/dl/1.mp3') { const rg = /^bytes=(\d+)-(\d+)$/.exec(req.headers.range || ''), data = Buffer.from('0123456789'); if (rg) { res.writeHead(206, { 'content-type': 'audio/mpeg', 'content-range': `bytes ${rg[1]}-${rg[2]}/10`, 'content-length': +rg[2] - +rg[1] + 1 }); return res.end(data.subarray(+rg[1], +rg[2] + 1)); } res.writeHead(200, { 'content-type': 'audio/mpeg', 'content-length': 10 }); return res.end(data); } + if (p === '/dl/moved.mp3') { res.writeHead(302, { location: '/dl/1.mp3' }); return res.end(); } if (p === '/cover.gif') { res.writeHead(200, { 'content-type': 'image/gif' }); return res.end(gif); } res.writeHead(404); res.end(); }).listen(0, () => { @@ -25,7 +27,8 @@ const fake = http.createServer((req, res) => { assert.deepEqual((await j(await call('/search?q=zzz'))).items, [], 'nothing found is an empty list, not an error'); assert.ok((await j(await call('/search'))).items.length >= 0, 'no search words lists the catalogue'); r = await call('/book/55'); const b = await j(r); assert.equal(b.chapters.length, 2, 'chapters on other sites are dropped'); assert.ok(!JSON.stringify(b).includes('127.0.0.1:' + port + '/dl'), 'audio addresses are not sent to the browser'); assert.equal(b.chapters[1].title, 'Chapter 3'); assert.equal(b.description, 'A novel'); - r = await call('/audio/55/0'); assert.equal(r.status, 302); assert.equal(r.headers.get('location'), `http://127.0.0.1:${port}/dl/1.mp3`); + r = await call('/audio/55/0'); assert.equal(r.status, 200); assert.equal(r.headers.get('content-type'), 'audio/mpeg'); assert.equal(await r.text(), '0123456789', 'the hub passes the recording on itself'); + r = await call('/audio/55/0', { headers: { range: 'bytes=2-4' } }); assert.equal(r.status, 206); assert.equal(await r.text(), '234'); assert.equal(r.headers.get('content-range'), 'bytes 2-4/10', 'seeking works'); assert.equal((await call('/audio/55/9')).status, 404); assert.equal((await call('/book/abc')).status, 404); r = await call('/cover/55'); assert.equal(r.headers.get('content-type'), 'image/gif'); assert.equal((await call('/cover/55?u=' + encodeURIComponent('http://169.254.169.254/x'))).status, 200, 'an address off the allowed sites is ignored (the book\'s own cover is used)'); diff --git a/tools/store-test.js b/tools/store-test.js index 245211c..cee0c26 100644 --- a/tools/store-test.js +++ b/tools/store-test.js @@ -3,7 +3,7 @@ const http = require('http'), assert = require('assert'), fs = require('fs'), os process.env.DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'nehub-store-')); const fx = fs.mkdtempSync(path.join(os.tmpdir(), 'nehub-storefx-')); cp.execFileSync('python3', [path.join(__dirname, 'mkbooks.py'), fx]); const epub = fs.readFileSync(path.join(fx, 'test.epub')), pdf = fs.readFileSync(path.join(fx, 'test.pdf')), gif = Buffer.from('R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==', 'base64'); -const seen = []; let port; +const seen = [], swPosts = []; let port; const feed = (body, links = '') => `${links}${body}`; const acq = (id, title, author, href, type = 'application/epub+zip') => `${title}${author}${id}A & B`; const seCard = (slug, title, who) => `

  • ${title}

    ${who}

    • `; @@ -28,6 +28,17 @@ const fake = http.createServer((req, res) => { if (p === '/ia/download/moby1/moby1.epub') { res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(Buffer.concat([epub, Buffer.from('ia')])); } // Open Library (search only; the book is an Internet Archive item) if (p === '/ol/search.json') return J({ numFound: 2, docs: [{ title: 'Moby Dick', author_name: ['Melville'], cover_i: 7, ia: ['moby1'], ebook_access: 'public' }, { title: 'Borrowable', ia: ['locked'], ebook_access: 'borrowable' }] }); + // Smashwords: its shelf feed (filtered), a sign-in form, and downloads that need the signed-in cookie + if (p === '/sw/signin') { res.writeHead(200, { 'content-type': 'text/html', 'set-cookie': 'XSRF-TOKEN=deleted; Path=/' }); return res.end('
    '); } + if (p === '/sw/auth/index' && req.method === 'POST') { let b = ''; req.on('data', c => b += c); req.on('end', () => { const f = new URLSearchParams(b); swPosts.push(Object.fromEntries(f)); if (f.get('email') === 'me@x.org' && f.get('password') === 'pw' && f.get('loginToken') === 'tok123') { res.writeHead(302, { location: '/sw/', 'set-cookie': 'sess=abc; Path=/; HttpOnly' }); } else res.writeHead(200); res.end(); }); return; } + if (p === '/sw/') { res.writeHead(200, { 'content-type': 'text/html' }); return res.end(``); } + if (p === '/sw/books/download/101/8/latest/0/0/book.epub') { if (!/sess=abc/.test(req.headers.cookie || '')) { res.writeHead(302, { location: '/sw/signup/login/books/101' }); return res.end(); } res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(epub); } + if (p === '/sw/signup/login/books/101') { res.writeHead(200); return res.end('login'); } + if (p.startsWith('/sw/api/bookshelves/category/v3/shelf/')) { const pg = +p.split('/').slice(-2)[0]; // (.../{page}/{pageSize}) + const bk = (id, o = {}) => ({ book: { id: String(id), title: 'Book ' + id, price: { type: 'FIXED_PRICE', prices: [{ amount: '0.00' }] }, short_description: 'A fine tale.', cover_url: 'bookCovers/c' + id + '.gif', adult: false, erotica: false, taboo_erotica: false, need_erotic_cert: false, categories: [{ id: '1', name: 'Fiction', parents: [] }], tags: ['adventure'], formats: ['EPUB'], contributors: [{ role: 'A01', account: { display_name: 'Ann Author' } }], ...o } }); + const list = pg === 1 ? [bk(101, { title: 'Pilgrim Story' }), bk(102, { categories: [{ id: '5', name: 'Romance', parents: [{ id: '886', name: 'LGBTQ+' }] }] }), bk(103, { erotica: true }), bk(104, { tags: ['gay romance'] }), bk(105, { title: 'Gaylord\'s Garden', contributors: [{ role: 'A01', account: { display_name: 'Gay Talese' } }] }), bk(106, { short_description: 'A steamy love story' }), bk(107, { price: { type: 'FIXED_PRICE', prices: [{ amount: '2.99' }] } }), bk(109, { categories: [{ id: '1235', name: 'Romance', parents: [{ id: '3', name: 'Fiction' }] }], tags: ['billionaire romance'] }), bk(110, { title: 'Sweet Home', categories: [{ id: '1235', name: 'Romance', parents: [{ id: '3', name: 'Fiction' }] }], tags: ['sweet romance'] }), bk(108, { categories: [{ id: '886', name: 'LGBTQ+', parents: [{ id: '3', name: 'Fiction' }] }] })] : []; + res.writeHead(200, { 'content-type': 'application/json' }); return res.end(JSON.stringify({ name: 'Most Downloaded', type: 'mostdownloads', book_list: list, total_books: 8 })); } + if (p.startsWith('/swc/')) { res.writeHead(200, { 'content-type': 'image/gif' }); return res.end(gif); } // an OPDS catalogue that needs a sign-in (Calibre-Web style) if (p.startsWith('/cal/')) { if (req.headers.authorization !== 'Basic ' + Buffer.from('reader:pw').toString('base64')) { res.writeHead(401); return res.end(); } @@ -39,13 +50,13 @@ const fake = http.createServer((req, res) => { res.writeHead(404); res.end('no'); }).listen(0, async () => { port = fake.address().port; const B = 'http://127.0.0.1:' + port; - Object.assign(process.env, { GUTENBERG_URL: B, ARCHIVE_URL: B + '/ia', OPENLIBRARY_URL: B + '/ol', STANDARD_EBOOKS_URL: B }); + Object.assign(process.env, { GUTENBERG_URL: B, ARCHIVE_URL: B + '/ia', OPENLIBRARY_URL: B + '/ol', SMASHWORDS_URL: B + '/sw', SMASHWORDS_COVERS: B + '/swc', STANDARD_EBOOKS_URL: B }); const books = require('../books'), users = require('../users'); const people = { a: users.create({ name: 'Ann', role: 'admin', handle: 'YQ' }), b: users.create({ name: 'Bob', role: 'user', tabs: ['read'], handle: 'Yg' }) }; const hub = http.createServer((req, res) => users.runAs(req.headers['x-user'] === 'b' ? people.b : people.a, () => books.handle(req, res))).listen(0, async () => { const base = 'http://127.0.0.1:' + hub.address().port + '/api/read/store', call = (p, o = {}) => fetch(base + p, o), post = (p, b, h = {}) => call(p, { method: 'POST', body: JSON.stringify(b), headers: h }); try { - let r = await call('/sources'); assert.deepEqual((await r.json()).map(s => s.id), ['gutenberg', 'standardebooks', 'archive', 'openlibrary']); + let r = await call('/sources'); assert.deepEqual((await r.json()).map(s => s.id), ['gutenberg', 'standardebooks', 'archive', 'smashwords', 'openlibrary']); r = await call('/search?q=alice'); let j = await r.json(); assert.deepEqual(j.items.map(i => i.source), ['gutenberg', 'standardebooks', 'archive', 'openlibrary', 'gutenberg', 'standardebooks'].slice(0, j.items.length), 'one result from each catalogue in turn'); assert.deepEqual(j.errors, {}); const g = j.items.filter(i => i.source === 'gutenberg'); assert.deepEqual(g.map(i => i.ref), ['11', '345'], 'catalogue entries that are not books, and repeats, are left out'); assert.equal(g[0].author, 'Lewis Carroll'); assert.equal(g[1].title, 'Dracula & Co'); assert.match(g[0].cover, /^\/api\/read\/store\/cover\?s=gutenberg/); assert.equal(j.more, true); @@ -67,12 +78,24 @@ const fake = http.createServer((req, res) => { r = await post('/add', { source: 'archive', ref: 'moby1', title: 'Moby', author: 'Melville' }); assert.equal((await r.json()).title, 'Moby'); assert.ok(seen.includes('/ia/download/moby1/moby1.epub'), 'the EPUB is chosen over the PDF'); const ol = (await (await call('/search?q=moby&source=openlibrary')).json()).items; assert.deepEqual(ol.map(i => i.ref), ['moby1'], 'only freely readable editions are listed'); assert.equal(ol[0].author, 'Melville'); assert.match(decodeURIComponent(ol[0].coverUrl), /covers\.openlibrary\.org\/b\/id\/7-M\.jpg/); r = await post('/add', { source: 'openlibrary', ref: 'moby1', title: 'Moby Dick' }); assert.equal(r.status, 200, 'an Open Library book is fetched from the Internet Archive'); + // Smashwords: the family filter, the sign-in, and the download + r = await call('/search?q=&source=smashwords'); j = await r.json(); assert.deepEqual(j.items.map(i => i.ref), ['101', '105', '110'], 'adult, LGBTQ+, keyword-flagged, unlabelled romance and paid books are left out; a clean romance and a harmless name are kept'); assert.equal(j.items[0].author, 'Ann Author'); assert.match(j.items[0].cover, /^\/api\/read\/store\/cover\?s=smashwords/); + j = await (await call('/search?q=pilgrim&source=smashwords')).json(); assert.deepEqual(j.items.map(i => i.ref), ['101'], 'searching looks through the free books it can see'); assert.deepEqual((await (await call('/search?q=gay&source=smashwords')).json()).items.map(i => i.ref), ['105']); + j = await (await call('/sources')).json(); assert.deepEqual(j.find(s => s.id === 'smashwords').account, { signedIn: false, email: '' }); + r = await post('/add', { source: 'smashwords', ref: '101', title: 'Pilgrim Story' }); assert.equal(r.status, 401); assert.match((await r.json()).error, /sign-in/, 'downloading needs the account'); + assert.equal((await post('/smashwords', { email: 'me@x.org', pass: 'pw' }, { 'x-user': 'b' })).status, 403, 'only an admin saves the sign-in'); + r = await post('/smashwords', { email: 'me@x.org', pass: 'wrong' }); assert.equal(r.status, 401); assert.equal((await (await call('/sources')).json()).find(s => s.id === 'smashwords').account.signedIn, false, 'a wrong password is not kept'); + r = await post('/smashwords', { email: 'me@x.org', pass: 'pw' }); assert.equal(r.status, 200); assert.deepEqual(await r.json(), { signedIn: true, email: 'me@x.org' }); assert.equal(swPosts.at(-1).loginToken, 'tok123', 'the sign-in form\'s own token is sent back'); + assert.ok(!fs.readFileSync(path.join(process.env.DATA_DIR, 'store-smashwords.json'), 'utf8').includes('"pw"') && !JSON.stringify(await (await call('/sources')).json()).includes('"pw"'), 'the password is encrypted on disk and never sent to the browser'); + r = await post('/add', { source: 'smashwords', ref: '101', title: 'Pilgrim Story', author: 'Ann Author', cover: B + '/swc/c101.gif' }); assert.equal(r.status, 200, 'the signed-in download is accepted'); assert.ok(seen.includes('/sw/books/download/101/8/latest/0/0/book.epub'), 'the EPUB is fetched with the signed-in cookie'); + assert.equal((await post('/add', { source: 'smashwords', ref: '../x' })).status, 400); + assert.equal((await call('/smashwords', { method: 'DELETE' })).status, 200); assert.equal((await post('/add', { source: 'smashwords', ref: '101' })).status, 401, 'removing the sign-in stops downloads'); r = await post('/add', { source: 'archive', ref: 'locked' }); assert.equal(r.status, 403); assert.match((await r.json()).error, /DRM/); r = await post('/add', { source: 'gutenberg', ref: '../x' }); assert.equal(r.status, 400); // custom catalogue assert.equal((await post('/sources', { name: 'Mine', url: B + '/cal/opds' }, { 'x-user': 'b' })).status, 403, 'only an admin adds catalogues'); assert.equal((await post('/sources', { name: 'Bad', url: B + '/cal/opds', user: 'reader', pass: 'wrong' })).status, 502, 'a catalogue that refuses the sign-in is not added'); - assert.equal((await call('/sources').then(x => x.json())).length, 4); + assert.equal((await call('/sources').then(x => x.json())).length, 5); r = await post('/sources', { name: 'Mine', url: B + '/cal/opds', user: 'reader', pass: 'pw' }); const mine = await r.json(); assert.equal(r.status, 200); assert.ok(!JSON.stringify(await (await call('/sources')).json()).includes('pw'), 'sign-ins are never sent to the browser'); assert.ok(!fs.readFileSync(path.join(process.env.DATA_DIR, 'store-sources.json'), 'utf8').includes('"pw"'), 'and are encrypted on disk'); j = await (await call('/search?q=hello%20world&source=' + mine.id)).json(); assert.equal(j.items[0].title, 'Own Book hello world', 'search found through the OpenSearch description'); assert.equal(j.items[0].format, 'pdf');