diff --git a/README.md b/README.md
index 9d1d865..e6c4814 100644
--- a/README.md
+++ b/README.md
@@ -164,3 +164,7 @@ One **Store** tab (turn it on per person) with sections for **Books** (the free
Files are kept in the data folder under `mystore/` (up to 1 GB each; extensions up to 2 MB). Only an admin can add, change or delete items.
+**Smashwords** is a book source in the Store (Books). It lists Smashwords' most downloaded free EPUBs through its own shelf feed, filtered for a family: anything Smashwords flags as adult or erotica, anything in LGBTQ+, anything whose categories, tags or description say so, and any romance that does not say it is clean (sweet, Christian, inspirational...). The filter is automatic and cannot be perfect, so a book can occasionally slip through. Smashwords has no public search, so searching looks through a remembered set of about 400 of its popular free books. Downloading needs a free Smashwords account: an admin enters it once under Books, *Catalogues* (kept encrypted on the server, used only to sign in at smashwords.com). `SMASHWORDS_LANG` (default `1`, English) picks the language.
+
+**My apps on Home:** the extensions a person installed from My Store, and any Sites in My Store, appear under *My apps* on Home and in the command palette.
+
diff --git a/librivox.js b/librivox.js
index 0307d89..06a9620 100644
--- a/librivox.js
+++ b/librivox.js
@@ -45,9 +45,22 @@ async function handle(req, res) {
return json(res, 200, { items: items.map(b => ({ ...b, cover: b.cover ? `/api/ab/free/cover/${b.id}?u=${encodeURIComponent(b.cover)}` : '' })), more: lists.flat().length >= 20 });
}
if ((m = /^\/book\/(\d+)$/.exec(p)) && req.method === 'GET') return json(res, 200, publicBook(await book(m[1])));
- if ((m = /^\/audio\/(\d+)\/(\d+)$/.exec(p)) && req.method === 'GET') { // the recording itself is on the Internet Archive; the browser is sent straight there (it supports seeking)
+ if ((m = /^\/audio\/(\d+)\/(\d+)$/.exec(p)) && req.method === 'GET') { // the recording is on the Internet Archive; the hub fetches it (redirects and all) and passes it on, with seeking
const b = await book(m[1]), c = b.chapters[+m[2]]; if (!c) throw fail(404, 'No such chapter');
- res.writeHead(302, { location: c.url, 'cache-control': 'private, max-age=600', 'referrer-policy': 'no-referrer' }); return res.end();
+ const ctl = new AbortController(); res.on('close', () => ctl.abort());
+ let url = c.url, r;
+ for (let i = 0; i < 6; i++) {
+ if (!allowed(url)) throw fail(502, 'That audio address is not allowed');
+ r = await fetch(url, { headers: { 'user-agent': 'NE-Hub/1.0', 'accept-encoding': 'identity', ...(req.headers.range ? { range: req.headers.range } : {}) }, redirect: 'manual', signal: ctl.signal }).catch(e => { throw fail(502, e.name === 'AbortError' ? 'Cancelled' : 'The Internet Archive did not answer'); });
+ if ([301, 302, 303, 307, 308].includes(r.status) && r.headers.get('location')) { url = new URL(r.headers.get('location'), url).href; try { await r.body?.cancel(); } catch {} continue; }
+ break;
+ }
+ if (r.status === 416) { res.writeHead(416, { 'content-range': r.headers.get('content-range') || 'bytes */0' }); return res.end(); }
+ if (!r.ok && r.status !== 206) throw fail(502, 'The Internet Archive answered ' + r.status + ' for that chapter');
+ const out = { 'content-type': /^audio\//.test(r.headers.get('content-type') || '') ? r.headers.get('content-type') : 'audio/mpeg', 'accept-ranges': 'bytes', 'cache-control': 'private, max-age=600', 'referrer-policy': 'no-referrer' };
+ for (const k of ['content-length', 'content-range']) if (r.headers.get(k)) out[k] = r.headers.get(k);
+ res.writeHead(r.status, out); if (!r.body) return res.end();
+ return require('stream').Readable.fromWeb(r.body).on('error', () => res.destroy()).pipe(res);
}
if ((m = /^\/cover\/(\d+)$/.exec(p)) && req.method === 'GET') {
let cu = u.searchParams.get('u'); if (!cu || !allowed(cu)) { const b = await book(m[1]); cu = b.cover; } if (!cu || !allowed(cu)) throw fail(404, 'No cover');
diff --git a/public/app.js b/public/app.js
index fdce390..64cb571 100644
--- a/public/app.js
+++ b/public/app.js
@@ -353,7 +353,8 @@ function viewHome() {
${recents.length && allowed('files') ? `Recent files ${recents.map((r, i) => `${icon('file')} ${esc(r.name)} ${esc(r.path.slice(0, r.path.lastIndexOf('/')) || '/')}
` : ''}
${evening ? '' : '
'}
- ${homeApps()}
`;
+ ${homeApps()}
`;
+ if (typeof stHomeExt === 'function' && allowed('store')) stHomeExt($('#homeext'));
if (typeof homeResume === 'function') homeResume($('#resume'));
renderNpCard(); // the live Now Playing card (it follows the song and has the controls)
$('#view').querySelectorAll('button[data-r]').forEach(b => b.onclick = () => {
diff --git a/public/palette.js b/public/palette.js
index 199a0a3..f04776d 100644
--- a/public/palette.js
+++ b/public/palette.js
@@ -54,6 +54,7 @@ function openPalette() {
const out = navTabs().flatMap(t => MERGED[t[0]] ? MERGED[t[0]][1].filter(allowed).map(id => [id, tabLabel(id)]) : [t]).map(([id, label]) => ({ label: 'Go to ' + label, hint: (navGroups().find(g => g.tabs.some(t => t[0] === (leafOf(id) || id))) || {}).label || 'Page', ic: TAB_ICON[id], run: go('#/' + id), kw: label }));
if (phone.matches) out.push({ label: 'Edit the bottom bar', hint: 'View', ic: 'menu', run: () => openPinsSheet(), kw: 'menu tabs pins more' });
if (allowed('tasks') && allowed('calendar')) out.push({ label: 'Go to Tasks', hint: 'Planner', ic: 'check', run: go('#/tasks'), kw: 'todo' }, { label: 'Go to Calendar', hint: 'Planner', ic: 'cal', run: go('#/calendar'), kw: 'agenda events' });
+ if (typeof stItems !== 'undefined' && allowed('store')) for (const i of stItems) { if (i.kind === 'extension' && stInstalled.includes(i.id)) out.push({ label: 'Open ' + i.title, hint: 'Extension', ic: 'cog', run: go('#/store/ext/' + i.id), kw: 'app extension ' + i.title.toLowerCase() }); else if (i.kind === 'link') out.push({ label: 'Open ' + i.title, hint: 'Site', ic: 'link', run: i.embed ? go('#/store/site/' + i.id) : () => window.open(i.url, '_blank', 'noopener,noreferrer'), kw: 'app site ' + i.title.toLowerCase() }); }
if (allowed('tasks')) out.push({ label: 'New task', hint: 'Create', ic: 'plus', run: () => quickAdd(), kw: 'add todo' });
if (allowed('calendar')) out.push({ label: 'New event', hint: 'Create', ic: 'plus', run: () => newEvent(ymd(new Date())), kw: 'add calendar' });
if (allowed('mail') && cfg.mail) out.push({ label: 'New email', hint: 'Create', ic: 'mail', run: () => { location.hash = '#/mail'; setTimeout(() => mailCompose(), 300); }, kw: 'compose write message' });
diff --git a/public/reader.js b/public/reader.js
index 7ab1f28..a572e23 100644
--- a/public/reader.js
+++ b/public/reader.js
@@ -198,10 +198,14 @@ async function rdStore(host) { // host: { el, hash } when shown inside the Store
};
const catalogues = () => {
const el = openSheet('Catalogues', 'Anything that speaks OPDS: Calibre-Web, Kavita, Komga, Standard Ebooks and others', '
');
- const list = async () => { sources = await rdApi('/store/sources'); chips(); el.querySelector('#rdcl').innerHTML = `${sources.map(x => `
${esc(x.name)} ${x.builtin ? 'Built in' : 'Added by you'} Remove `}
`).join('')}
Smashwords account ${acct && acct.signedIn ? 'Signed in as ' + esc(acct.email) + '. Books are downloaded with this account.' : 'Free to make. Smashwords only lets signed-in members download, even free books. The sign-in is kept encrypted on this server.'} Remove ' : ''}
${sources.map(x => `
${esc(x.name)} ${x.builtin ? 'Built in' : 'Added by you'} Remove `}
`).join('')}
If the page stays blank, that site does not allow being shown here. Use New tab.
`;
}
+
+/** On Home, under the app tiles: the extensions this person installed, and the Sites that were added to My Store, one tap away. */
+async function stHomeExt(el) {
+ if (!el) return;
+ try { await stLoad(); } catch { return; }
+ if (!el.isConnected) return;
+ const mine = stItems.filter(i => (i.kind === 'extension' && stInstalled.includes(i.id)) || i.kind === 'link');
+ if (!mine.length) { el.innerHTML = ''; return; }
+ const tileOf = i => { const href = i.kind === 'extension' ? '#/store/ext/' + i.id : i.embed ? '#/store/site/' + i.id : i.url, ext = i.kind === 'link' && !i.embed;
+ return `${esc(i.icon || (i.title.match(/\p{L}|\p{N}/u) || ['?'])[0].toUpperCase())} ${esc(i.title)} ${i.kind === 'extension' ? 'Extension' : 'Site'} My apps ${mine.map(tileOf).join('')}
`;
+}
diff --git a/public/sw.js b/public/sw.js
index f3065d9..269e7c2 100644
--- a/public/sw.js
+++ b/public/sw.js
@@ -1,5 +1,5 @@
// App-shell cache only; Seafile/API traffic always goes to the network.
-const CACHE = 'hub-v54';
+const CACHE = 'hub-v55';
const SHARE = 'hub-share'; // files shared into the app from other apps, held until the person picks a folder
const SHELL = ['/icon.svg', '/manifest.webmanifest', '/icons/icon-192.png', '/icons/apple-touch-icon.png'];
diff --git a/smashwords.js b/smashwords.js
new file mode 100644
index 0000000..9039759
--- /dev/null
+++ b/smashwords.js
@@ -0,0 +1,139 @@
+// Smashwords as a book source for the Store: the free, DRM-free EPUBs. Browsing uses Smashwords' own shelf feed (no sign-in needed) and every book is run through a
+// strict family filter first: anything Smashwords flags as adult or erotica, anything filed under LGBTQ+, and anything whose categories, tags or description say so, is
+// dropped. Downloading a book needs a (free) Smashwords account, so an admin saves its sign-in once; it is kept encrypted on the server and used only to sign in at
+// smashwords.com. Smashwords has no public search, so searching looks through a remembered set of its popular free books.
+const fs = require('fs'), path = require('path'), crypto = require('crypto');
+const atomicWrite = require('./atomic');
+const auth = require('./auth');
+
+const BASE = () => (process.env.SMASHWORDS_URL || 'https://www.smashwords.com').replace(/\/+$/, '');
+const COVERS = () => (process.env.SMASHWORDS_COVERS || 'https://dwtr67e3ikfml.cloudfront.net').replace(/\/+$/, '');
+const LANG = () => process.env.SMASHWORDS_LANG || '1'; // 1 = English
+const UA = 'Mozilla/5.0 (compatible; NE-Hub/1.0; self-hosted e-book reader)';
+const DATA_DIR = process.env.DATA_DIR || path.join(__dirname, 'data'), FILE = () => path.join(DATA_DIR, 'store-smashwords.json');
+const MAX_BOOK = 300 * 1024 * 1024;
+const fail = (code, message) => Object.assign(new Error(message), { code });
+
+// ---- the family filter
+const LGBTQ = '886'; // Smashwords' own category for it
+const BAD_WORDS = '(?:erotic\\w*|erotica|bdsm|smut\\w*|lgbt\\w*|queer|lesbian|bisexual|transgender|non-?binary|same-sex|m/m|f/f|m/f/m|sex scenes?|sexually explicit|explicit (?:sex|content|scenes?)|adult (?:content|themes?|romance|fiction)|mature (?:content|audiences?|themes?)|steamy|spicy romance|harem|omegaverse|gay (?:romance|fiction|love|couple|men|man|male|teen|characters?|protagonist)|18\\+|nsfw)';
+const TEXT_BAD = new RegExp('(? String(s || '').replace(/<[^>]+>/g, ' ').replace(/ /g, ' ').replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"').replace(/�?39;/g, "'").replace(/\s+/g, ' ').trim();
+/** true when a book (Smashwords' own record for it) is fine to show. */
+function keep(b) {
+ if (!b || b.adult || b.erotica || b.taboo_erotica || b.need_erotic_cert) return false;
+ for (const c of b.categories || []) {
+ if (String(c.id) === LGBTQ || CAT_BAD.test(c.name || '')) return false;
+ for (const p of c.parents || []) if (String(p.id) === LGBTQ || CAT_BAD.test(p.name || '')) return false;
+ }
+ if ((b.tags || []).some(t => TAG_BAD.test(String(t)))) return false;
+ // Romance is where adult content hides without being flagged, so a romance is only shown when it says it is clean (sweet, Christian, inspirational...)
+ const romance = (b.categories || []).some(c => String(c.id) === ROMANCE || /romance/i.test(c.name || '') || (c.parents || []).some(p => String(p.id) === ROMANCE || /romance/i.test(p.name || ''))) || (b.tags || []).some(t => /romance/i.test(String(t)));
+ if (romance && !CLEAN.test([...(b.tags || []), ...(b.categories || []).flatMap(c => [c.name, ...(c.parents || []).map(p => p.name)])].join(' | '))) return false;
+ if (TEXT_BAD.test(plain(b.title) + ' ' + plain(b.short_description) + ' ' + plain(b.long_description))) return false;
+ return true;
+}
+const authorOf = b => (b.contributors || []).filter(c => !c.role || /^A0?1$/.test(c.role)).map(c => c.account && c.account.display_name).filter(Boolean).slice(0, 3).join(', ');
+const free = b => !b.price || (b.price.prices || []).every(p => +p.amount === 0) || b.price.type === 'FREE';
+const hasEpub = b => !b.formats || b.formats.some(f => /^epub$/i.test(f));
+const item = b => ({ ref: String(b.id), format: 'epub', title: plain(b.title), author: authorOf(b), summary: plain(b.short_description).slice(0, 600), cover: b.cover_url ? COVERS() + '/' + String(b.cover_url).replace(/^\/+/, '') : '', _t: [b.title, authorOf(b), (b.tags || []).join(' '), plain(b.short_description)].join(' ').toLowerCase() });
+const clean = ({ _t, ...rest }) => rest;
+
+// ---- Smashwords' shelf feed
+async function getJson(url) {
+ let r; try { r = await fetch(url, { headers: { 'user-agent': UA, accept: 'application/json' }, signal: AbortSignal.timeout(20000) }); } catch (e) { throw fail(502, e.name === 'TimeoutError' ? 'Smashwords did not answer in time' : 'Could not reach Smashwords'); }
+ if (!r.ok) throw fail(502, 'Smashwords answered ' + r.status);
+ try { return await r.json(); } catch { throw fail(502, 'Smashwords sent something unexpected'); }
+}
+const shelf = (type, cat, page) => getJson(`${BASE()}/api/bookshelves/category/v3/shelf/${type}/${cat}/free/any/no_erotica/${LANG()}/${page}/10`);
+const PER_API_PAGE = 10, API_PAGES_PER_PAGE = 3; // a page of the Store is three of Smashwords' pages of ten, since the filter removes some
+
+/** The most downloaded free books, a page at a time. */
+async function browse(page) {
+ const first = (page - 1) * API_PAGES_PER_PAGE + 1, pages = await Promise.all([0, 1, 2].map(i => shelf('mostdownloads', 1, first + i).catch(e => { if (i === 0) throw e; return { book_list: [], total_books: 0 }; })));
+ const seen = new Set(), items = [];
+ for (const p of pages) for (const e of p.book_list || []) { const b = e.book; if (b && !seen.has(b.id) && free(b) && hasEpub(b) && keep(b)) { seen.add(b.id); items.push(clean(item(b))); } }
+ const total = pages[0].total_books || 0;
+ return { items, more: first + API_PAGES_PER_PAGE - 1 < Math.ceil(total / PER_API_PAGE) && (pages[2].book_list || []).length > 0 };
+}
+
+// Smashwords has no public search, so one is made from its three best free shelves (most downloaded, newest, highest rated), remembered for a few hours.
+let poolP = null, poolAt = 0, poolDone = false;
+async function buildPool() {
+ const seen = new Set(), out = [];
+ for (const type of ['mostdownloads', 'highlyrated', 'newest']) {
+ const first = await shelf(type, 1, 1), n = Math.min(16, Math.ceil((first.total_books || 10) / PER_API_PAGE)), pages = [first];
+ for (let i = 2; i <= n; i += 6) pages.push(...await Promise.all(Array.from({ length: Math.min(6, n - i + 1) }, (_, k) => shelf(type, 1, i + k).catch(() => ({ book_list: [] })))));
+ for (const p of pages) for (const e of p.book_list || []) { const b = e.book; if (b && !seen.has(b.id) && free(b) && hasEpub(b) && keep(b)) { seen.add(b.id); out.push(item(b)); } }
+ }
+ return out;
+}
+const pool = () => { if (!poolP || Date.now() - poolAt > 6 * 3600e3) { poolAt = Date.now(); poolDone = false; poolP = buildPool().then(l => { poolDone = true; return l; }, e => { poolP = null; throw e; }); } return poolP; };
+async function search(q, page, wait = true) { // (wait: false is for a search across every source, which must not stall for the first-time build; it starts and the next search has it)
+ const words = q.toLowerCase().split(/\s+/).filter(Boolean);
+ if (!words.length) return browse(page);
+ if (!wait && !poolDone) { pool().catch(() => {}); return { items: [], more: false }; }
+ const hits = (await pool()).filter(b => words.every(w => b._t.includes(w)));
+ return { items: hits.slice((page - 1) * 20, page * 20).map(clean), more: hits.length > page * 20 };
+}
+
+// ---- the sign-in (admin saves it once; encrypted with a key from the server secret)
+let _key; const key = () => _key || (_key = crypto.scryptSync(auth.secret, 'nehub-smashwords', 32));
+const seal = v => { const iv = crypto.randomBytes(12), c = crypto.createCipheriv('aes-256-gcm', key(), iv), d = Buffer.concat([c.update(JSON.stringify(v), 'utf8'), c.final()]); return Buffer.concat([iv, c.getAuthTag(), d]).toString('base64'); };
+const unseal = s => { try { const b = Buffer.from(s, 'base64'), d = crypto.createDecipheriv('aes-256-gcm', key(), b.subarray(0, 12)); d.setAuthTag(b.subarray(12, 28)); return JSON.parse(Buffer.concat([d.update(b.subarray(28)), d.final()]).toString('utf8')); } catch { return null; } };
+const saved = () => { try { return unseal(JSON.parse(fs.readFileSync(FILE(), 'utf8')).data); } catch { return null; } };
+const account = () => { const s = saved(); return { signedIn: !!s, email: s ? s.email : '' }; };
+let session = null; // { cookie, t }
+function signOut() { try { fs.unlinkSync(FILE()); } catch {} session = null; }
+
+/** Cookies from a response ("name=value" pairs), merged into the jar. */
+function eat(jar, r) { for (const c of (r.headers.getSetCookie ? r.headers.getSetCookie() : [])) { const [kv] = c.split(';'), i = kv.indexOf('='); if (i > 0) { const v = kv.slice(i + 1).trim(); if (v && v !== 'deleted') jar.set(kv.slice(0, i).trim(), v); else jar.delete(kv.slice(0, i).trim()); } } }
+const header = jar => [...jar].map(([k, v]) => k + '=' + v).join('; ');
+async function req(url, jar, init = {}) { // one request; redirects are followed by hand so the cookies from each step are kept, and only smashwords' own address is used
+ for (let hop = 0; hop < 6; hop++) {
+ const u = new URL(url); if (u.origin !== new URL(BASE()).origin) throw fail(502, 'That address is not part of Smashwords');
+ const r = await fetch(u, { ...init, headers: { 'user-agent': UA, ...(init.headers || {}), cookie: header(jar) }, redirect: 'manual', signal: AbortSignal.timeout(init.timeout || 30000) }); eat(jar, r);
+ if ([301, 302, 303, 307, 308].includes(r.status) && r.headers.get('location')) { url = new URL(r.headers.get('location'), u).href; if (r.status !== 307 && r.status !== 308) { const { 'content-type': _c, ...hs } = init.headers || {}; init = { ...init, method: 'GET', body: undefined, headers: hs }; } try { await r.body?.cancel(); } catch {} if (/\/(signin|signup\/login)\b/.test(url) && init.stopAtLogin) return { r: { status: 401, headers: new Headers(), body: null }, url }; continue; }
+ return { r, url };
+ }
+ throw fail(502, 'Too many redirects from Smashwords');
+}
+async function login(email, pass) {
+ const jar = new Map(), page = await req(BASE() + '/signin', jar), html = await page.r.text(), token = (/name="loginToken"\s+value="([^"]*)"/.exec(html) || [])[1];
+ if (!token) throw fail(502, 'Smashwords\' sign-in page was not what was expected');
+ const out = await req(BASE() + '/auth/index', jar, { method: 'POST', headers: { 'content-type': 'application/x-www-form-urlencoded', referer: BASE() + '/signin' }, body: new URLSearchParams({ email, password: pass, loginToken: token, secToken: '' }).toString() });
+ const home = await (await req(BASE() + '/', jar)).r.text(); // the front page says whether this cookie is a signed-in one
+ if (!/loggedIn\s*=\s*true/.test(home)) { try { await out.r.body?.cancel(); } catch {} throw fail(401, 'Smashwords did not accept that email and password'); }
+ return jar;
+}
+/** Check a sign-in and save it. */
+async function signIn(email, pass) {
+ email = String(email || '').trim(); pass = String(pass || ''); if (!/^\S+@\S+\.\S+$/.test(email) || !pass) throw fail(400, 'Enter your Smashwords email and password');
+ const jar = await login(email, pass); session = { jar, t: Date.now() };
+ fs.mkdirSync(DATA_DIR, { recursive: true }); atomicWrite(FILE(), JSON.stringify({ data: seal({ email, pass }) }), { mode: 0o600 }); return account();
+}
+async function jar() {
+ if (session && Date.now() - session.t < 3 * 3600e3) return session.jar;
+ const s = saved(); if (!s) throw fail(401, 'An admin needs to add a Smashwords sign-in first (Store, Books, Catalogues)');
+ session = { jar: await login(s.email, s.pass), t: Date.now() }; return session.jar;
+}
+/** The EPUB of a free book, as the signed-in account. */
+async function download(id) {
+ if (!/^\d{1,9}$/.test(id)) throw fail(400, 'Bad book');
+ for (let attempt = 0; attempt < 2; attempt++) {
+ const j = await jar(), { r } = await req(`${BASE()}/books/download/${id}/8/latest/0/0/book.epub`, j, { timeout: 120000, stopAtLogin: true, headers: { accept: 'application/epub+zip, application/octet-stream, */*;q=0.5' } });
+ if (r.status === 401) { session = null; continue; } // the saved cookie ran out: sign in again once
+ if (!r.ok) throw fail(r.status === 404 ? 404 : 502, 'Smashwords answered ' + r.status);
+ if ((+r.headers.get('content-length') || 0) > MAX_BOOK) throw fail(413, 'That file is too large');
+ const chunks = []; let n = 0; for await (const c of r.body) { n += c.length; if (n > MAX_BOOK) throw fail(413, 'That file is too large'); chunks.push(c); }
+ const buf = Buffer.concat(chunks); if (buf.subarray(0, 2).toString('latin1') !== 'PK') throw fail(502, 'Smashwords did not send an EPUB for that book (it may not be free, or may not have one)');
+ return buf;
+ }
+ throw fail(401, 'Smashwords would not sign in with the saved email and password');
+}
+
+module.exports = { keep, browse, search, account, signIn, signOut, download, item, COVERS };
diff --git a/store.js b/store.js
index 61cc98f..ab7c85c 100644
--- a/store.js
+++ b/store.js
@@ -2,7 +2,7 @@
// admin adds (Calibre-Web, Kavita, ...). The hub server does all the talking to those sites: searching, covers and downloading a chosen book straight onto the
// signed-in person's shelf (books.js), so the browser never contacts them and no catalogue sign-in reaches the browser.
const fs = require('fs'), path = require('path'), crypto = require('crypto');
-const users = require('./users'), auth = require('./auth'), books = require('./books');
+const users = require('./users'), auth = require('./auth'), books = require('./books'), smashwords = require('./smashwords');
const UA = 'NE-Hub/1.0 (self-hosted e-book reader)';
const MAX_BOOK = 300 * 1024 * 1024, MAX_COVER = 3 * 1024 * 1024, MAX_TEXT = 4 * 1024 * 1024;
@@ -11,6 +11,7 @@ const BUILTIN = {
gutenberg: { name: 'Project Gutenberg', base: () => env('GUTENBERG_URL', 'https://www.gutenberg.org'), hosts: ['gutenberg.org'] }, // its own catalogue feed (fast and official)
standardebooks: { name: 'Standard Ebooks', base: () => env('STANDARD_EBOOKS_URL', 'https://standardebooks.org'), hosts: ['standardebooks.org'] }, // its public pages: the OPDS feed is for paying members only
archive: { name: 'Internet Archive', base: () => env('ARCHIVE_URL', 'https://archive.org'), hosts: ['archive.org'], needsQuery: true },
+ smashwords: { name: 'Smashwords', base: () => env('SMASHWORDS_URL', 'https://www.smashwords.com'), hosts: ['smashwords.com', 'dwtr67e3ikfml.cloudfront.net'] }, // free EPUBs, with a family filter; downloading needs an account (see smashwords.js)
openlibrary: { name: 'Open Library', base: () => env('OPENLIBRARY_URL', 'https://openlibrary.org'), hosts: ['openlibrary.org'], needsQuery: true }, // search here; the book itself comes from the Internet Archive
};
const FORMAT_OF = { 'application/epub+zip': 'epub', 'application/pdf': 'pdf', 'application/x-mobipocket-ebook': 'mobi', 'application/vnd.amazon.mobi8-ebook': 'azw3', 'application/vnd.amazon.ebook': 'azw' };
@@ -144,15 +145,16 @@ async function openLibrarySearch(src, q, page) {
return { items, more: page * 20 < (j.numFound || 0) };
}
-async function searchOne(src, q, page) {
+async function searchOne(src, q, page, everywhere) {
if (src.id === 'gutenberg') return gutenbergSearch(src, q, page);
if (src.id === 'standardebooks') return standardSearch(src, q, page);
if (src.id === 'archive') return q ? archiveSearch(src, q, page) : { items: [], more: false };
+ if (src.id === 'smashwords') return smashwords.search(q, page, !everywhere);
if (src.id === 'openlibrary') return q ? openLibrarySearch(src, q, page) : { items: [], more: false };
return opdsSearch(src, q, page);
}
const stamp = (src, it) => ({ ...it, source: src.id, sourceName: src.name, cover: it.cover ? `/api/read/store/cover?s=${encodeURIComponent(src.id)}&u=${encodeURIComponent(it.cover)}` : '', coverUrl: it.cover || '' });
-const listSources = () => [...Object.keys(BUILTIN), ...customs().map(c => c.id)].map(id => { const s = sourceOf(id); return { id, name: s.name, builtin: s.builtin, needsQuery: !!s.needsQuery }; });
+const listSources = () => [...Object.keys(BUILTIN), ...customs().map(c => c.id)].map(id => { const s = sourceOf(id); return { id, name: s.name, builtin: s.builtin, needsQuery: !!s.needsQuery, ...(id === 'smashwords' ? { account: smashwords.account() } : {}) }; });
async function handle(req, res, p) {
const me = users.current();
@@ -168,6 +170,11 @@ async function handle(req, res, p) {
catch (e) { saveCustoms(customs().filter(x => x.id !== id)); throw fail(e.code || 502, e.code ? e.message : 'Could not read that catalogue: ' + e.message); }
return json(res, 200, { id, name });
}
+ if (p === '/smashwords' && req.method === 'POST') { // the Smashwords sign-in (admin): checked by signing in, then kept encrypted
+ if (!me || me.role !== 'admin') throw fail(403, 'Only an admin can add the Smashwords sign-in');
+ const b = await readBody(req); return json(res, 200, await smashwords.signIn(b.email, b.pass));
+ }
+ if (p === '/smashwords' && req.method === 'DELETE') { if (!me || me.role !== 'admin') throw fail(403, 'Only an admin can remove the Smashwords sign-in'); smashwords.signOut(); return json(res, 200, smashwords.account()); }
let m;
if ((m = /^\/sources\/(\w+)$/.exec(p)) && req.method === 'DELETE') {
if (!me || me.role !== 'admin') throw fail(403, 'Only an admin can remove a catalogue');
@@ -176,7 +183,7 @@ async function handle(req, res, p) {
if (p === '/search' && req.method === 'GET') {
const u = new URL(req.url, 'http://x'), q = (u.searchParams.get('q') || '').trim().slice(0, 120), page = Math.min(20, Math.max(1, +u.searchParams.get('page') || 1)), want = u.searchParams.get('source') || 'all';
const ids = want === 'all' ? listSources().map(s => s.id) : [want], lists = [], errors = {}; let more = false;
- await Promise.all(ids.map(async id => { let src; try { src = sourceOf(id); const r = await searchOne(src, q, page); lists.push([id, r.items.map(it => stamp(src, it))]); more = more || r.more; } catch (e) { const nm = src ? src.name : id, msg = e.name === 'TimeoutError' ? 'did not answer in time' : e.message; errors[id] = msg.startsWith(nm) ? msg : nm + ': ' + msg; } }));
+ await Promise.all(ids.map(async id => { let src; try { src = sourceOf(id); const r = await searchOne(src, q, page, want === 'all'); lists.push([id, r.items.map(it => stamp(src, it))]); more = more || r.more; } catch (e) { const nm = src ? src.name : id, msg = e.name === 'TimeoutError' ? 'did not answer in time' : e.message; errors[id] = msg.startsWith(nm) ? msg : nm + ': ' + msg; } }));
lists.sort((a, b) => ids.indexOf(a[0]) - ids.indexOf(b[0]));
const items = []; for (let i = 0; lists.some(l => l[1][i]); i++) for (const l of lists) if (l[1][i]) items.push(l[1][i]); // one from each catalogue in turn
return json(res, 200, { items, errors, more });
@@ -192,10 +199,11 @@ async function handle(req, res, p) {
if (src.id === 'gutenberg') target = await gutenbergResolve(src, ref);
else if (src.id === 'standardebooks') target = await standardResolve(src, ref);
else if (src.id === 'archive') target = await archiveResolve(src, ref);
+ else if (src.id === 'smashwords') target = { format: 'epub' };
else if (src.id === 'openlibrary') { dl = sourceOf('archive'); target = await archiveResolve(dl, ref); } // (Open Library's free editions are Internet Archive items)
else { if (!sameSite(src, new URL(ref, src.base))) throw fail(400, 'Bad book address'); target = { url: ref, format: String(b.format || 'epub') }; }
const ext = books.TYPES[target.format] ? target.format : 'epub';
- const buf = await limited(await get(dl, target.url, { accept: 'application/epub+zip, application/octet-stream, */*;q=0.5', timeout: 120000 }), MAX_BOOK);
+ const buf = src.id === 'smashwords' ? await smashwords.download(ref) : await limited(await get(dl, target.url, { accept: 'application/epub+zip, application/octet-stream, */*;q=0.5', timeout: 120000 }), MAX_BOOK);
let cover; const cu = String(b.cover || ''); if (cu) { try { const r = await get(src, cu, { accept: 'image/*' }), type = (r.headers.get('content-type') || '').split(';')[0].trim().toLowerCase(); cover = { type, data: await limited(r, MAX_COVER) }; } catch {} }
let book; try { book = books.addBook(buf, ext, { title: String(b.title || '').slice(0, 200), author: String(b.author || '').slice(0, 200) }, cover); } catch (e) { throw fail(400, e.message); }
return json(res, 200, book);
diff --git a/tools/librivox-test.js b/tools/librivox-test.js
index c662fcd..3a0c20a 100644
--- a/tools/librivox-test.js
+++ b/tools/librivox-test.js
@@ -12,6 +12,8 @@ const fake = http.createServer((req, res) => {
if (u.searchParams.get('title') === 'prejudice' || u.searchParams.get('author') === 'austen') return J({ books: [book] });
return J({ error: 'Audiobooks could not be found' }, 404);
}
+ if (p === '/dl/1.mp3') { const rg = /^bytes=(\d+)-(\d+)$/.exec(req.headers.range || ''), data = Buffer.from('0123456789'); if (rg) { res.writeHead(206, { 'content-type': 'audio/mpeg', 'content-range': `bytes ${rg[1]}-${rg[2]}/10`, 'content-length': +rg[2] - +rg[1] + 1 }); return res.end(data.subarray(+rg[1], +rg[2] + 1)); } res.writeHead(200, { 'content-type': 'audio/mpeg', 'content-length': 10 }); return res.end(data); }
+ if (p === '/dl/moved.mp3') { res.writeHead(302, { location: '/dl/1.mp3' }); return res.end(); }
if (p === '/cover.gif') { res.writeHead(200, { 'content-type': 'image/gif' }); return res.end(gif); }
res.writeHead(404); res.end();
}).listen(0, () => {
@@ -25,7 +27,8 @@ const fake = http.createServer((req, res) => {
assert.deepEqual((await j(await call('/search?q=zzz'))).items, [], 'nothing found is an empty list, not an error');
assert.ok((await j(await call('/search'))).items.length >= 0, 'no search words lists the catalogue');
r = await call('/book/55'); const b = await j(r); assert.equal(b.chapters.length, 2, 'chapters on other sites are dropped'); assert.ok(!JSON.stringify(b).includes('127.0.0.1:' + port + '/dl'), 'audio addresses are not sent to the browser'); assert.equal(b.chapters[1].title, 'Chapter 3'); assert.equal(b.description, 'A novel');
- r = await call('/audio/55/0'); assert.equal(r.status, 302); assert.equal(r.headers.get('location'), `http://127.0.0.1:${port}/dl/1.mp3`);
+ r = await call('/audio/55/0'); assert.equal(r.status, 200); assert.equal(r.headers.get('content-type'), 'audio/mpeg'); assert.equal(await r.text(), '0123456789', 'the hub passes the recording on itself');
+ r = await call('/audio/55/0', { headers: { range: 'bytes=2-4' } }); assert.equal(r.status, 206); assert.equal(await r.text(), '234'); assert.equal(r.headers.get('content-range'), 'bytes 2-4/10', 'seeking works');
assert.equal((await call('/audio/55/9')).status, 404); assert.equal((await call('/book/abc')).status, 404);
r = await call('/cover/55'); assert.equal(r.headers.get('content-type'), 'image/gif');
assert.equal((await call('/cover/55?u=' + encodeURIComponent('http://169.254.169.254/x'))).status, 200, 'an address off the allowed sites is ignored (the book\'s own cover is used)');
diff --git a/tools/store-test.js b/tools/store-test.js
index 245211c..cee0c26 100644
--- a/tools/store-test.js
+++ b/tools/store-test.js
@@ -3,7 +3,7 @@ const http = require('http'), assert = require('assert'), fs = require('fs'), os
process.env.DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'nehub-store-'));
const fx = fs.mkdtempSync(path.join(os.tmpdir(), 'nehub-storefx-')); cp.execFileSync('python3', [path.join(__dirname, 'mkbooks.py'), fx]);
const epub = fs.readFileSync(path.join(fx, 'test.epub')), pdf = fs.readFileSync(path.join(fx, 'test.pdf')), gif = Buffer.from('R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==', 'base64');
-const seen = []; let port;
+const seen = [], swPosts = []; let port;
const feed = (body, links = '') => `${links}${body} `;
const acq = (id, title, author, href, type = 'application/epub+zip') => `${title} ${author} ${id} A & B ${title}
${who}
}.gif)