${title}

// Book store: search across Gutenberg, Standard Ebooks, the Internet Archive and an OPDS catalogue; adding a book puts it on the shelf; sites only reach their own addresses. const http = require('http'), assert = require('assert'), fs = require('fs'), os = require('os'), path = require('path'), cp = require('child_process'); process.env.DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'nehub-store-')); const fx = fs.mkdtempSync(path.join(os.tmpdir(), 'nehub-storefx-')); cp.execFileSync('python3', [path.join(__dirname, 'mkbooks.py'), fx]); const epub = fs.readFileSync(path.join(fx, 'test.epub')), pdf = fs.readFileSync(path.join(fx, 'test.pdf')), gif = Buffer.from('R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==', 'base64'); const seen = [], swPosts = []; let port; const feed = (body, links = '') => `${links}${body}`; const acq = (id, title, author, href, type = 'application/epub+zip') => `${title}${author}${id}

A & B

`; const seCard = (slug, title, who) => `

`; const fake = http.createServer((req, res) => { const u = new URL(req.url, 'http://x'), p = u.pathname; seen.push(p + u.search); const J = o => { res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify(o)); }, X = x => { res.writeHead(200, { 'content-type': 'application/atom+xml' }); res.end(x); }, HT = x => { res.writeHead(200, { 'content-type': 'application/xhtml+xml' }); res.end(x); }; // Project Gutenberg's catalogue feed if (p === '/ebooks/search.opds/') { const more = u.searchParams.get('start_index') ? '' : ''; return X(feed(`Authorshttps://x/ebooks/author/1.opdsAlice in Wonderlandhttps://www.gutenberg.org/ebooks/11.opdsLewis CarrollAlice in Wonderlandhttps://www.gutenberg.org/ebooks/11.opdsLewis CarrollDracula & Cohttps://www.gutenberg.org/ebooks/345.opdsBram Stoker`, more)); } if (p === '/ebooks/11.epub3.images') { res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(epub); } if (p === '/ebooks/66.epub3.images') { res.writeHead(302, { location: `http://localhost:${port}/secret` }); return res.end(); } if (p.startsWith('/cache/epub/') || p.startsWith('/img/')) { res.writeHead(200, { 'content-type': 'image/gif' }); return res.end(gif); } // Standard Ebooks' public pages (their OPDS feed is for members only) if (p === '/ebooks') { const q = u.searchParams.get('query') || ''; return HT(`

${seCard('bram-stoker/dracula', 'Dracula ' + q, 'Bram Stoker')}${seCard('john-bunyan/the-pilgrims-progress', 'The Pilgrim’s Progress', 'John Bunyan')}`); } if (p === '/ebooks/bram-stoker/dracula') return HT(`k a e`); if (p === '/ebooks/bram-stoker/dracula/downloads/bram-stoker_dracula.epub') { if (u.searchParams.get('source') !== 'download') { res.writeHead(200, { 'content-type': 'text/html' }); return res.end('Your Download Has Started!'); } res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(Buffer.concat([epub, Buffer.from('se')])); } if (p === '/feeds/opds') { res.writeHead(401); return res.end('members only'); } // the Internet Archive if (p === '/ia/advancedsearch.php') return J({ response: { numFound: 1, docs: [{ identifier: 'moby1', title: 'Moby', creator: ['Melville'] }, { identifier: '../bad', title: 'x' }] } }); if (p === '/ia/metadata/moby1') return J({ metadata: {}, files: [{ name: 'moby1.pdf', format: 'Text PDF' }, { name: 'moby1.epub', format: 'EPUB' }] }); if (p === '/ia/metadata/locked') return J({ metadata: { 'access-restricted-item': 'true' }, files: [{ name: 'x.epub', format: 'EPUB' }] }); if (p === '/ia/download/moby1/moby1.epub') { res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(Buffer.concat([epub, Buffer.from('ia')])); } // Open Library (search only; the book is an Internet Archive item) if (p === '/ol/search.json') return J({ numFound: 2, docs: [{ title: 'Moby Dick', author_name: ['Melville'], cover_i: 7, ia: ['moby1'], ebook_access: 'public' }, { title: 'Borrowable', ia: ['locked'], ebook_access: 'borrowable' }] }); // Smashwords: its shelf feed (filtered), a sign-in form, and downloads that need the signed-in cookie if (p === '/sw/signin') { res.writeHead(200, { 'content-type': 'text/html', 'set-cookie': 'XSRF-TOKEN=deleted; Path=/' }); return res.end(''); } if (p === '/sw/auth/index' && req.method === 'POST') { let b = ''; req.on('data', c => b += c); req.on('end', () => { const f = new URLSearchParams(b); swPosts.push(Object.fromEntries(f)); if (f.get('email') === 'me@x.org' && f.get('password') === 'pw' && f.get('loginToken') === 'tok123') { res.writeHead(302, { location: '/sw/', 'set-cookie': 'sess=abc; Path=/; HttpOnly' }); } else res.writeHead(200); res.end(); }); return; } if (p === '/sw/') { res.writeHead(200, { 'content-type': 'text/html' }); return res.end(``); } if (p === '/sw/books/download/101/8/latest/0/0/book.epub') { if (!/sess=abc/.test(req.headers.cookie || '')) { res.writeHead(302, { location: '/sw/signup/login/books/101' }); return res.end(); } res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(epub); } if (p === '/sw/signup/login/books/101') { res.writeHead(200); return res.end('login'); } if (p.startsWith('/sw/api/bookshelves/category/v3/shelf/')) { const pg = +p.split('/').slice(-2)[0]; // (.../{page}/{pageSize}) const bk = (id, o = {}) => ({ book: { id: String(id), title: 'Book ' + id, price: { type: 'FIXED_PRICE', prices: [{ amount: '0.00' }] }, short_description: 'A fine tale.', cover_url: 'bookCovers/c' + id + '.gif', adult: false, erotica: false, taboo_erotica: false, need_erotic_cert: false, categories: [{ id: '1', name: 'Fiction', parents: [] }], tags: ['adventure'], formats: ['EPUB'], contributors: [{ role: 'A01', account: { display_name: 'Ann Author' } }], ...o } }); const list = pg === 1 ? [bk(101, { title: 'Pilgrim Story' }), bk(102, { categories: [{ id: '5', name: 'Romance', parents: [{ id: '886', name: 'LGBTQ+' }] }] }), bk(103, { erotica: true }), bk(104, { tags: ['gay romance'] }), bk(105, { title: 'Gaylord\'s Garden', contributors: [{ role: 'A01', account: { display_name: 'Gay Talese' } }] }), bk(106, { short_description: 'A steamy love story' }), bk(107, { price: { type: 'FIXED_PRICE', prices: [{ amount: '2.99' }] } }), bk(109, { categories: [{ id: '1235', name: 'Romance', parents: [{ id: '3', name: 'Fiction' }] }], tags: ['billionaire romance'] }), bk(110, { title: 'Sweet Home', categories: [{ id: '1235', name: 'Romance', parents: [{ id: '3', name: 'Fiction' }] }], tags: ['sweet romance'] }), bk(108, { categories: [{ id: '886', name: 'LGBTQ+', parents: [{ id: '3', name: 'Fiction' }] }] })] : []; res.writeHead(200, { 'content-type': 'application/json' }); return res.end(JSON.stringify({ name: 'Most Downloaded', type: 'mostdownloads', book_list: list, total_books: 8 })); } if (p.startsWith('/swc/')) { res.writeHead(200, { 'content-type': 'image/gif' }); return res.end(gif); } // an OPDS catalogue that needs a sign-in (Calibre-Web style) if (p.startsWith('/cal/')) { if (req.headers.authorization !== 'Basic ' + Buffer.from('reader:pw').toString('base64')) { res.writeHead(401); return res.end(); } if (p === '/cal/opds') return X(feed('', '')); if (p === '/cal/opds/osd') return X(''); if (p.startsWith('/cal/opds/search/')) return X(feed(acq('c1', 'Own Book ' + decodeURIComponent(p.split('/').pop()), 'Me', '/cal/get/1.pdf', 'application/pdf'))); if (p === '/cal/get/1.pdf') { res.writeHead(200, { 'content-type': 'application/pdf' }); return res.end(pdf); } } res.writeHead(404); res.end('no'); }).listen(0, async () => { port = fake.address().port; const B = 'http://127.0.0.1:' + port; Object.assign(process.env, { GUTENBERG_URL: B, ARCHIVE_URL: B + '/ia', OPENLIBRARY_URL: B + '/ol', SMASHWORDS_URL: B + '/sw', SMASHWORDS_COVERS: B + '/swc', STANDARD_EBOOKS_URL: B }); const books = require('../books'), users = require('../users'); const people = { a: users.create({ name: 'Ann', role: 'admin', handle: 'YQ' }), b: users.create({ name: 'Bob', role: 'user', tabs: ['read'], handle: 'Yg' }) }; const hub = http.createServer((req, res) => users.runAs(req.headers['x-user'] === 'b' ? people.b : people.a, () => books.handle(req, res))).listen(0, async () => { const base = 'http://127.0.0.1:' + hub.address().port + '/api/read/store', call = (p, o = {}) => fetch(base + p, o), post = (p, b, h = {}) => call(p, { method: 'POST', body: JSON.stringify(b), headers: h }); try { let r = await call('/sources'); assert.deepEqual((await r.json()).map(s => s.id), ['gutenberg', 'standardebooks', 'archive', 'smashwords', 'openlibrary']); r = await call('/search?q=alice'); let j = await r.json(); assert.deepEqual(j.items.map(i => i.source), ['gutenberg', 'standardebooks', 'archive', 'openlibrary', 'gutenberg', 'standardebooks'].slice(0, j.items.length), 'one result from each catalogue in turn'); assert.deepEqual(j.errors, {}); const g = j.items.filter(i => i.source === 'gutenberg'); assert.deepEqual(g.map(i => i.ref), ['11', '345'], 'catalogue entries that are not books, and repeats, are left out'); assert.equal(g[0].author, 'Lewis Carroll'); assert.equal(g[1].title, 'Dracula & Co'); assert.match(g[0].cover, /^\/api\/read\/store\/cover\?s=gutenberg/); assert.equal(j.more, true); const se = j.items.filter(i => i.source === 'standardebooks'); assert.equal(se[0].title, 'Dracula alice'); assert.equal(se[0].author, 'Bram Stoker'); assert.equal(se[1].title, 'The Pilgrim’s Progress'); assert.equal(se[0].ref, '/ebooks/bram-stoker/dracula'); assert.ok(!j.items.some(i => i.ref === '../bad'), 'odd identifiers are dropped'); assert.ok(!seen.some(x => x.startsWith('/feeds/opds')), 'the members-only feed is never asked for'); r = await call('/search?q=&source=gutenberg'); j = await r.json(); assert.ok(seen.some(x => x.includes('sort_order=downloads')), 'no search words: the most downloaded books'); assert.equal(j.items.length, 2); r = await call('/search?q=x&source=gutenberg&page=2'); assert.ok(seen.some(x => x.includes('start_index=26')), 'later pages ask for the next 25'); assert.equal((await r.json()).more, false); r = await call('/search?q=x&source=nope'); j = await r.json(); assert.ok(j.errors.nope, 'a broken catalogue is reported and does not stop the rest'); r = await call('/cover?s=gutenberg&u=' + encodeURIComponent(B + '/img/11.gif')); assert.equal(r.headers.get('content-type'), 'image/gif'); assert.equal((await call('/cover?s=gutenberg&u=' + encodeURIComponent('http://localhost:' + port + '/img/11.gif'))).status, 502, 'covers only come from the catalogue\'s own site'); // adding r = await post('/add', { source: 'gutenberg', ref: '11', title: 'Alice', author: 'Lewis Carroll', cover: B + '/cache/epub/11/pg11.cover.medium.jpg' }); let book = await r.json(); assert.equal(r.status, 200); assert.equal(book.ext, 'epub'); assert.equal(book.author, 'Lewis Carroll'); assert.equal(book.cover, true); assert.ok(seen.includes('/ebooks/11.epub3.images'), 'the EPUB with pictures is fetched'); r = await post('/add', { source: 'gutenberg', ref: '11', title: 'Alice' }); assert.equal((await r.json()).duplicate, true, 'adding the same book twice does not store it twice'); assert.equal((await (await fetch(base.replace('/store', '/list'))).json()).length, 1); assert.equal((await post('/add', { source: 'gutenberg', ref: '11' }, { 'x-user': 'b' })).status, 200); assert.equal((await (await fetch(base.replace('/store', '/list'), { headers: { 'x-user': 'b' } })).json()).length, 1, "each person's shelf is their own"); r = await post('/add', { source: 'standardebooks', ref: '/ebooks/bram-stoker/dracula', title: 'Dracula', author: 'Bram Stoker' }); assert.equal(r.status, 200, 'the real EPUB is fetched, not the "download has started" page'); assert.ok(seen.some(x => x.includes('bram-stoker_dracula.epub?source=download')) && !seen.some(x => x.includes('kepub') && x.includes('source')), 'the plain EPUB (not the Kobo or advanced one) is chosen'); r = await post('/add', { source: 'standardebooks', ref: '/ebooks/../etc/passwd' }); assert.equal(r.status, 400); r = await post('/add', { source: 'archive', ref: 'moby1', title: 'Moby', author: 'Melville' }); assert.equal((await r.json()).title, 'Moby'); assert.ok(seen.includes('/ia/download/moby1/moby1.epub'), 'the EPUB is chosen over the PDF'); const ol = (await (await call('/search?q=moby&source=openlibrary')).json()).items; assert.deepEqual(ol.map(i => i.ref), ['moby1'], 'only freely readable editions are listed'); assert.equal(ol[0].author, 'Melville'); assert.match(decodeURIComponent(ol[0].coverUrl), /covers\.openlibrary\.org\/b\/id\/7-M\.jpg/); r = await post('/add', { source: 'openlibrary', ref: 'moby1', title: 'Moby Dick' }); assert.equal(r.status, 200, 'an Open Library book is fetched from the Internet Archive'); // genres: each source is asked in its own words assert.deepEqual((await (await call('/genres')).json()).slice(0, 3).map(g => g.id), ['fiction', 'mystery', 'fantasy']); const sawSince = (n, re) => seen.slice(n).some(x => re.test(decodeURIComponent(x))); let n0 = seen.length; await call('/search?genre=mystery&source=gutenberg'); assert.ok(sawSince(n0, /search\.opds\/\?query=s\.mystery/), 'Gutenberg gets a subject search'); n0 = seen.length; await call('/search?q=hound&genre=mystery&source=gutenberg'); assert.ok(sawSince(n0, /query=hound s\.mystery/), 'with words and a genre'); n0 = seen.length; await call('/search?genre=fantasy&source=standardebooks'); assert.ok(sawSince(n0, /tags\[\]=Fantasy/), 'Standard Ebooks gets a tag'); n0 = seen.length; await call('/search?genre=classics&source=standardebooks'); assert.ok(!sawSince(n0, /\/ebooks\?/), 'a genre it has no tag for is not asked'); n0 = seen.length; await call('/search?genre=history&source=archive'); assert.ok(sawSince(n0, /advancedsearch[^ ]*subject:\(history\)/), 'the Archive is asked by subject even with no words'); n0 = seen.length; await call('/search?genre=poetry&source=openlibrary'); assert.ok(sawSince(n0, /subject=poetry&has_fulltext=true/), 'Open Library by subject'); n0 = seen.length; await call('/search?genre=mystery&source=smashwords'); assert.ok(sawSince(n0, /shelf\/mostdownloads\/879\//), 'Smashwords by its category number'); n0 = seen.length; j = await (await call('/search?genre=philosophy&source=smashwords')).json(); assert.deepEqual(j.items, [], 'a genre Smashwords has no category for gives nothing'); j = await (await call('/search?genre=mystery')).json(); assert.ok(j.items.length > 0 && Object.keys(j.errors || {}).length === 0, 'asking every source at once works'); assert.deepEqual((await (await call('/search?genre=nope&source=gutenberg')).json()).errors, {}, 'an unknown genre is just ignored'); // a child account sees a few kid-friendly sources and genres users.update(people.b.id, { kid: { on: true, limitMin: 0, bedtime: { on: false }, safe: true } }); const kb = { 'x-user': 'b' }; assert.deepEqual((await (await call('/sources', { headers: kb })).json()).map(s => s.id), ['gutenberg', 'standardebooks'], 'only two sources'); n0 = seen.length; j = await (await call('/search?q=oz&source=all', { headers: kb })).json(); assert.ok(sawSince(n0, /search\.opds\/\?query=oz s\.juvenile/), 'with no genre, it is the children\'s genre'); assert.ok(!sawSince(n0, /\/ia\/|\/ol\/|\/sw\//), 'and the other sources are not asked'); n0 = seen.length; await call('/search?genre=mystery&source=gutenberg', { headers: kb }); assert.ok(sawSince(n0, /s\.juvenile/) && !sawSince(n0, /s\.mystery/), 'a genre outside the kid list becomes the children\'s one'); assert.equal((await call('/search?source=smashwords', { headers: kb })).status, 403); assert.equal((await post('/add', { source: 'smashwords', ref: '101' }, { 'x-user': 'b' })).status, 403, 'and cannot add from the others'); users.update(people.b.id, { kid: { on: false } }); // Smashwords: the family filter, the sign-in, and the download r = await call('/search?q=&source=smashwords'); j = await r.json(); assert.deepEqual(j.items.map(i => i.ref), ['101', '105', '110'], 'adult, LGBTQ+, keyword-flagged, unlabelled romance and paid books are left out; a clean romance and a harmless name are kept'); assert.equal(j.items[0].author, 'Ann Author'); assert.match(j.items[0].cover, /^\/api\/read\/store\/cover\?s=smashwords/); j = await (await call('/search?q=pilgrim&source=smashwords')).json(); assert.deepEqual(j.items.map(i => i.ref), ['101'], 'searching looks through the free books it can see'); assert.deepEqual((await (await call('/search?q=gay&source=smashwords')).json()).items.map(i => i.ref), ['105']); j = await (await call('/sources')).json(); assert.deepEqual(j.find(s => s.id === 'smashwords').account, { signedIn: false, email: '' }); r = await post('/add', { source: 'smashwords', ref: '101', title: 'Pilgrim Story' }); assert.equal(r.status, 401); assert.match((await r.json()).error, /sign-in/, 'downloading needs the account'); assert.equal((await post('/smashwords', { email: 'me@x.org', pass: 'pw' }, { 'x-user': 'b' })).status, 403, 'only an admin saves the sign-in'); r = await post('/smashwords', { email: 'me@x.org', pass: 'wrong' }); assert.equal(r.status, 401); assert.equal((await (await call('/sources')).json()).find(s => s.id === 'smashwords').account.signedIn, false, 'a wrong password is not kept'); r = await post('/smashwords', { email: 'me@x.org', pass: 'pw' }); assert.equal(r.status, 200); assert.deepEqual(await r.json(), { signedIn: true, email: 'me@x.org' }); assert.equal(swPosts.at(-1).loginToken, 'tok123', 'the sign-in form\'s own token is sent back'); assert.ok(!fs.readFileSync(path.join(process.env.DATA_DIR, 'store-smashwords.json'), 'utf8').includes('"pw"') && !JSON.stringify(await (await call('/sources')).json()).includes('"pw"'), 'the password is encrypted on disk and never sent to the browser'); r = await post('/add', { source: 'smashwords', ref: '101', title: 'Pilgrim Story', author: 'Ann Author', cover: B + '/swc/c101.gif' }); assert.equal(r.status, 200, 'the signed-in download is accepted'); assert.ok(seen.includes('/sw/books/download/101/8/latest/0/0/book.epub'), 'the EPUB is fetched with the signed-in cookie'); assert.equal((await post('/add', { source: 'smashwords', ref: '../x' })).status, 400); assert.equal((await call('/smashwords', { method: 'DELETE' })).status, 200); assert.equal((await post('/add', { source: 'smashwords', ref: '101' })).status, 401, 'removing the sign-in stops downloads'); r = await post('/add', { source: 'archive', ref: 'locked' }); assert.equal(r.status, 403); assert.match((await r.json()).error, /DRM/); r = await post('/add', { source: 'gutenberg', ref: '../x' }); assert.equal(r.status, 400); // custom catalogue assert.equal((await post('/sources', { name: 'Mine', url: B + '/cal/opds' }, { 'x-user': 'b' })).status, 403, 'only an admin adds catalogues'); assert.equal((await post('/sources', { name: 'Bad', url: B + '/cal/opds', user: 'reader', pass: 'wrong' })).status, 502, 'a catalogue that refuses the sign-in is not added'); assert.equal((await call('/sources').then(x => x.json())).length, 5); r = await post('/sources', { name: 'Mine', url: B + '/cal/opds', user: 'reader', pass: 'pw' }); const mine = await r.json(); assert.equal(r.status, 200); assert.ok(!JSON.stringify(await (await call('/sources')).json()).includes('pw'), 'sign-ins are never sent to the browser'); assert.ok(!fs.readFileSync(path.join(process.env.DATA_DIR, 'store-sources.json'), 'utf8').includes('"pw"'), 'and are encrypted on disk'); j = await (await call('/search?q=hello%20world&source=' + mine.id)).json(); assert.equal(j.items[0].title, 'Own Book hello world', 'search found through the OpenSearch description'); assert.equal(j.items[0].format, 'pdf'); r = await post('/add', { source: mine.id, ref: j.items[0].ref, title: j.items[0].title, format: 'pdf' }, { 'x-user': 'b' }); assert.equal((await r.json()).ext, 'pdf'); assert.equal((await call('/sources/' + mine.id, { method: 'DELETE', headers: { 'x-user': 'b' } })).status, 403); assert.equal((await call('/sources/' + mine.id, { method: 'DELETE' })).status, 200); console.log('store test ok'); } catch (err) { console.error(err); process.exitCode = 1; } hub.close(); fake.close(); }); });