hub/tools/store-test.js

// Book store: search across Gutenberg, Standard Ebooks, the Internet Archive and an OPDS catalogue; adding a book puts it on the shelf; sites only reach their own addresses.
const http = require('http'), assert = require('assert'), fs = require('fs'), os = require('os'), path = require('path'), cp = require('child_process');
process.env.DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'nehub-store-'));
const fx = fs.mkdtempSync(path.join(os.tmpdir(), 'nehub-storefx-')); cp.execFileSync('python3', [path.join(__dirname, 'mkbooks.py'), fx]);
const epub = fs.readFileSync(path.join(fx, 'test.epub')), pdf = fs.readFileSync(path.join(fx, 'test.pdf')), gif = Buffer.from('R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==', 'base64');
const seen = [], swPosts = []; let port;
const feed = (body, links = '') => `<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom">${links}${body}</feed>`;
const acq = (id, title, author, href, type = 'application/epub+zip') => `<entry><title>${title}</title><author><name>${author}</name></author><id>${id}</id><summary>A &amp; B</summary><link rel="http://opds-spec.org/image" href="/img/${id}.gif" type="image/gif"/><link rel="http://opds-spec.org/acquisition/open-access" href="${href}" type="${type}"/></entry>`;
const seCard = (slug, title, who) => `<li typeof="schema:Book" about="/ebooks/${slug}"><div><a href="/ebooks/${slug}"><picture><img src="/img/${slug.replace(/\W/g, '')}.gif" alt="" property="schema:image"/></picture></a></div><p><a href="/ebooks/${slug}" property="schema:url"><span property="schema:name">${title}</span></a></p><p class="author" typeof="schema:Person"><a href="https://x/y" property="schema:url"><span property="schema:name">${who}</span></a></p></li>`;
const fake = http.createServer((req, res) => {
  const u = new URL(req.url, 'http://x'), p = u.pathname; seen.push(p + u.search);
  const J = o => { res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify(o)); }, X = x => { res.writeHead(200, { 'content-type': 'application/atom+xml' }); res.end(x); }, HT = x => { res.writeHead(200, { 'content-type': 'application/xhtml+xml' }); res.end(x); };
  // Project Gutenberg's catalogue feed
  if (p === '/ebooks/search.opds/') { const more = u.searchParams.get('start_index') ? '' : '<link rel="next" type="application/atom+xml" href="/ebooks/search.opds/?query=x&amp;start_index=26"/>';
    return X(feed(`<entry><title>Authors</title><id>https://x/ebooks/author/1.opds</id></entry><entry><title>Alice in Wonderland</title><id>https://www.gutenberg.org/ebooks/11.opds</id><content type="text">Lewis Carroll</content></entry><entry><title>Alice in Wonderland</title><id>https://www.gutenberg.org/ebooks/11.opds</id><content>Lewis Carroll</content></entry><entry><title>Dracula &amp; Co</title><id>https://www.gutenberg.org/ebooks/345.opds</id><content>Bram Stoker</content></entry>`, more)); }
  if (p === '/ebooks/11.epub3.images') { res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(epub); }
  if (p === '/ebooks/66.epub3.images') { res.writeHead(302, { location: `http://localhost:${port}/secret` }); return res.end(); }
  if (p.startsWith('/cache/epub/') || p.startsWith('/img/')) { res.writeHead(200, { 'content-type': 'image/gif' }); return res.end(gif); }
  // Standard Ebooks' public pages (their OPDS feed is for members only)
  if (p === '/ebooks') { const q = u.searchParams.get('query') || ''; return HT(`<html><body><ol class="ebooks-list grid">${seCard('bram-stoker/dracula', 'Dracula ' + q, 'Bram Stoker')}${seCard('john-bunyan/the-pilgrims-progress', 'The Pilgrim&#8217;s Progress', 'John Bunyan')}</ol><nav><a href="/ebooks?page=2">2</a></nav></body></html>`); }
  if (p === '/ebooks/bram-stoker/dracula') return HT(`<a href="/ebooks/bram-stoker/dracula/downloads/bram-stoker_dracula.kepub.epub">k</a><a href="/ebooks/bram-stoker/dracula/downloads/bram-stoker_dracula_advanced.epub">a</a><a href="/ebooks/bram-stoker/dracula/downloads/bram-stoker_dracula.epub?source=download">e</a>`);
  if (p === '/ebooks/bram-stoker/dracula/downloads/bram-stoker_dracula.epub') { if (u.searchParams.get('source') !== 'download') { res.writeHead(200, { 'content-type': 'text/html' }); return res.end('<html>Your Download Has Started!</html>'); } res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(Buffer.concat([epub, Buffer.from('se')])); }
  if (p === '/feeds/opds') { res.writeHead(401); return res.end('members only'); }
  // the Internet Archive
  if (p === '/ia/advancedsearch.php') return J({ response: { numFound: 1, docs: [{ identifier: 'moby1', title: 'Moby', creator: ['Melville'] }, { identifier: '../bad', title: 'x' }] } });
  if (p === '/ia/metadata/moby1') return J({ metadata: {}, files: [{ name: 'moby1.pdf', format: 'Text PDF' }, { name: 'moby1.epub', format: 'EPUB' }] });
  if (p === '/ia/metadata/locked') return J({ metadata: { 'access-restricted-item': 'true' }, files: [{ name: 'x.epub', format: 'EPUB' }] });
  if (p === '/ia/download/moby1/moby1.epub') { res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(Buffer.concat([epub, Buffer.from('ia')])); }
  // Open Library (search only; the book is an Internet Archive item)
  if (p === '/ol/search.json') return J({ numFound: 2, docs: [{ title: 'Moby Dick', author_name: ['Melville'], cover_i: 7, ia: ['moby1'], ebook_access: 'public' }, { title: 'Borrowable', ia: ['locked'], ebook_access: 'borrowable' }] });
  // Smashwords: its shelf feed (filtered), a sign-in form, and downloads that need the signed-in cookie
  if (p === '/sw/signin') { res.writeHead(200, { 'content-type': 'text/html', 'set-cookie': 'XSRF-TOKEN=deleted; Path=/' }); return res.end('<form action="/sw/auth/index" method="post" id="loginForm"><input type="hidden" name="secToken" value="" /><input type="hidden" name="loginToken" value="tok123" /></form>'); }
  if (p === '/sw/auth/index' && req.method === 'POST') { let b = ''; req.on('data', c => b += c); req.on('end', () => { const f = new URLSearchParams(b); swPosts.push(Object.fromEntries(f)); if (f.get('email') === 'me@x.org' && f.get('password') === 'pw' && f.get('loginToken') === 'tok123') { res.writeHead(302, { location: '/sw/', 'set-cookie': 'sess=abc; Path=/; HttpOnly' }); } else res.writeHead(200); res.end(); }); return; }
  if (p === '/sw/') { res.writeHead(200, { 'content-type': 'text/html' }); return res.end(`<script>window.angularData.user.loggedIn = ${/sess=abc/.test(req.headers.cookie || '')};</script>`); }
  if (p === '/sw/books/download/101/8/latest/0/0/book.epub') { if (!/sess=abc/.test(req.headers.cookie || '')) { res.writeHead(302, { location: '/sw/signup/login/books/101' }); return res.end(); } res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(epub); }
  if (p === '/sw/signup/login/books/101') { res.writeHead(200); return res.end('login'); }
  if (p.startsWith('/sw/api/bookshelves/category/v3/shelf/')) { const pg = +p.split('/').slice(-2)[0]; // (.../{page}/{pageSize})
    const bk = (id, o = {}) => ({ book: { id: String(id), title: 'Book ' + id, price: { type: 'FIXED_PRICE', prices: [{ amount: '0.00' }] }, short_description: 'A fine tale.', cover_url: 'bookCovers/c' + id + '.gif', adult: false, erotica: false, taboo_erotica: false, need_erotic_cert: false, categories: [{ id: '1', name: 'Fiction', parents: [] }], tags: ['adventure'], formats: ['EPUB'], contributors: [{ role: 'A01', account: { display_name: 'Ann Author' } }], ...o } });
    const list = pg === 1 ? [bk(101, { title: 'Pilgrim Story' }), bk(102, { categories: [{ id: '5', name: 'Romance', parents: [{ id: '886', name: 'LGBTQ+' }] }] }), bk(103, { erotica: true }), bk(104, { tags: ['gay romance'] }), bk(105, { title: 'Gaylord\'s Garden', contributors: [{ role: 'A01', account: { display_name: 'Gay Talese' } }] }), bk(106, { short_description: 'A steamy love story' }), bk(107, { price: { type: 'FIXED_PRICE', prices: [{ amount: '2.99' }] } }), bk(109, { categories: [{ id: '1235', name: 'Romance', parents: [{ id: '3', name: 'Fiction' }] }], tags: ['billionaire romance'] }), bk(110, { title: 'Sweet Home', categories: [{ id: '1235', name: 'Romance', parents: [{ id: '3', name: 'Fiction' }] }], tags: ['sweet romance'] }), bk(108, { categories: [{ id: '886', name: 'LGBTQ+', parents: [{ id: '3', name: 'Fiction' }] }] })] : [];
    res.writeHead(200, { 'content-type': 'application/json' }); return res.end(JSON.stringify({ name: 'Most Downloaded', type: 'mostdownloads', book_list: list, total_books: 8 })); }
  if (p.startsWith('/swc/')) { res.writeHead(200, { 'content-type': 'image/gif' }); return res.end(gif); }
  // an OPDS catalogue that needs a sign-in (Calibre-Web style)
  if (p.startsWith('/cal/')) {
    if (req.headers.authorization !== 'Basic ' + Buffer.from('reader:pw').toString('base64')) { res.writeHead(401); return res.end(); }
    if (p === '/cal/opds') return X(feed('', '<link rel="search" href="/cal/opds/osd" type="application/opensearchdescription+xml"/>'));
    if (p === '/cal/opds/osd') return X('<OpenSearchDescription><Url type="application/atom+xml" template="/cal/opds/search/{searchTerms}"/></OpenSearchDescription>');
    if (p.startsWith('/cal/opds/search/')) return X(feed(acq('c1', 'Own Book ' + decodeURIComponent(p.split('/').pop()), 'Me', '/cal/get/1.pdf', 'application/pdf')));
    if (p === '/cal/get/1.pdf') { res.writeHead(200, { 'content-type': 'application/pdf' }); return res.end(pdf); }
  }
  res.writeHead(404); res.end('no');
}).listen(0, async () => {
  port = fake.address().port; const B = 'http://127.0.0.1:' + port;
  Object.assign(process.env, { GUTENBERG_URL: B, ARCHIVE_URL: B + '/ia', OPENLIBRARY_URL: B + '/ol', SMASHWORDS_URL: B + '/sw', SMASHWORDS_COVERS: B + '/swc', STANDARD_EBOOKS_URL: B });
  const books = require('../books'), users = require('../users');
  const people = { a: users.create({ name: 'Ann', role: 'admin', handle: 'YQ' }), b: users.create({ name: 'Bob', role: 'user', tabs: ['read'], handle: 'Yg' }) };
  const hub = http.createServer((req, res) => users.runAs(req.headers['x-user'] === 'b' ? people.b : people.a, () => books.handle(req, res))).listen(0, async () => {
    const base = 'http://127.0.0.1:' + hub.address().port + '/api/read/store', call = (p, o = {}) => fetch(base + p, o), post = (p, b, h = {}) => call(p, { method: 'POST', body: JSON.stringify(b), headers: h });
    try {
      let r = await call('/sources'); assert.deepEqual((await r.json()).map(s => s.id), ['gutenberg', 'standardebooks', 'archive', 'smashwords', 'openlibrary']);
      r = await call('/search?q=alice'); let j = await r.json();
      assert.deepEqual(j.items.map(i => i.source), ['gutenberg', 'standardebooks', 'archive', 'openlibrary', 'gutenberg', 'standardebooks'].slice(0, j.items.length), 'one result from each catalogue in turn'); assert.deepEqual(j.errors, {});
      const g = j.items.filter(i => i.source === 'gutenberg'); assert.deepEqual(g.map(i => i.ref), ['11', '345'], 'catalogue entries that are not books, and repeats, are left out'); assert.equal(g[0].author, 'Lewis Carroll'); assert.equal(g[1].title, 'Dracula & Co'); assert.match(g[0].cover, /^\/api\/read\/store\/cover\?s=gutenberg/); assert.equal(j.more, true);
      const se = j.items.filter(i => i.source === 'standardebooks'); assert.equal(se[0].title, 'Dracula alice'); assert.equal(se[0].author, 'Bram Stoker'); assert.equal(se[1].title, 'The Pilgrim’s Progress'); assert.equal(se[0].ref, '/ebooks/bram-stoker/dracula');
      assert.ok(!j.items.some(i => i.ref === '../bad'), 'odd identifiers are dropped'); assert.ok(!seen.some(x => x.startsWith('/feeds/opds')), 'the members-only feed is never asked for');
      r = await call('/search?q=&source=gutenberg'); j = await r.json(); assert.ok(seen.some(x => x.includes('sort_order=downloads')), 'no search words: the most downloaded books'); assert.equal(j.items.length, 2);
      r = await call('/search?q=x&source=gutenberg&page=2'); assert.ok(seen.some(x => x.includes('start_index=26')), 'later pages ask for the next 25'); assert.equal((await r.json()).more, false);
      r = await call('/search?q=x&source=nope'); j = await r.json(); assert.ok(j.errors.nope, 'a broken catalogue is reported and does not stop the rest');
      r = await call('/cover?s=gutenberg&u=' + encodeURIComponent(B + '/img/11.gif')); assert.equal(r.headers.get('content-type'), 'image/gif');
      assert.equal((await call('/cover?s=gutenberg&u=' + encodeURIComponent('http://localhost:' + port + '/img/11.gif'))).status, 502, 'covers only come from the catalogue\'s own site');
      // adding
      r = await post('/add', { source: 'gutenberg', ref: '11', title: 'Alice', author: 'Lewis Carroll', cover: B + '/cache/epub/11/pg11.cover.medium.jpg' }); let book = await r.json(); assert.equal(r.status, 200); assert.equal(book.ext, 'epub'); assert.equal(book.author, 'Lewis Carroll'); assert.equal(book.cover, true);
      assert.ok(seen.includes('/ebooks/11.epub3.images'), 'the EPUB with pictures is fetched');
      r = await post('/add', { source: 'gutenberg', ref: '11', title: 'Alice' }); assert.equal((await r.json()).duplicate, true, 'adding the same book twice does not store it twice');
      assert.equal((await (await fetch(base.replace('/store', '/list'))).json()).length, 1);
      assert.equal((await post('/add', { source: 'gutenberg', ref: '11' }, { 'x-user': 'b' })).status, 200); assert.equal((await (await fetch(base.replace('/store', '/list'), { headers: { 'x-user': 'b' } })).json()).length, 1, "each person's shelf is their own");
      r = await post('/add', { source: 'standardebooks', ref: '/ebooks/bram-stoker/dracula', title: 'Dracula', author: 'Bram Stoker' }); assert.equal(r.status, 200, 'the real EPUB is fetched, not the "download has started" page'); assert.ok(seen.some(x => x.includes('bram-stoker_dracula.epub?source=download')) && !seen.some(x => x.includes('kepub') && x.includes('source')), 'the plain EPUB (not the Kobo or advanced one) is chosen');
      r = await post('/add', { source: 'standardebooks', ref: '/ebooks/../etc/passwd' }); assert.equal(r.status, 400);
      r = await post('/add', { source: 'archive', ref: 'moby1', title: 'Moby', author: 'Melville' }); assert.equal((await r.json()).title, 'Moby'); assert.ok(seen.includes('/ia/download/moby1/moby1.epub'), 'the EPUB is chosen over the PDF');
      const ol = (await (await call('/search?q=moby&source=openlibrary')).json()).items; assert.deepEqual(ol.map(i => i.ref), ['moby1'], 'only freely readable editions are listed'); assert.equal(ol[0].author, 'Melville'); assert.match(decodeURIComponent(ol[0].coverUrl), /covers\.openlibrary\.org\/b\/id\/7-M\.jpg/);
      r = await post('/add', { source: 'openlibrary', ref: 'moby1', title: 'Moby Dick' }); assert.equal(r.status, 200, 'an Open Library book is fetched from the Internet Archive');
      // Smashwords: the family filter, the sign-in, and the download
      r = await call('/search?q=&source=smashwords'); j = await r.json(); assert.deepEqual(j.items.map(i => i.ref), ['101', '105', '110'], 'adult, LGBTQ+, keyword-flagged, unlabelled romance and paid books are left out; a clean romance and a harmless name are kept'); assert.equal(j.items[0].author, 'Ann Author'); assert.match(j.items[0].cover, /^\/api\/read\/store\/cover\?s=smashwords/);
      j = await (await call('/search?q=pilgrim&source=smashwords')).json(); assert.deepEqual(j.items.map(i => i.ref), ['101'], 'searching looks through the free books it can see'); assert.deepEqual((await (await call('/search?q=gay&source=smashwords')).json()).items.map(i => i.ref), ['105']);
      j = await (await call('/sources')).json(); assert.deepEqual(j.find(s => s.id === 'smashwords').account, { signedIn: false, email: '' });
      r = await post('/add', { source: 'smashwords', ref: '101', title: 'Pilgrim Story' }); assert.equal(r.status, 401); assert.match((await r.json()).error, /sign-in/, 'downloading needs the account');
      assert.equal((await post('/smashwords', { email: 'me@x.org', pass: 'pw' }, { 'x-user': 'b' })).status, 403, 'only an admin saves the sign-in');
      r = await post('/smashwords', { email: 'me@x.org', pass: 'wrong' }); assert.equal(r.status, 401); assert.equal((await (await call('/sources')).json()).find(s => s.id === 'smashwords').account.signedIn, false, 'a wrong password is not kept');
      r = await post('/smashwords', { email: 'me@x.org', pass: 'pw' }); assert.equal(r.status, 200); assert.deepEqual(await r.json(), { signedIn: true, email: 'me@x.org' }); assert.equal(swPosts.at(-1).loginToken, 'tok123', 'the sign-in form\'s own token is sent back');
      assert.ok(!fs.readFileSync(path.join(process.env.DATA_DIR, 'store-smashwords.json'), 'utf8').includes('"pw"') && !JSON.stringify(await (await call('/sources')).json()).includes('"pw"'), 'the password is encrypted on disk and never sent to the browser');
      r = await post('/add', { source: 'smashwords', ref: '101', title: 'Pilgrim Story', author: 'Ann Author', cover: B + '/swc/c101.gif' }); assert.equal(r.status, 200, 'the signed-in download is accepted'); assert.ok(seen.includes('/sw/books/download/101/8/latest/0/0/book.epub'), 'the EPUB is fetched with the signed-in cookie');
      assert.equal((await post('/add', { source: 'smashwords', ref: '../x' })).status, 400);
      assert.equal((await call('/smashwords', { method: 'DELETE' })).status, 200); assert.equal((await post('/add', { source: 'smashwords', ref: '101' })).status, 401, 'removing the sign-in stops downloads');
      r = await post('/add', { source: 'archive', ref: 'locked' }); assert.equal(r.status, 403); assert.match((await r.json()).error, /DRM/);
      r = await post('/add', { source: 'gutenberg', ref: '../x' }); assert.equal(r.status, 400);
      // custom catalogue
      assert.equal((await post('/sources', { name: 'Mine', url: B + '/cal/opds' }, { 'x-user': 'b' })).status, 403, 'only an admin adds catalogues');
      assert.equal((await post('/sources', { name: 'Bad', url: B + '/cal/opds', user: 'reader', pass: 'wrong' })).status, 502, 'a catalogue that refuses the sign-in is not added');
      assert.equal((await call('/sources').then(x => x.json())).length, 5);
      r = await post('/sources', { name: 'Mine', url: B + '/cal/opds', user: 'reader', pass: 'pw' }); const mine = await r.json(); assert.equal(r.status, 200);
      assert.ok(!JSON.stringify(await (await call('/sources')).json()).includes('pw'), 'sign-ins are never sent to the browser'); assert.ok(!fs.readFileSync(path.join(process.env.DATA_DIR, 'store-sources.json'), 'utf8').includes('"pw"'), 'and are encrypted on disk');
      j = await (await call('/search?q=hello%20world&source=' + mine.id)).json(); assert.equal(j.items[0].title, 'Own Book hello world', 'search found through the OpenSearch description'); assert.equal(j.items[0].format, 'pdf');
      r = await post('/add', { source: mine.id, ref: j.items[0].ref, title: j.items[0].title, format: 'pdf' }, { 'x-user': 'b' }); assert.equal((await r.json()).ext, 'pdf');
      assert.equal((await call('/sources/' + mine.id, { method: 'DELETE', headers: { 'x-user': 'b' } })).status, 403); assert.equal((await call('/sources/' + mine.id, { method: 'DELETE' })).status, 200);
      console.log('store test ok');
    } catch (err) { console.error(err); process.exitCode = 1; }
    hub.close(); fake.close();
  });
});