Books: book store with Project Gutenberg, Standard Ebooks, Internet Archive and OPDS catalogues

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UaC9HmtHMdRP8dAbN4wAKj

tools/store-test.js

@@ -0,0 +1,76 @@
+// Book store: search across Gutenberg, Standard Ebooks, the Internet Archive and an OPDS catalogue; adding a book puts it on the shelf; sites only reach their own addresses.
+const http = require('http'), assert = require('assert'), fs = require('fs'), os = require('os'), path = require('path'), cp = require('child_process');
+process.env.DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'nehub-store-'));
+const fx = fs.mkdtempSync(path.join(os.tmpdir(), 'nehub-storefx-')); cp.execFileSync('python3', [path.join(__dirname, 'mkbooks.py'), fx]);
+const epub = fs.readFileSync(path.join(fx, 'test.epub')), pdf = fs.readFileSync(path.join(fx, 'test.pdf')), gif = Buffer.from('R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==', 'base64');
+const seen = []; let port;
+const feed = (body, links = '') => `<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom">${links}${body}</feed>`;
+const acq = (id, title, author, href, type = 'application/epub+zip') => `<entry><title>${title}</title><author><name>${author}</name></author><id>${id}</id><summary>A &amp; B</summary><link rel="http://opds-spec.org/image" href="/img/${id}.gif" type="image/gif"/><link rel="http://opds-spec.org/acquisition/open-access" href="${href}" type="${type}"/></entry>`;
+const fake = http.createServer((req, res) => {
+  const u = new URL(req.url, 'http://x'), p = u.pathname; seen.push(req.method + ' ' + p + u.search + (req.headers.authorization ? ' [auth]' : ''));
+  const J = o => { res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify(o)); }, X = x => { res.writeHead(200, { 'content-type': 'application/atom+xml' }); res.end(x); };
+  if (p === '/gut/books/' ) return J({ next: u.searchParams.get('page') === '1' ? 'x' : null, results: [{ id: 11, title: 'Alice', authors: [{ name: 'Carroll, Lewis' }], subjects: ['Fantasy'], formats: { 'application/epub+zip': `http://127.0.0.1:${port}/gut/file/11.epub`, 'image/jpeg': `http://127.0.0.1:${port}/img/11.gif` } }, { id: 12, title: 'No epub', authors: [], formats: {} }] });
+  if (p === '/gut/books/11') return J({ id: 11, formats: { 'application/epub+zip': `http://127.0.0.1:${port}/gut/file/11.epub` } });
+  if (p === '/gut/file/11.epub') { res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(epub); }
+  if (p === '/gut/books/66') return J({ id: 66, formats: { 'application/epub+zip': `http://127.0.0.1:${port}/gut/evil` } });
+  if (p === '/gut/evil') { res.writeHead(302, { location: `http://localhost:${port}/secret` }); return res.end(); }
+  if (p === '/img/11.gif' || p.startsWith('/img/')) { res.writeHead(200, { 'content-type': 'image/gif' }); return res.end(gif); }
+  if (p === '/ia/advancedsearch.php') return J({ response: { numFound: 1, docs: [{ identifier: 'moby1', title: 'Moby', creator: ['Melville'] }, { identifier: '../bad', title: 'x' }] } });
+  if (p === '/ia/metadata/moby1') return J({ metadata: {}, files: [{ name: 'moby1.pdf', format: 'Text PDF' }, { name: 'moby1.epub', format: 'EPUB' }] });
+  if (p === '/ia/metadata/locked') return J({ metadata: { 'access-restricted-item': 'true' }, files: [{ name: 'x.epub', format: 'EPUB' }] });
+  if (p === '/ia/download/moby1/moby1.epub') { res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(Buffer.concat([epub, Buffer.from('ia')])); }
+  if (p === '/se/opds') return X(feed('<entry><title>New releases</title><id>n</id><link rel="http://opds-spec.org/sort/new" href="/se/opds/new" type="application/atom+xml"/></entry>', '<link rel="search" href="/se/opds/all?query={searchTerms}" type="application/atom+xml"/>'));
+  if (p === '/se/opds/all') return X(feed(acq('se1', 'Dracula ' + u.searchParams.get('query'), 'Bram Stoker', '/se/dl/dracula.epub')));
+  if (p === '/se/opds/new') return X(feed(acq('se2', 'Newest Book', 'Someone', '/se/dl/new.epub'), '<link rel="next" href="/se/opds/new2" type="application/atom+xml"/>'));
+  if (p === '/se/opds/new2') return X(feed(acq('se3', 'Page Two', 'Someone', '/se/dl/new.epub')));
+  if (p === '/se/dl/dracula.epub' || p === '/se/dl/new.epub') { res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(epub); }
+  if (p.startsWith('/cal/')) { // Calibre-Web style: needs a sign-in; search is described by an OpenSearch file
+    if (req.headers.authorization !== 'Basic ' + Buffer.from('reader:pw').toString('base64')) { res.writeHead(401); return res.end(); }
+    if (p === '/cal/opds') return X(feed('', '<link rel="search" href="/cal/opds/osd" type="application/opensearchdescription+xml"/>'));
+    if (p === '/cal/opds/osd') return X('<OpenSearchDescription><Url type="application/atom+xml" template="/cal/opds/search/{searchTerms}"/></OpenSearchDescription>');
+    if (p.startsWith('/cal/opds/search/')) return X(feed(acq('c1', 'Own Book ' + decodeURIComponent(p.split('/').pop()), 'Me', '/cal/get/1.pdf', 'application/pdf')));
+    if (p === '/cal/get/1.pdf') { res.writeHead(200, { 'content-type': 'application/pdf' }); return res.end(pdf); }
+  }
+  res.writeHead(404); res.end('no');
+}).listen(0, async () => {
+  port = fake.address().port; const B = 'http://127.0.0.1:' + port;
+  Object.assign(process.env, { GUTENDEX_URL: B + '/gut', ARCHIVE_URL: B + '/ia', STANDARD_EBOOKS_URL: B + '/se/opds' });
+  const books = require('../books'), users = require('../users');
+  const people = { a: users.create({ name: 'Ann', role: 'admin', handle: 'YQ' }), b: users.create({ name: 'Bob', role: 'user', tabs: ['read'], handle: 'Yg' }) };
+  const hub = http.createServer((req, res) => users.runAs(req.headers['x-user'] === 'b' ? people.b : people.a, () => books.handle(req, res))).listen(0, async () => {
+    const base = 'http://127.0.0.1:' + hub.address().port + '/api/read/store', call = (p, o = {}) => fetch(base + p, o), post = (p, b, h = {}) => call(p, { method: 'POST', body: JSON.stringify(b), headers: h });
+    try {
+      let r = await call('/sources'); assert.deepEqual((await r.json()).map(s => s.id), ['gutenberg', 'standardebooks', 'archive']);
+      r = await call('/search?q=alice'); let j = await r.json();
+      assert.deepEqual(j.items.map(i => i.source), ['gutenberg', 'standardebooks', 'archive'], 'one result from each catalogue in turn, books with no EPUB left out');
+      assert.equal(j.items[0].author, 'Lewis Carroll'); assert.equal(j.items[1].summary, 'A & B'); assert.match(j.items[0].cover, /^\/api\/read\/store\/cover\?s=gutenberg/); assert.equal(j.more, true);
+      assert.ok(seen.includes('GET /se/opds/all?query=alice'), 'the search link in the catalogue is used'); assert.ok(!j.items.some(i => i.ref === '../bad'), 'odd identifiers are dropped');
+      r = await call('/search?q=&source=standardebooks&page=2'); j = await r.json(); assert.deepEqual(j.items.map(i => i.title), ['Page Two'], 'no search words: the newest list, and its next pages');
+      r = await call('/search?q=x&source=nope'); j = await r.json(); assert.ok(j.errors.nope, 'a broken catalogue is reported and does not stop the rest');
+      r = await call(j.items ? '/cover?s=gutenberg&u=' + encodeURIComponent(B + '/img/11.gif') : ''); assert.equal(r.headers.get('content-type'), 'image/gif');
+      assert.equal((await call('/cover?s=gutenberg&u=' + encodeURIComponent('http://localhost:' + port + '/img/11.gif'))).status, 502, 'covers only come from the catalogue\'s own site');
+      // adding
+      r = await post('/add', { source: 'gutenberg', ref: '11', title: 'Alice', author: 'Lewis Carroll', cover: B + '/img/11.gif' }); let book = await r.json(); assert.equal(r.status, 200); assert.equal(book.ext, 'epub'); assert.equal(book.author, 'Lewis Carroll'); assert.equal(book.cover, true);
+      r = await post('/add', { source: 'gutenberg', ref: '11', title: 'Alice' }); assert.equal((await r.json()).duplicate, true, 'adding the same book twice does not store it twice');
+      assert.equal((await (await fetch(base.replace('/store', '/list'))).json()).length, 1);
+      assert.equal((await post('/add', { source: 'gutenberg', ref: '11' }, { 'x-user': 'b' })).status, 200); assert.equal((await (await fetch(base.replace('/store', '/list'), { headers: { 'x-user': 'b' } })).json()).length, 1, "each person's shelf is their own");
+      r = await post('/add', { source: 'archive', ref: 'moby1', title: 'Moby', author: 'Melville' }); assert.equal((await r.json()).title, 'Moby'); assert.ok(seen.includes('GET /ia/download/moby1/moby1.epub'), 'the EPUB is chosen over the PDF');
+      r = await post('/add', { source: 'archive', ref: 'locked' }); assert.equal(r.status, 403); assert.match((await r.json()).error, /DRM/);
+      r = await post('/add', { source: 'gutenberg', ref: '66' }); assert.equal(r.status, 502, 'a download that redirects off the site is refused'); assert.ok(!seen.some(x => x.includes('/secret')));
+      r = await post('/add', { source: 'gutenberg', ref: '../x' }); assert.equal(r.status, 400);
+      r = await post('/add', { source: 'standardebooks', ref: 'http://localhost:' + port + '/se/dl/dracula.epub' }); assert.equal(r.status, 400, 'a catalogue book must live on the catalogue\'s own address');
+      r = await post('/add', { source: 'standardebooks', ref: B + '/se/dl/dracula.epub', title: 'Dracula', format: 'epub' }); assert.equal(r.status, 200);
+      // custom catalogue
+      assert.equal((await post('/sources', { name: 'Mine', url: B + '/cal/opds' }, { 'x-user': 'b' })).status, 403, 'only an admin adds catalogues');
+      assert.equal((await post('/sources', { name: 'Bad', url: B + '/cal/opds', user: 'reader', pass: 'wrong' })).status, 502, 'a catalogue that refuses the sign-in is not added');
+      assert.equal((await call('/sources').then(x => x.json())).length, 3);
+      r = await post('/sources', { name: 'Mine', url: B + '/cal/opds', user: 'reader', pass: 'pw' }); const mine = await r.json(); assert.equal(r.status, 200);
+      assert.ok(!JSON.stringify(await (await call('/sources')).json()).includes('pw'), 'sign-ins are never sent to the browser'); assert.ok(!fs.readFileSync(path.join(process.env.DATA_DIR, 'store-sources.json'), 'utf8').includes('"pw"'), 'and are encrypted on disk');
+      j = await (await call('/search?q=hello%20world&source=' + mine.id)).json(); assert.equal(j.items[0].title, 'Own Book hello world', 'search found through the OpenSearch description'); assert.equal(j.items[0].format, 'pdf');
+      r = await post('/add', { source: mine.id, ref: j.items[0].ref, title: j.items[0].title, format: 'pdf' }, { 'x-user': 'b' }); assert.equal((await r.json()).ext, 'pdf');
+      assert.equal((await call('/sources/' + mine.id, { method: 'DELETE', headers: { 'x-user': 'b' } })).status, 403); assert.equal((await call('/sources/' + mine.id, { method: 'DELETE' })).status, 200);
+      console.log('store test ok');
+    } catch (err) { console.error(err); process.exitCode = 1; }
+    hub.close(); fake.close();
+  });
+});