nate/hub
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Book store: Project Gutenberg through its own catalogue feed (Gutendex timed out), Standard Ebooks through its public pages (its OPDS feed needs a paid sign-in)

Browse Source 
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UaC9HmtHMdRP8dAbN4wAKj
This commit is contained in:
Claude committed 2026-10-25 03:11:35 +00:00
1 parent 047ac32b19
commit 239730fd3b
4 files changed
+62 -39
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/store-test.js
+30 -25
View File
@@ -6,25 +6,28 @@ const epub = fs.readFileSync(path.join(fx, 'test.epub')), pdf = fs.readFileSync(
const seen = []; let port;
const feed = (body, links = '') => `<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom">${links}${body}</feed>`;
const acq = (id, title, author, href, type = 'application/epub+zip') => `<entry><title>${title}</title><author><name>${author}</name></author><id>${id}</id><summary>A &amp; B</summary><link rel="http://opds-spec.org/image" href="/img/${id}.gif" type="image/gif"/><link rel="http://opds-spec.org/acquisition/open-access" href="${href}" type="${type}"/></entry>`;
const seCard = (slug, title, who) => `<li typeof="schema:Book" about="/ebooks/${slug}"><div><a href="/ebooks/${slug}"><picture><img src="/img/${slug.replace(/\W/g, '')}.gif" alt="" property="schema:image"/></picture></a></div><p><a href="/ebooks/${slug}" property="schema:url"><span property="schema:name">${title}</span></a></p><p class="author" typeof="schema:Person"><a href="https://x/y" property="schema:url"><span property="schema:name">${who}</span></a></p></li>`;
const fake = http.createServer((req, res) => {
const u = new URL(req.url, 'http://x'), p = u.pathname; seen.push(req.method + ' ' + p + u.search + (req.headers.authorization ? ' [auth]' : ''));
const J = o => { res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify(o)); }, X = x => { res.writeHead(200, { 'content-type': 'application/atom+xml' }); res.end(x); };
if (p === '/gut/books/' ) return J({ next: u.searchParams.get('page') === '1' ? 'x' : null, results: [{ id: 11, title: 'Alice', authors: [{ name: 'Carroll, Lewis' }], subjects: ['Fantasy'], formats: { 'application/epub+zip': `http://127.0.0.1:${port}/gut/file/11.epub`, 'image/jpeg': `http://127.0.0.1:${port}/img/11.gif` } }, { id: 12, title: 'No epub', authors: [], formats: {} }] });
if (p === '/gut/books/11') return J({ id: 11, formats: { 'application/epub+zip': `http://127.0.0.1:${port}/gut/file/11.epub` } });
if (p === '/gut/file/11.epub') { res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(epub); }
if (p === '/gut/books/66') return J({ id: 66, formats: { 'application/epub+zip': `http://127.0.0.1:${port}/gut/evil` } });
if (p === '/gut/evil') { res.writeHead(302, { location: `http://localhost:${port}/secret` }); return res.end(); }
if (p === '/img/11.gif' || p.startsWith('/img/')) { res.writeHead(200, { 'content-type': 'image/gif' }); return res.end(gif); }
const u = new URL(req.url, 'http://x'), p = u.pathname; seen.push(p + u.search);
const J = o => { res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify(o)); }, X = x => { res.writeHead(200, { 'content-type': 'application/atom+xml' }); res.end(x); }, HT = x => { res.writeHead(200, { 'content-type': 'application/xhtml+xml' }); res.end(x); };
// Project Gutenberg's catalogue feed
if (p === '/ebooks/search.opds/') { const more = u.searchParams.get('start_index') ? '' : '<link rel="next" type="application/atom+xml" href="/ebooks/search.opds/?query=x&amp;start_index=26"/>';
return X(feed(`<entry><title>Authors</title><id>https://x/ebooks/author/1.opds</id></entry><entry><title>Alice in Wonderland</title><id>https://www.gutenberg.org/ebooks/11.opds</id><content type="text">Lewis Carroll</content></entry><entry><title>Alice in Wonderland</title><id>https://www.gutenberg.org/ebooks/11.opds</id><content>Lewis Carroll</content></entry><entry><title>Dracula &amp; Co</title><id>https://www.gutenberg.org/ebooks/345.opds</id><content>Bram Stoker</content></entry>`, more)); }
if (p === '/ebooks/11.epub3.images') { res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(epub); }
if (p === '/ebooks/66.epub3.images') { res.writeHead(302, { location: `http://localhost:${port}/secret` }); return res.end(); }
if (p.startsWith('/cache/epub/') || p.startsWith('/img/')) { res.writeHead(200, { 'content-type': 'image/gif' }); return res.end(gif); }
// Standard Ebooks' public pages (their OPDS feed is for members only)
if (p === '/ebooks') { const q = u.searchParams.get('query') || ''; return HT(`<html><body><ol class="ebooks-list grid">${seCard('bram-stoker/dracula', 'Dracula ' + q, 'Bram Stoker')}${seCard('john-bunyan/the-pilgrims-progress', 'The Pilgrim&#8217;s Progress', 'John Bunyan')}</ol><nav><a href="/ebooks?page=2">2</a></nav></body></html>`); }
if (p === '/ebooks/bram-stoker/dracula') return HT(`<a href="/ebooks/bram-stoker/dracula/downloads/bram-stoker_dracula.kepub.epub">k</a><a href="/ebooks/bram-stoker/dracula/downloads/bram-stoker_dracula_advanced.epub">a</a><a href="/ebooks/bram-stoker/dracula/downloads/bram-stoker_dracula.epub?source=download">e</a>`);
if (p === '/ebooks/bram-stoker/dracula/downloads/bram-stoker_dracula.epub') { if (u.searchParams.get('source') !== 'download') { res.writeHead(200, { 'content-type': 'text/html' }); return res.end('<html>Your Download Has Started!</html>'); } res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(Buffer.concat([epub, Buffer.from('se')])); }
if (p === '/feeds/opds') { res.writeHead(401); return res.end('members only'); }
// the Internet Archive
if (p === '/ia/advancedsearch.php') return J({ response: { numFound: 1, docs: [{ identifier: 'moby1', title: 'Moby', creator: ['Melville'] }, { identifier: '../bad', title: 'x' }] } });
if (p === '/ia/metadata/moby1') return J({ metadata: {}, files: [{ name: 'moby1.pdf', format: 'Text PDF' }, { name: 'moby1.epub', format: 'EPUB' }] });
if (p === '/ia/metadata/locked') return J({ metadata: { 'access-restricted-item': 'true' }, files: [{ name: 'x.epub', format: 'EPUB' }] });
if (p === '/ia/download/moby1/moby1.epub') { res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(Buffer.concat([epub, Buffer.from('ia')])); }
if (p === '/se/opds') return X(feed('<entry><title>New releases</title><id>n</id><link rel="http://opds-spec.org/sort/new" href="/se/opds/new" type="application/atom+xml"/></entry>', '<link rel="search" href="/se/opds/all?query={searchTerms}" type="application/atom+xml"/>'));
if (p === '/se/opds/all') return X(feed(acq('se1', 'Dracula ' + u.searchParams.get('query'), 'Bram Stoker', '/se/dl/dracula.epub')));
if (p === '/se/opds/new') return X(feed(acq('se2', 'Newest Book', 'Someone', '/se/dl/new.epub'), '<link rel="next" href="/se/opds/new2" type="application/atom+xml"/>'));
if (p === '/se/opds/new2') return X(feed(acq('se3', 'Page Two', 'Someone', '/se/dl/new.epub')));
if (p === '/se/dl/dracula.epub' || p === '/se/dl/new.epub') { res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(epub); }
if (p.startsWith('/cal/')) { // Calibre-Web style: needs a sign-in; search is described by an OpenSearch file
// an OPDS catalogue that needs a sign-in (Calibre-Web style)
if (p.startsWith('/cal/')) {
if (req.headers.authorization !== 'Basic ' + Buffer.from('reader:pw').toString('base64')) { res.writeHead(401); return res.end(); }
if (p === '/cal/opds') return X(feed('', '<link rel="search" href="/cal/opds/osd" type="application/opensearchdescription+xml"/>'));
if (p === '/cal/opds/osd') return X('<OpenSearchDescription><Url type="application/atom+xml" template="/cal/opds/search/{searchTerms}"/></OpenSearchDescription>');
@@ -34,7 +37,7 @@ const fake = http.createServer((req, res) => {
res.writeHead(404); res.end('no');
}).listen(0, async () => {
port = fake.address().port; const B = 'http://127.0.0.1:' + port;
Object.assign(process.env, { GUTENDEX_URL: B + '/gut', ARCHIVE_URL: B + '/ia', STANDARD_EBOOKS_URL: B + '/se/opds' });
Object.assign(process.env, { GUTENBERG_URL: B, ARCHIVE_URL: B + '/ia', STANDARD_EBOOKS_URL: B });
const books = require('../books'), users = require('../users');
const people = { a: users.create({ name: 'Ann', role: 'admin', handle: 'YQ' }), b: users.create({ name: 'Bob', role: 'user', tabs: ['read'], handle: 'Yg' }) };
const hub = http.createServer((req, res) => users.runAs(req.headers['x-user'] === 'b' ? people.b : people.a, () => books.handle(req, res))).listen(0, async () => {
@@ -42,24 +45,26 @@ const fake = http.createServer((req, res) => {
try {
let r = await call('/sources'); assert.deepEqual((await r.json()).map(s => s.id), ['gutenberg', 'standardebooks', 'archive']);
r = await call('/search?q=alice'); let j = await r.json();
assert.deepEqual(j.items.map(i => i.source), ['gutenberg', 'standardebooks', 'archive'], 'one result from each catalogue in turn, books with no EPUB left out');
assert.equal(j.items[0].author, 'Lewis Carroll'); assert.equal(j.items[1].summary, 'A & B'); assert.match(j.items[0].cover, /^\/api\/read\/store\/cover\?s=gutenberg/); assert.equal(j.more, true);
assert.ok(seen.includes('GET /se/opds/all?query=alice'), 'the search link in the catalogue is used'); assert.ok(!j.items.some(i => i.ref === '../bad'), 'odd identifiers are dropped');
r = await call('/search?q=&source=standardebooks&page=2'); j = await r.json(); assert.deepEqual(j.items.map(i => i.title), ['Page Two'], 'no search words: the newest list, and its next pages');
assert.deepEqual(j.items.map(i => i.source), ['gutenberg', 'standardebooks', 'archive', 'gutenberg', 'standardebooks'].slice(0, j.items.length), 'one result from each catalogue in turn'); assert.deepEqual(j.errors, {});
const g = j.items.filter(i => i.source === 'gutenberg'); assert.deepEqual(g.map(i => i.ref), ['11', '345'], 'catalogue entries that are not books, and repeats, are left out'); assert.equal(g[0].author, 'Lewis Carroll'); assert.equal(g[1].title, 'Dracula & Co'); assert.match(g[0].cover, /^\/api\/read\/store\/cover\?s=gutenberg/); assert.equal(j.more, true);
const se = j.items.filter(i => i.source === 'standardebooks'); assert.equal(se[0].title, 'Dracula alice'); assert.equal(se[0].author, 'Bram Stoker'); assert.equal(se[1].title, 'The Pilgrim’s Progress'); assert.equal(se[0].ref, '/ebooks/bram-stoker/dracula');
assert.ok(!j.items.some(i => i.ref === '../bad'), 'odd identifiers are dropped'); assert.ok(!seen.some(x => x.startsWith('/feeds/opds')), 'the members-only feed is never asked for');
r = await call('/search?q=&source=gutenberg'); j = await r.json(); assert.ok(seen.some(x => x.includes('sort_order=downloads')), 'no search words: the most downloaded books'); assert.equal(j.items.length, 2);
r = await call('/search?q=x&source=gutenberg&page=2'); assert.ok(seen.some(x => x.includes('start_index=26')), 'later pages ask for the next 25'); assert.equal((await r.json()).more, false);
r = await call('/search?q=x&source=nope'); j = await r.json(); assert.ok(j.errors.nope, 'a broken catalogue is reported and does not stop the rest');
r = await call(j.items ? '/cover?s=gutenberg&u=' + encodeURIComponent(B + '/img/11.gif') : ''); assert.equal(r.headers.get('content-type'), 'image/gif');
r = await call('/cover?s=gutenberg&u=' + encodeURIComponent(B + '/img/11.gif')); assert.equal(r.headers.get('content-type'), 'image/gif');
assert.equal((await call('/cover?s=gutenberg&u=' + encodeURIComponent('http://localhost:' + port + '/img/11.gif'))).status, 502, 'covers only come from the catalogue\'s own site');
// adding
r = await post('/add', { source: 'gutenberg', ref: '11', title: 'Alice', author: 'Lewis Carroll', cover: B + '/img/11.gif' }); let book = await r.json(); assert.equal(r.status, 200); assert.equal(book.ext, 'epub'); assert.equal(book.author, 'Lewis Carroll'); assert.equal(book.cover, true);
r = await post('/add', { source: 'gutenberg', ref: '11', title: 'Alice', author: 'Lewis Carroll', cover: B + '/cache/epub/11/pg11.cover.medium.jpg' }); let book = await r.json(); assert.equal(r.status, 200); assert.equal(book.ext, 'epub'); assert.equal(book.author, 'Lewis Carroll'); assert.equal(book.cover, true);
assert.ok(seen.includes('/ebooks/11.epub3.images'), 'the EPUB with pictures is fetched');
r = await post('/add', { source: 'gutenberg', ref: '11', title: 'Alice' }); assert.equal((await r.json()).duplicate, true, 'adding the same book twice does not store it twice');
assert.equal((await (await fetch(base.replace('/store', '/list'))).json()).length, 1);
assert.equal((await post('/add', { source: 'gutenberg', ref: '11' }, { 'x-user': 'b' })).status, 200); assert.equal((await (await fetch(base.replace('/store', '/list'), { headers: { 'x-user': 'b' } })).json()).length, 1, "each person's shelf is their own");
r = await post('/add', { source: 'archive', ref: 'moby1', title: 'Moby', author: 'Melville' }); assert.equal((await r.json()).title, 'Moby'); assert.ok(seen.includes('GET /ia/download/moby1/moby1.epub'), 'the EPUB is chosen over the PDF');
r = await post('/add', { source: 'standardebooks', ref: '/ebooks/bram-stoker/dracula', title: 'Dracula', author: 'Bram Stoker' }); assert.equal(r.status, 200, 'the real EPUB is fetched, not the "download has started" page'); assert.ok(seen.some(x => x.includes('bram-stoker_dracula.epub?source=download')) && !seen.some(x => x.includes('kepub') && x.includes('source')), 'the plain EPUB (not the Kobo or advanced one) is chosen');
r = await post('/add', { source: 'standardebooks', ref: '/ebooks/../etc/passwd' }); assert.equal(r.status, 400);
r = await post('/add', { source: 'archive', ref: 'moby1', title: 'Moby', author: 'Melville' }); assert.equal((await r.json()).title, 'Moby'); assert.ok(seen.includes('/ia/download/moby1/moby1.epub'), 'the EPUB is chosen over the PDF');
r = await post('/add', { source: 'archive', ref: 'locked' }); assert.equal(r.status, 403); assert.match((await r.json()).error, /DRM/);
r = await post('/add', { source: 'gutenberg', ref: '66' }); assert.equal(r.status, 502, 'a download that redirects off the site is refused'); assert.ok(!seen.some(x => x.includes('/secret')));
r = await post('/add', { source: 'gutenberg', ref: '../x' }); assert.equal(r.status, 400);
r = await post('/add', { source: 'standardebooks', ref: 'http://localhost:' + port + '/se/dl/dracula.epub' }); assert.equal(r.status, 400, 'a catalogue book must live on the catalogue\'s own address');
r = await post('/add', { source: 'standardebooks', ref: B + '/se/dl/dracula.epub', title: 'Dracula', format: 'epub' }); assert.equal(r.status, 200);
// custom catalogue
assert.equal((await post('/sources', { name: 'Mine', url: B + '/cal/opds' }, { 'x-user': 'b' })).status, 403, 'only an admin adds catalogues');
assert.equal((await post('/sources', { name: 'Bad', url: B + '/cal/opds', user: 'reader', pass: 'wrong' })).status, 502, 'a catalogue that refuses the sign-in is not added');