// Book store: search across Gutenberg, Standard Ebooks, the Internet Archive and an OPDS catalogue; adding a book puts it on the shelf; sites only reach their own addresses.
const http = require('http'), assert = require('assert'), fs = require('fs'), os = require('os'), path = require('path'), cp = require('child_process');
process.env.DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'nehub-store-'));
const fx = fs.mkdtempSync(path.join(os.tmpdir(), 'nehub-storefx-')); cp.execFileSync('python3', [path.join(__dirname, 'mkbooks.py'), fx]);
const epub = fs.readFileSync(path.join(fx, 'test.epub')), pdf = fs.readFileSync(path.join(fx, 'test.pdf')), gif = Buffer.from('R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==', 'base64');
const seen = []; let port;
const feed = (body, links = '') => `${links}${body}`;
const acq = (id, title, author, href, type = 'application/epub+zip') => `${title}${author}${id}A & B`;
const seCard = (slug, title, who) => `${title}
${who}
`;
const fake = http.createServer((req, res) => {
const u = new URL(req.url, 'http://x'), p = u.pathname; seen.push(p + u.search);
const J = o => { res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify(o)); }, X = x => { res.writeHead(200, { 'content-type': 'application/atom+xml' }); res.end(x); }, HT = x => { res.writeHead(200, { 'content-type': 'application/xhtml+xml' }); res.end(x); };
// Project Gutenberg's catalogue feed
if (p === '/ebooks/search.opds/') { const more = u.searchParams.get('start_index') ? '' : '';
return X(feed(`Authorshttps://x/ebooks/author/1.opdsAlice in Wonderlandhttps://www.gutenberg.org/ebooks/11.opdsLewis CarrollAlice in Wonderlandhttps://www.gutenberg.org/ebooks/11.opdsLewis CarrollDracula & Cohttps://www.gutenberg.org/ebooks/345.opdsBram Stoker`, more)); }
if (p === '/ebooks/11.epub3.images') { res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(epub); }
if (p === '/ebooks/66.epub3.images') { res.writeHead(302, { location: `http://localhost:${port}/secret` }); return res.end(); }
if (p.startsWith('/cache/epub/') || p.startsWith('/img/')) { res.writeHead(200, { 'content-type': 'image/gif' }); return res.end(gif); }
// Standard Ebooks' public pages (their OPDS feed is for members only)
if (p === '/ebooks') { const q = u.searchParams.get('query') || ''; return HT(`${seCard('bram-stoker/dracula', 'Dracula ' + q, 'Bram Stoker')}${seCard('john-bunyan/the-pilgrims-progress', 'The Pilgrim’s Progress', 'John Bunyan')}
`); }
if (p === '/ebooks/bram-stoker/dracula') return HT(`kae`);
if (p === '/ebooks/bram-stoker/dracula/downloads/bram-stoker_dracula.epub') { if (u.searchParams.get('source') !== 'download') { res.writeHead(200, { 'content-type': 'text/html' }); return res.end('Your Download Has Started!'); } res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(Buffer.concat([epub, Buffer.from('se')])); }
if (p === '/feeds/opds') { res.writeHead(401); return res.end('members only'); }
// the Internet Archive
if (p === '/ia/advancedsearch.php') return J({ response: { numFound: 1, docs: [{ identifier: 'moby1', title: 'Moby', creator: ['Melville'] }, { identifier: '../bad', title: 'x' }] } });
if (p === '/ia/metadata/moby1') return J({ metadata: {}, files: [{ name: 'moby1.pdf', format: 'Text PDF' }, { name: 'moby1.epub', format: 'EPUB' }] });
if (p === '/ia/metadata/locked') return J({ metadata: { 'access-restricted-item': 'true' }, files: [{ name: 'x.epub', format: 'EPUB' }] });
if (p === '/ia/download/moby1/moby1.epub') { res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(Buffer.concat([epub, Buffer.from('ia')])); }
// an OPDS catalogue that needs a sign-in (Calibre-Web style)
if (p.startsWith('/cal/')) {
if (req.headers.authorization !== 'Basic ' + Buffer.from('reader:pw').toString('base64')) { res.writeHead(401); return res.end(); }
if (p === '/cal/opds') return X(feed('', ''));
if (p === '/cal/opds/osd') return X('');
if (p.startsWith('/cal/opds/search/')) return X(feed(acq('c1', 'Own Book ' + decodeURIComponent(p.split('/').pop()), 'Me', '/cal/get/1.pdf', 'application/pdf')));
if (p === '/cal/get/1.pdf') { res.writeHead(200, { 'content-type': 'application/pdf' }); return res.end(pdf); }
}
res.writeHead(404); res.end('no');
}).listen(0, async () => {
port = fake.address().port; const B = 'http://127.0.0.1:' + port;
Object.assign(process.env, { GUTENBERG_URL: B, ARCHIVE_URL: B + '/ia', STANDARD_EBOOKS_URL: B });
const books = require('../books'), users = require('../users');
const people = { a: users.create({ name: 'Ann', role: 'admin', handle: 'YQ' }), b: users.create({ name: 'Bob', role: 'user', tabs: ['read'], handle: 'Yg' }) };
const hub = http.createServer((req, res) => users.runAs(req.headers['x-user'] === 'b' ? people.b : people.a, () => books.handle(req, res))).listen(0, async () => {
const base = 'http://127.0.0.1:' + hub.address().port + '/api/read/store', call = (p, o = {}) => fetch(base + p, o), post = (p, b, h = {}) => call(p, { method: 'POST', body: JSON.stringify(b), headers: h });
try {
let r = await call('/sources'); assert.deepEqual((await r.json()).map(s => s.id), ['gutenberg', 'standardebooks', 'archive']);
r = await call('/search?q=alice'); let j = await r.json();
assert.deepEqual(j.items.map(i => i.source), ['gutenberg', 'standardebooks', 'archive', 'gutenberg', 'standardebooks'].slice(0, j.items.length), 'one result from each catalogue in turn'); assert.deepEqual(j.errors, {});
const g = j.items.filter(i => i.source === 'gutenberg'); assert.deepEqual(g.map(i => i.ref), ['11', '345'], 'catalogue entries that are not books, and repeats, are left out'); assert.equal(g[0].author, 'Lewis Carroll'); assert.equal(g[1].title, 'Dracula & Co'); assert.match(g[0].cover, /^\/api\/read\/store\/cover\?s=gutenberg/); assert.equal(j.more, true);
const se = j.items.filter(i => i.source === 'standardebooks'); assert.equal(se[0].title, 'Dracula alice'); assert.equal(se[0].author, 'Bram Stoker'); assert.equal(se[1].title, 'The Pilgrim’s Progress'); assert.equal(se[0].ref, '/ebooks/bram-stoker/dracula');
assert.ok(!j.items.some(i => i.ref === '../bad'), 'odd identifiers are dropped'); assert.ok(!seen.some(x => x.startsWith('/feeds/opds')), 'the members-only feed is never asked for');
r = await call('/search?q=&source=gutenberg'); j = await r.json(); assert.ok(seen.some(x => x.includes('sort_order=downloads')), 'no search words: the most downloaded books'); assert.equal(j.items.length, 2);
r = await call('/search?q=x&source=gutenberg&page=2'); assert.ok(seen.some(x => x.includes('start_index=26')), 'later pages ask for the next 25'); assert.equal((await r.json()).more, false);
r = await call('/search?q=x&source=nope'); j = await r.json(); assert.ok(j.errors.nope, 'a broken catalogue is reported and does not stop the rest');
r = await call('/cover?s=gutenberg&u=' + encodeURIComponent(B + '/img/11.gif')); assert.equal(r.headers.get('content-type'), 'image/gif');
assert.equal((await call('/cover?s=gutenberg&u=' + encodeURIComponent('http://localhost:' + port + '/img/11.gif'))).status, 502, 'covers only come from the catalogue\'s own site');
// adding
r = await post('/add', { source: 'gutenberg', ref: '11', title: 'Alice', author: 'Lewis Carroll', cover: B + '/cache/epub/11/pg11.cover.medium.jpg' }); let book = await r.json(); assert.equal(r.status, 200); assert.equal(book.ext, 'epub'); assert.equal(book.author, 'Lewis Carroll'); assert.equal(book.cover, true);
assert.ok(seen.includes('/ebooks/11.epub3.images'), 'the EPUB with pictures is fetched');
r = await post('/add', { source: 'gutenberg', ref: '11', title: 'Alice' }); assert.equal((await r.json()).duplicate, true, 'adding the same book twice does not store it twice');
assert.equal((await (await fetch(base.replace('/store', '/list'))).json()).length, 1);
assert.equal((await post('/add', { source: 'gutenberg', ref: '11' }, { 'x-user': 'b' })).status, 200); assert.equal((await (await fetch(base.replace('/store', '/list'), { headers: { 'x-user': 'b' } })).json()).length, 1, "each person's shelf is their own");
r = await post('/add', { source: 'standardebooks', ref: '/ebooks/bram-stoker/dracula', title: 'Dracula', author: 'Bram Stoker' }); assert.equal(r.status, 200, 'the real EPUB is fetched, not the "download has started" page'); assert.ok(seen.some(x => x.includes('bram-stoker_dracula.epub?source=download')) && !seen.some(x => x.includes('kepub') && x.includes('source')), 'the plain EPUB (not the Kobo or advanced one) is chosen');
r = await post('/add', { source: 'standardebooks', ref: '/ebooks/../etc/passwd' }); assert.equal(r.status, 400);
r = await post('/add', { source: 'archive', ref: 'moby1', title: 'Moby', author: 'Melville' }); assert.equal((await r.json()).title, 'Moby'); assert.ok(seen.includes('/ia/download/moby1/moby1.epub'), 'the EPUB is chosen over the PDF');
r = await post('/add', { source: 'archive', ref: 'locked' }); assert.equal(r.status, 403); assert.match((await r.json()).error, /DRM/);
r = await post('/add', { source: 'gutenberg', ref: '../x' }); assert.equal(r.status, 400);
// custom catalogue
assert.equal((await post('/sources', { name: 'Mine', url: B + '/cal/opds' }, { 'x-user': 'b' })).status, 403, 'only an admin adds catalogues');
assert.equal((await post('/sources', { name: 'Bad', url: B + '/cal/opds', user: 'reader', pass: 'wrong' })).status, 502, 'a catalogue that refuses the sign-in is not added');
assert.equal((await call('/sources').then(x => x.json())).length, 3);
r = await post('/sources', { name: 'Mine', url: B + '/cal/opds', user: 'reader', pass: 'pw' }); const mine = await r.json(); assert.equal(r.status, 200);
assert.ok(!JSON.stringify(await (await call('/sources')).json()).includes('pw'), 'sign-ins are never sent to the browser'); assert.ok(!fs.readFileSync(path.join(process.env.DATA_DIR, 'store-sources.json'), 'utf8').includes('"pw"'), 'and are encrypted on disk');
j = await (await call('/search?q=hello%20world&source=' + mine.id)).json(); assert.equal(j.items[0].title, 'Own Book hello world', 'search found through the OpenSearch description'); assert.equal(j.items[0].format, 'pdf');
r = await post('/add', { source: mine.id, ref: j.items[0].ref, title: j.items[0].title, format: 'pdf' }, { 'x-user': 'b' }); assert.equal((await r.json()).ext, 'pdf');
assert.equal((await call('/sources/' + mine.id, { method: 'DELETE', headers: { 'x-user': 'b' } })).status, 403); assert.equal((await call('/sources/' + mine.id, { method: 'DELETE' })).status, 200);
console.log('store test ok');
} catch (err) { console.error(err); process.exitCode = 1; }
hub.close(); fake.close();
});
});
}.gif)