Books: book store with Project Gutenberg, Standard Ebooks, Internet Archive and OPDS catalogues

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UaC9HmtHMdRP8dAbN4wAKj

README.md

@@ -59,6 +59,7 @@ e.g. hub.necloud.us. Settings: `SEAFILE_URL`, `ONLYOFFICE_URL` env vars in docke
 - **Books**: an e-book reader for **EPUB, PDF, DOCX, MOBI, AZW and AZW3** (Media group; give it to people in Settings -> People as *Books*). Add files with the button or by dropping them on the page (up to 300 MB each; the server checks a file really is what its name says and does not store the same book twice). Titles, authors and covers are read out of the books in the browser; long-press or right-click a book to edit, download or delete it. Each person has their own shelf in `/data/users/<id>/books/`.
   - **Reading**: EPUB uses epub.js (pages or scroll, two-page spreads on wide screens), PDF uses pdf.js (continuous scroll, zoom, selectable text, dark and sepia tints) DOCX is converted to HTML by mammoth, and MOBI, AZW and AZW3 are read by `@lingo-reader/mobi-parser`; both are shown in a frame that cannot run scripts (scroll only, with contents from the book or its headings). Contents, search inside the book, a progress slider, tap the edges or use the arrow keys, swipe on touch screens, tap the middle to hide the bars. Appearance (theme, text size, font, line spacing, margins, layout, zoom) is remembered on each device.
   - **Synced**: where you got to, bookmarks, highlights (four colours) and notes are kept on the server, so a book opens where you left off on any device; two devices adding notes at once keep both (the newest edit of the same note wins). Continue reading on the library page, sorting and search, and *Copy as text* exports a book's notes.
+  - **Book store** (Books -> Book store): search free, DRM-free books from **Project Gutenberg** (through Gutendex), **Standard Ebooks**, the **Internet Archive** (public-domain EPUBs only; borrow-only items are refused) and any **OPDS catalogue** an admin adds under *Catalogues* (Calibre-Web, Kavita, Komga and so on; an optional username and password are kept encrypted on the server and never sent to the browser). One search covers every source, results show their cover and source, and *Add to library* downloads the book on the server straight onto your shelf. The server only ever contacts a source's own addresses (redirects to other sites are refused), covers come through the hub, and a catalogue book must live on that catalogue's own address. Settings: `GUTENDEX_URL`, `STANDARD_EBOOKS_URL` and `ARCHIVE_URL` can point at mirrors. `node tools/store-test.js` tests it against fake catalogues; it has not been run against the live sites from the build environment, so if one of them changes its feed format, the line under the search box names the source that failed.
   - The reader scripts are in `public/vendor/reader/` (epub.js, JSZip, pdf.js, mammoth, the MOBI parser) and load only when a book is opened or added. `node tools/books-test.js` tests the server; `python3 tools/mkbooks.py <dir>` makes a small EPUB, PDF and MOBI to try. **DRM-protected books are refused** (a Kindle file with copy protection is detected on upload and the reason is shown); only DRM-free books work. The MOBI reader was tried on a generated MOBI 6 file, not on real Kindle files, so tell me about any book that opens oddly. Not included: text-to-speech, offline reading.
 - **Signal**: your Signal chats inside the app, **on desktop-class screens only** (a window at least 900 px wide with a mouse or trackpad; phones and touch tablets don't
   see it). Besides the Signal tab there is a chat button on **every** screen that opens a panel over whatever you are doing (close it with the x; the

books.js

@@ -48,23 +48,32 @@ function mergeNotes(have, sent) {
   return [...by.values()].filter(n => !n.del || Date.now() - n.t < 30 * 864e5);
 }
 
+/** Put a book on the signed-in person's shelf (the same checks for an upload and for a download from the store). Returns the book; `duplicate` is set when it was already there. */
+function addBook(buf, ext, meta = {}, cover) {
+  if (!TYPES[ext]) throw new Error('Only EPUB, PDF, DOCX, MOBI, AZW and AZW3 books can be added');
+  if (buf.length > MAX_BOOK) throw new Error('That file is larger than 300 MB');
+  if (!looksLike(ext, buf)) throw new Error('That does not look like a real ' + ext.toUpperCase() + ' file');
+  if (MOBI.has(ext) && mobiLocked(buf)) throw new Error('This book has DRM (copy protection), which NE Hub cannot open. Only DRM-free books work.');
+  const hash = crypto.createHash('sha1').update(buf).digest('hex'), list = index(), dup = list.find(b => b.hash === hash);
+  if (dup) return { ...dup, duplicate: true };
+  const id = crypto.randomBytes(8).toString('hex');
+  fs.writeFileSync(path.join(dir(), id + '.' + ext), buf, { mode: 0o600 });
+  const book = { id, ext, hash, size: buf.length, added: Date.now(), title: clip(meta.title, 200) || 'Untitled', author: clip(meta.author, 200), cover: false };
+  if (cover && COVERS[cover.type] && cover.data.length <= MAX_COVER) { fs.writeFileSync(path.join(dir(), `${id}.cover.${COVERS[cover.type]}`), cover.data, { mode: 0o600 }); book.cover = true; }
+  list.push(book); saveIndex(list); return book;
+}
+
 async function handle(req, res) {
   const u = new URL(req.url, 'http://x'), p = u.pathname.slice('/api/read'.length) || '/';
   let m;
+  if (p.startsWith('/store/')) return require('./store').handle(req, res, p.slice('/store'.length));
   if (p === '/list' && req.method === 'GET') { const st = state(); return json(res, 200, index().map(b => withState(b, st))); }
 
   if (p === '/upload' && req.method === 'POST') {
     const name = clip(u.searchParams.get('name'), 200), ext = path.extname(name).slice(1).toLowerCase();
     if (!TYPES[ext]) return json(res, 400, { error: 'Only EPUB, PDF, DOCX, MOBI, AZW and AZW3 books can be added' });
     const buf = await body(req, MAX_BOOK); if (!buf) return json(res, 413, { error: 'That file is larger than 300 MB' });
-    if (!looksLike(ext, buf)) return json(res, 400, { error: 'That does not look like a real ' + ext.toUpperCase() + ' file' });
-    if (MOBI.has(ext) && mobiLocked(buf)) return json(res, 400, { error: 'This book has DRM (copy protection), which NE Hub cannot open. Only DRM-free books work.' });
-    const hash = crypto.createHash('sha1').update(buf).digest('hex'), list = index(), dup = list.find(b => b.hash === hash);
-    if (dup) return json(res, 200, { ...dup, duplicate: true });
-    const id = crypto.randomBytes(8).toString('hex');
-    fs.writeFileSync(path.join(dir(), id + '.' + ext), buf, { mode: 0o600 });
-    const book = { id, ext, hash, size: buf.length, added: Date.now(), title: clip(path.basename(name, '.' + ext).replace(/[_]+/g, ' '), 200) || 'Untitled', author: '', cover: false };
-    list.push(book); saveIndex(list); return json(res, 200, book);
+    try { return json(res, 200, addBook(buf, ext, { title: path.basename(name, '.' + ext).replace(/[_]+/g, ' ') })); } catch (e) { return json(res, 400, { error: e.message }); }
   }
 
   if (p === '/state' && req.method === 'GET') return json(res, 200, state());
@@ -135,4 +144,4 @@ async function handle(req, res) {
   return json(res, 404, { error: 'Not found' });
 }
 
-module.exports = { handle, mergeNotes, looksLike };
+module.exports = { handle, mergeNotes, looksLike, addBook, index, TYPES };

package.json

@@ -5,7 +5,7 @@
   "main": "server.js",
   "scripts": {
     "start": "node server.js",
-    "test": "for t in smoke tasks push users cal plex player lyrics signal mail mail-imap files podcast abapi abdirect books; do node tools/$t-test.js || exit 1; done",
+    "test": "for t in smoke tasks push users cal plex player lyrics signal mail mail-imap files podcast abapi abdirect books store; do node tools/$t-test.js || exit 1; done",
     "audit": "npm audit --omit=dev --audit-level=high"
   },
   "keywords": [],

public/mail.js

@@ -206,6 +206,7 @@ function mailConnectScreen(st) {
 }
 
 function mailShell(mode) {
+  mail.sel = null;
   const fl = mail.folders.length ? mail.folders : [{ id: 'inbox', name: 'Inbox', unread: mailUnread }];
   $('#view').innerHTML = `<div class="mailapp" data-mode="${mode}">
     <aside class="mfold" aria-label="Folders">${mail.accts.length > 1 ? mailChips() : ''}${fl.map(f => `<a href="#/mail/f/${encodeURIComponent(f.id)}" class="${f.id === mail.folder ? 'on' : ''}"><span class="grow">${esc(f.name)}</span>${f.unread ? `<i class="nbadge" style="position:static">${f.unread}</i>` : ''}</a>`).join('')}</aside>
@@ -213,13 +214,14 @@ function mailShell(mode) {
       ${mail.accts.length > 1 ? mailChips() : ''}
       <div class="mtool"><select id="mfsel" aria-label="Folder">${fl.map(f => `<option value="${esc(f.id)}" ${f.id === mail.folder ? 'selected' : ''}>${esc(f.name)}${f.unread ? ' (' + f.unread + ')' : ''}</option>`).join('')}</select>
         <input type="search" id="mq" placeholder="Search mail" value="${esc(mail.q)}" autocomplete="off" aria-label="Search mail">
-        <button class="ibtn" id="mrf" aria-label="Refresh">${icon('refresh')}</button><button class="btn" id="mnew">${icon('edit')}New</button></div>
-      <div id="mrows" class="list"></div><div id="mmore"></div>
+        <button class="ibtn" id="msel" aria-label="Select messages" title="Select messages">${icon('check')}</button><button class="ibtn" id="mrf" aria-label="Refresh">${icon('refresh')}</button><button class="btn" id="mnew">${icon('edit')}New</button></div>
+      <div id="mrows" class="list"></div><div id="mmore"></div><div id="mselbar" class="mselbar" hidden></div>
     </section>
     <section class="mread" id="mread" aria-label="Message"></section></div>`;
   $('#mfsel').onchange = e => { mail.q = ''; location.hash = '#/mail/f/' + encodeURIComponent(e.target.value); };
   mailBindChips();
   $('#mnew').onclick = () => mailCompose();
+  $('#msel').onclick = () => mailSelMode(!mail.sel);
   $('#mrf').onclick = () => { mailLoadList(true); jfetch('/api/mail/folders?acct=' + encodeURIComponent(mail.acct)).then(f => { mail.folders = f; mailPoll(); }).catch(() => {}); };
   let t; $('#mq').oninput = e => { clearTimeout(t); t = setTimeout(() => { mail.q = e.target.value.trim(); mailLoadList(true); }, 400); };
 }
@@ -243,13 +245,144 @@ async function mailLoadList(reset) {
 }
 function mailMarkSel(id) { document.querySelectorAll('a.mrow').forEach(a => a.classList.toggle('sel', !!id && decodeURIComponent(a.getAttribute('href').slice(9)) === id)); }
 function mailDrawList() {
-  const rows = $('#mrows'); if (!rows) return;
+  const rows = $('#mrows'); if (!rows) return; mailSwipeInit(rows);
   const sel = (location.hash.match(/^#\/mail\/m\/([^/]+)/) || [])[1];
-  rows.innerHTML = mail.items.length ? mail.items.map(m => `<a class="row mrow${m.read ? '' : ' unread'}${sel && decodeURIComponent(sel) === m.id ? ' sel' : ''}" href="#/mail/m/${encodeURIComponent(m.id)}">
+  rows.innerHTML = mail.items.length ? mail.items.map(m => `<a class="row mrow${m.read ? '' : ' unread'}${sel && !mail.sel && decodeURIComponent(sel) === m.id ? ' sel' : ''}${mail.sel && mail.sel.has(m.id) ? ' picked' : ''}" href="#/mail/m/${encodeURIComponent(m.id)}">${mail.sel ? `<span class="mchk${mail.sel.has(m.id) ? ' on' : ''}"></span>` : ''}
       <span class="grow"><b>${esc(mailFrom(m.from))}</b><span class="msub">${esc(m.subject)}</span><small>${esc(m.preview)}</small></span>
-      <span class="mmeta"><small>${esc(mailWhen(m.date))}</small>${m.flagged ? icon('check') : ''}${m.attach ? icon('clip') : ''}</span></a>`).join('') : `<p class="empty" style="padding:12px">${mail.q ? 'Nothing matches.' : 'No messages here.'}</p>`;
+      <span class="mmeta"><small>${esc(mailWhen(m.date))}</small>${m.flagged ? `<span class="mflag" title="Flagged">${icon('bookmark')}</span>` : ''}${m.attach ? icon('clip') : ''}</span></a>`).join('') : `<p class="empty" style="padding:12px">${mail.q ? 'Nothing matches.' : 'No messages here.'}</p>`;
   const more = $('#mmore'); more.innerHTML = mail.more ? '<div class="bar" style="justify-content:center"><button class="btn ghost" id="mmb">Load more</button></div>' : '';
   if ($('#mmb')) $('#mmb').onclick = () => mailLoadList(false);
+  mailDrawSelBar();
+}
+
+// ---- Swipe a message in the list: left deletes it, right archives it (touch screens). The row follows your finger and the colour under it shows what will happen;
+// let go past a third of the way (or flick it) and it leaves at once and finishes in the background, otherwise it springs back.
+function mailSwipeInit(rows) {
+  if (rows.dataset.swipe) return; rows.dataset.swipe = '1';
+  let row = null, x0 = 0, y0 = 0, dx = 0, t0 = 0, mode = '', w = 0;
+  const reset = (anim) => { if (!row) return; const r = row; r.style.transition = anim ? 'transform .18s ease-out' : ''; r.style.transform = ''; setTimeout(() => { r.classList.remove('swl', 'swr'); r.style.transition = ''; r.style.removeProperty('--sww'); }, anim ? 200 : 0); };
+  rows.addEventListener('touchstart', e => {
+    const a = e.target.closest && e.target.closest('a.mrow'); if (!a || e.touches.length !== 1 || mail.sel) { row = null; return; }
+    row = a; x0 = e.touches[0].clientX; y0 = e.touches[0].clientY; dx = 0; mode = ''; t0 = Date.now(); w = a.offsetWidth;
+  }, { passive: true });
+  rows.addEventListener('touchmove', e => {
+    if (!row) return; const t = e.touches[0], mx = t.clientX - x0, my = t.clientY - y0;
+    if (!mode) { if (Math.abs(my) > 12 && Math.abs(my) > Math.abs(mx)) { row = null; return; } if (Math.abs(mx) > 12 && Math.abs(mx) > Math.abs(my) * 1.4) { mode = 'swipe'; row.style.setProperty('--sww', w + 'px'); } else return; }
+    dx = mx; row.classList.toggle('swl', dx < 0); row.classList.toggle('swr', dx > 0); row.style.transition = ''; row.style.transform = `translateX(${dx}px)`;
+    row.dataset.sw = dx < 0 ? 'Delete' : 'Archive'; row.classList.toggle('swgo', Math.abs(dx) > w / 3);
+  }, { passive: true });
+  const end = () => {
+    if (!row) return; const r = row; row = null;
+    if (mode !== 'swipe') return;
+    const fast = Math.abs(dx) > 60 && Date.now() - t0 < 250; r.dataset.swiped = '1'; setTimeout(() => delete r.dataset.swiped, 400);
+    if (Math.abs(dx) > w / 3 || fast) {
+      const id = decodeURIComponent(r.getAttribute('href').slice(9)), kind = dx < 0 ? 'delete' : 'archive';
+      if (kind === 'delete' && /(^|~)deleteditems$/.test(mail.folder)) { row = r; reset(true); row = null; const it = mail.items.find(x => x.id === id); return confirmSheet('Delete for good?', it ? it.subject : '', 'Delete', () => mailRowAway(id, kind)); }
+      r.style.transition = 'transform .16s ease-in, opacity .16s'; r.style.transform = `translateX(${dx < 0 ? -w : w}px)`; r.style.opacity = '0.2';
+      setTimeout(() => mailRowAway(id, kind), 160);
+    } else { row = r; reset(true); row = null; }
+  };
+  rows.addEventListener('touchend', end); rows.addEventListener('touchcancel', () => { if (row) { reset(true); row = null; } });
+  rows.addEventListener('click', e => { const a = e.target.closest && e.target.closest('a.mrow'); if (a && a.dataset.swiped) { e.preventDefault(); e.stopPropagation(); } }, true); // the finger lifting after a swipe is not a tap
+  // Selecting: while it is on, a tap picks or unpicks a message; a long press on a phone (or Ctrl/Cmd-click, Shift-click for a range, on a computer) starts it.
+  let lp = 0, lpRow = null, lx = 0, ly = 0, lpFired = false;
+  rows.addEventListener('touchstart', e => { const a = e.target.closest && e.target.closest('a.mrow'); if (!a || e.touches.length !== 1) return; lpRow = a; lx = e.touches[0].clientX; ly = e.touches[0].clientY; clearTimeout(lp);
+    lpFired = false; lp = setTimeout(() => { if (!lpRow) return; const r = lpRow; lpRow = null; lpFired = true; if (navigator.vibrate) navigator.vibrate(15); mailSelMode(true, decodeURIComponent(r.getAttribute('href').slice(9))); }, 520); }, { passive: true });
+  rows.addEventListener('touchmove', e => { if (lpRow && (Math.abs(e.touches[0].clientX - lx) > 8 || Math.abs(e.touches[0].clientY - ly) > 8)) { clearTimeout(lp); lpRow = null; } }, { passive: true });
+  ['touchend', 'touchcancel'].forEach(t => rows.addEventListener(t, e => { clearTimeout(lp); lpRow = null; if (lpFired) { lpFired = false; mail.quiet = Date.now() + 250; if (e.cancelable) e.preventDefault(); } })); // lifting the finger after a long press must not count as a tap on anything
+  rows.addEventListener('contextmenu', e => { if (e.target.closest && e.target.closest('a.mrow') && 'ontouchstart' in window) e.preventDefault(); });
+  rows.addEventListener('click', e => {
+    const a = e.target.closest && e.target.closest('a.mrow'); if (!a || a.dataset.swiped) return; if (Date.now() < (mail.quiet || 0)) { e.preventDefault(); e.stopPropagation(); return; } const id = decodeURIComponent(a.getAttribute('href').slice(9));
+    if (!mail.sel && !(e.ctrlKey || e.metaKey || e.shiftKey)) return;
+    e.preventDefault(); e.stopPropagation();
+    if (!mail.sel) mailSelMode(true, id);
+    else if (e.shiftKey && mail.selLast) { const ids = mail.items.map(x => x.id), i = ids.indexOf(mail.selLast), j = ids.indexOf(id); if (i >= 0 && j >= 0) ids.slice(Math.min(i, j), Math.max(i, j) + 1).forEach(x => mail.sel.add(x)); mailSelPaint(); }
+    else mailSelToggle(id);
+  }, true);
+}
+function mailSelMode(on, first) {
+  mail.sel = on ? new Set(first ? [first] : []) : null; mail.selLast = first || null;
+  if (on && /^#\/mail\/m\//.test(location.hash) && matchMedia('(max-width: 899px)').matches) location.hash = '#/mail/f/' + encodeURIComponent(mail.folder);
+  mailSelPaint();
+}
+/** Show who is picked without rebuilding the list (the row under a finger must stay in the page, or the finger's release never reaches it). */
+function mailSelPaint() {
+  document.querySelectorAll('#mrows a.mrow').forEach(a => {
+    const id = decodeURIComponent(a.getAttribute('href').slice(9)), on = !!mail.sel && mail.sel.has(id); let c = a.querySelector(':scope > .mchk');
+    if (mail.sel && !c) { c = document.createElement('span'); c.className = 'mchk'; a.insertBefore(c, a.firstChild); } else if (!mail.sel && c) c.remove();
+    if (c) c.classList.toggle('on', on); a.classList.toggle('picked', on); if (mail.sel) a.classList.remove('sel');
+  });
+  mailDrawSelBar();
+}
+function mailSelToggle(id) { if (!mail.sel) return; if (mail.sel.has(id)) mail.sel.delete(id); else mail.sel.add(id); mail.selLast = id; mailSelPaint(); }
+function mailDrawSelBar() {
+  const bar = $('#mselbar'); if (!bar) return;
+  document.body.classList.toggle('mselecting', !!mail.sel);
+  if (!mail.sel) { bar.hidden = true; bar.innerHTML = ''; return; }
+  const picked = mail.items.filter(m => mail.sel.has(m.id)), n = picked.length, anyUnread = picked.some(m => !m.read), anyUnflag = picked.some(m => !m.flagged), all = n && n === mail.items.length;
+  const b = (a, label, ic, cls = '') => `<button class="btn ${cls || 'ghost'} msb" data-s="${a}" ${n || a === 'all' || a === 'done' ? '' : 'disabled'}>${ic ? icon(ic) : ''}${label}</button>`;
+  bar.hidden = false;
+  bar.innerHTML = `<div class="msbtop"><b>${n ? n + ' selected' : 'Select messages'}</b><span class="grow"></span>${b('all', all ? 'Select none' : 'Select all', '')}${b('done', 'Done', '', 'btn')}</div>
+    <div class="msbrow">${b(anyUnread ? 'read' : 'unread', anyUnread ? 'Mark read' : 'Mark unread', 'mail')}${b(anyUnflag ? 'flag' : 'unflag', anyUnflag ? 'Flag' : 'Unflag', 'check')}${b('archive', 'Archive', 'files')}${b('move', 'Move', 'newdir')}${b('delete', 'Delete', 'trash', 'danger')}</div>`;
+  bar.querySelectorAll('[data-s]').forEach(x => x.onclick = () => mailSelAct(x.dataset.s));
+}
+function mailSelAct(a) {
+  if (Date.now() < (mail.quiet || 0)) return; // the finger-lift that ended a long press
+  if (a === 'done') return mailSelMode(false);
+  if (a === 'all') { if (mail.sel.size === mail.items.length) mail.sel.clear(); else mail.items.forEach(m => mail.sel.add(m.id)); return mailSelPaint(); }
+  const ids = [...mail.sel].filter(id => mail.items.some(m => m.id === id)); if (!ids.length) return;
+  if (a === 'move') {
+    const fl = mail.folders.filter(f => f.id !== mail.folder);
+    const el = openSheet('Move to', ids.length + (ids.length > 1 ? ' messages' : ' message'), `<div class="list">${fl.map(f => `<button class="row" data-f="${esc(f.id)}"><span class="grow"><b>${esc(f.name)}</b></span>${icon('chev', 'chev')}</button>`).join('') || '<p class="empty">No other folders.</p>'}</div>`);
+    return el.querySelectorAll('[data-f]').forEach(r => r.onclick = () => { closeSheet(); mailBatch(ids, 'move', r.dataset.f); });
+  }
+  if (a === 'delete' && /(^|~)deleteditems$/.test(mail.folder)) return confirmSheet('Delete for good?', ids.length + (ids.length > 1 ? ' messages' : ' message') + ' will be deleted for good.', 'Delete', () => mailBatch(ids, 'delete', true));
+  mailBatch(ids, a);
+}
+/** Change one message in the open list and in the saved copy of it. */
+function mailPatchItem(id, patch) {
+  const it = mail.items.find(x => x.id === id); if (it) Object.assign(it, patch);
+  const c = mcGet('l:' + mail.folder); if (c && c.items) { const x = c.items.find(y => y.id === id); if (x) { Object.assign(x, patch); mcPut('l:' + mail.folder, { items: c.items, more: c.more, next: c.next }); } }
+}
+/** Do the same thing to several messages. The list changes at once; each message is then sent to the server (a few at a time), and any the server refuses go back as they were. */
+async function mailBatch(ids, a, extra) {
+  const items = ids.map(id => mail.items.find(x => x.id === id)).filter(Boolean), before = new Map(items.map(m => [m.id, { ...m }])), at = new Map(items.map(m => [m.id, mail.items.indexOf(m)]));
+  const removes = ['archive', 'move', 'delete'].includes(a), inbox = mailInInbox(); let dUnread = 0;
+  mail.sel = null; mail.selLast = null;
+  if (removes) {
+    const gone = new Set(ids); mail.items = mail.items.filter(m => !gone.has(m.id)); dUnread = -items.filter(m => !m.read).length;
+    const sv = mcGet('l:' + mail.folder); if (sv && sv.items) mcPut('l:' + mail.folder, { items: sv.items.filter(x => !gone.has(x.id)), more: sv.more, next: sv.next });
+    if (mail.msg && gone.has(mail.msg.id)) { mail.msg = null; location.hash = '#/mail/f/' + encodeURIComponent(mail.folder); }
+  } else if (a === 'read' || a === 'unread') { const read = a === 'read'; dUnread = items.filter(m => m.read !== read).length * (read ? -1 : 1); items.forEach(m => { mailSetRead(m.id, read); }); }
+  else if (a === 'flag' || a === 'unflag') items.forEach(m => mailPatchItem(m.id, { flagged: a === 'flag' }));
+  if (inbox && dUnread) { mailUnread = Math.max(0, mailUnread + dUnread); renderNav(); }
+  mailDrawList();
+  const call = m => a === 'move' ? jfetch('/api/mail/move', { method: 'POST', body: JSON.stringify({ id: m.id, to: extra }) })
+    : a === 'archive' ? jfetch('/api/mail/move', { method: 'POST', body: JSON.stringify({ id: m.id, to: 'archive' }) })
+    : a === 'delete' ? jfetch('/api/mail/delete', { method: 'POST', body: JSON.stringify({ id: m.id, ...(extra ? { permanent: true } : {}) }) })
+    : jfetch('/api/mail/update', { method: 'POST', body: JSON.stringify(a === 'read' || a === 'unread' ? { id: m.id, read: a === 'read' } : { id: m.id, flagged: a === 'flag' }) });
+  const failed = []; let next = 0, lastErr = '';
+  await Promise.all([0, 1, 2, 3].map(async () => { while (next < items.length) { const m = items[next++]; try { await call(m); } catch (e) { failed.push(m); lastErr = e.message; } } }));
+  const word = { read: 'Marked read', unread: 'Marked unread', flag: 'Flagged', unflag: 'Unflagged', archive: 'Archived', move: 'Moved', delete: extra ? 'Deleted' : 'Moved to Deleted' }[a], ok = items.length - failed.length;
+  if (failed.length) {
+    for (const m of failed) { const o = before.get(m.id); if (removes) mail.items.splice(Math.min(at.get(m.id), mail.items.length), 0, o); else { if (a === 'read' || a === 'unread') mailSetRead(m.id, o.read); else mailPatchItem(m.id, { flagged: o.flagged }); } }
+    if (inbox) { jfetch('/api/mail/status').catch(() => {}); } mailDrawList();
+    toast(`${ok ? word + ' ' + ok + ', ' : ''}${failed.length} failed: ${lastErr}`);
+  } else toast(`${word}${items.length > 1 ? ' (' + items.length + ')' : ''}`);
+  mailPoll(); jfetch('/api/mail/folders?acct=' + encodeURIComponent(mail.acct)).then(f => { mail.folders = f; mailRefreshFolders(); }).catch(() => {});
+}
+function mailRowAway(id, kind) {
+  const at = mail.items.findIndex(x => x.id === id), had = mail.items[at]; if (!had) return mailDrawList();
+  const permanent = kind === 'delete' && /(^|~)deleteditems$/.test(mail.folder);
+  mail.items.splice(at, 1);
+  const sv = mcGet('l:' + mail.folder); if (sv && sv.items) mcPut('l:' + mail.folder, { items: sv.items.filter(x => x.id !== id), more: sv.more, next: sv.next });
+  if (mail.msg && mail.msg.id === id) { mail.msg = null; location.hash = '#/mail/f/' + encodeURIComponent(mail.folder); }
+  mailDrawList();
+  if (!had.read && mailInInbox() && mailUnread > 0) { mailUnread--; renderNav(); }
+  toast(kind === 'archive' ? 'Archived' : permanent ? 'Deleted' : 'Moved to Deleted');
+  const refresh = () => { mailPoll(); jfetch('/api/mail/folders?acct=' + encodeURIComponent(mail.acct)).then(f => { mail.folders = f; mailRefreshFolders(); }).catch(() => {}); };
+  jfetch('/api/mail/' + (kind === 'archive' ? 'move' : 'delete'), { method: 'POST', body: JSON.stringify(kind === 'archive' ? { id, to: 'archive' } : { id, ...(permanent ? { permanent: true } : {}) }) })
+    .then(refresh, e => { toast(e.message); if (!had.read && mailInInbox()) { mailUnread++; renderNav(); } mail.items.splice(Math.min(at, mail.items.length), 0, had); mailDrawList(); refresh(); });
 }
 
 // Opened messages are also kept on this device (IndexedDB, the newest 80), so one you have read opens at once, even offline. A message's text never

public/reader.js

@@ -27,6 +27,7 @@ const rdTitleOf = f => f.replace(/\.[^.]+$/, '').replace(/[_]+/g, ' ');
 // ======================================================= Library
 async function viewReader(args = []) {
   if (args[0] === 'open' && args[1]) return rdOpen(args[1]);
+  if (args[0] === 'store') return rdStore();
   const v = $('#view');
   v.innerHTML = '<div class="wrap"><h1>Books</h1><p class="empty">Loading…</p></div>';
   try { await rdLibrary(); } catch (e) {
@@ -43,7 +44,7 @@ async function rdLibrary() {
   const v = $('#view');
   const cover = b => `<span class="vimg rdcover" style="aspect-ratio:2/3">${b.cover ? `<img loading="lazy" alt="" src="/api/read/book/${b.id}/cover" onerror="this.remove()">` : ''}<em>${esc(b.title)}</em><span class="rdext">${esc(b.ext)}</span>${b.pct > 0 ? `<u class="rdprog"><s style="width:${Math.round(b.pct * 100)}%"></s></u>` : ''}</span>`;
   const card = b => `<a class="vcard" href="#/read/open/${b.id}" data-id="${b.id}">${cover(b)}<b>${esc(b.title)}</b><small>${esc(b.author || b.ext.toUpperCase())}${b.pct > 0 ? ' · ' + Math.round(b.pct * 100) + '%' : ''}</small></a>`;
-  v.innerHTML = `<div class="wrap" id="rdlib"><div class="ptop"><h1>Books</h1><span class="pspace grow"></span><button class="btn" id="rdadd">${icon('plus')}Add books</button></div>
+  v.innerHTML = `<div class="wrap" id="rdlib"><div class="ptop"><h1>Books</h1><span class="pspace grow"></span><a class="btn ghost" href="#/read/store">${icon('search')}Book store</a><button class="btn" id="rdadd">${icon('plus')}Add books</button></div>
     <input type="file" id="rdfile" accept=".epub,.pdf,.docx,.mobi,.azw,.azw3" multiple hidden>
     <div id="rdbody"></div><p class="sub" id="rdhint" style="margin-top:16px">EPUB, PDF, DOCX, MOBI and AZW3 (without DRM). Add files with the button or drop them on this page. Your place, bookmarks and highlights follow you to every device.</p></div>`;
   const draw = () => {
@@ -148,6 +149,71 @@ async function rdMobiOpen(u8) {
   return MobiParser.initMobiFile(u8);
 }
 
+// ======================================================= Book store: free, DRM-free books from Project Gutenberg, Standard Ebooks, the Internet Archive and any OPDS catalogue
+async function rdStore() {
+  const v = $('#view'); v.innerHTML = '<div class="wrap"><h1>Book store</h1><p class="empty">Loading…</p></div>';
+  let sources, mine;
+  try { [sources, mine] = await Promise.all([rdApi('/store/sources'), rdApi('/list')]); } catch (e) { v.innerHTML = `<div class="wrap"><a class="btn ghost" href="#/read">${icon('back')}Books</a><p class="err">${esc(e.message)}</p></div>`; return; }
+  if (!location.hash.startsWith('#/read/store')) return;
+  const key = b => (b.title + '|' + (b.author || '')).toLowerCase().replace(/\s+/g, ' ').trim(), have = new Map(mine.map(b => [key(b), b.id]));
+  const S = { q: '', source: 'all', page: 1, items: [], more: false, seq: 0 };
+  v.innerHTML = `<div class="wrap" id="rdst"><div class="ptop"><a class="btn ghost" href="#/read">${icon('back')}Books</a><h1 style="margin:0">Book store</h1><span class="pspace grow"></span>${ME && ME.role === 'admin' ? `<button class="btn ghost" id="rdcat">${icon('cog')}Catalogues</button>` : ''}</div>
+    <p class="sub">Free books without DRM, from public sources. Search them all at once, then add a book to your library in one tap.</p>
+    <input type="search" id="rdsq" placeholder="Search titles and authors" autocomplete="off" autocapitalize="off" style="width:100%">
+    <div class="seg" id="rdchips"></div><div id="rdnote"></div><div id="rdres"></div><div id="rdmore" class="bar" style="justify-content:center"></div></div>`;
+  const chips = () => { $('#rdchips').innerHTML = [{ id: 'all', name: 'All' }, ...sources].map(x => `<a href="#" data-s="${esc(x.id)}" class="${S.source === x.id ? 'on' : ''}">${esc(x.name)}</a>`).join(''); $('#rdchips').querySelectorAll('a').forEach(a => a.onclick = e => { e.preventDefault(); S.source = a.dataset.s; chips(); run(true); }); };
+  const draw = () => {
+    const res = $('#rdres'); if (!res) return;
+    res.innerHTML = S.items.length ? `<div class="vgrid">${S.items.map((b, i) => `<a class="vcard" href="#" data-i="${i}"><span class="vimg rdcover" style="aspect-ratio:2/3">${b.cover ? `<img loading="lazy" alt="" referrerpolicy="no-referrer" src="${esc(b.cover)}" onerror="this.remove()">` : ''}<em>${esc(b.title)}</em><span class="rdext">${esc(b.format)}</span>${have.has(key(b)) ? '<span class="rdhave">In library</span>' : ''}</span><b>${esc(b.title)}</b><small>${esc(b.author || '')}${b.author ? ' · ' : ''}${esc(b.sourceName)}</small></a>`).join('')}</div>` : (S.busy ? '' : `<p class="empty">${S.q ? 'Nothing found.' : 'Search for a book or an author.'}</p>`);
+    res.querySelectorAll('.vcard').forEach(a => a.onclick = e => { e.preventDefault(); detail(S.items[a.dataset.i]); });
+    $('#rdmore').innerHTML = S.more && !S.busy ? '<button class="btn ghost" id="rdmb">Show more</button>' : S.busy ? '<span class="sub">Searching…</span>' : '';
+    if ($('#rdmb')) $('#rdmb').onclick = () => run(false);
+  };
+  const run = async reset => {
+    const my = ++S.seq; if (reset) { S.page = 1; S.items = []; } else S.page++;
+    S.busy = true; $('#rdnote').innerHTML = ''; draw();
+    try {
+      const r = await rdApi(`/store/search?q=${encodeURIComponent(S.q)}&source=${encodeURIComponent(S.source)}&page=${S.page}`); if (my !== S.seq) return;
+      S.items = S.items.concat(r.items); S.more = r.more;
+      const errs = Object.values(r.errors || {}).map(esc);
+      $('#rdnote').innerHTML = errs.length ? `<p class="sub" style="color:var(--danger)">${errs.join('<br>')}</p>` : '';
+    } catch (e) { if (my === S.seq) $('#rdnote').innerHTML = `<p class="err">${esc(e.message)}</p>`; }
+    if (my === S.seq) { S.busy = false; draw(); }
+  };
+  const detail = b => {
+    const el = openSheet(b.title, b.author || '', `<div style="display:flex;gap:14px;align-items:flex-start"><span class="vimg rdcover" style="width:96px;flex:none;aspect-ratio:2/3">${b.cover ? `<img alt="" referrerpolicy="no-referrer" src="${esc(b.cover)}" onerror="this.remove()">` : ''}<em>${esc(b.title)}</em></span>
+      <div style="min-width:0"><p class="sub" style="margin:0">${esc(b.sourceName)} · ${esc(b.format.toUpperCase())}</p>${b.summary ? `<p style="margin:8px 0 0;white-space:normal">${esc(b.summary)}</p>` : ''}</div></div>
+      <div class="err" id="rdae"></div><div class="bar"><button class="btn ghost" id="sx" style="flex:1;justify-content:center">Close</button><button class="btn" id="rdaa2" style="flex:1;justify-content:center">${have.has(key(b)) ? 'Open' : icon('plus') + 'Add to library'}</button></div>`);
+    el.querySelector('#sx').onclick = closeSheet;
+    const btn = el.querySelector('#rdaa2');
+    btn.onclick = async () => {
+      if (have.has(key(b))) { closeSheet(); location.hash = '#/read/open/' + have.get(key(b)); return; }
+      btn.disabled = true; btn.textContent = 'Adding…'; el.querySelector('#rdae').textContent = '';
+      try {
+        const r = await rdApi('/store/add', rdJson('POST', { source: b.source, ref: b.ref, title: b.title, author: b.author, cover: b.coverUrl, format: b.format }));
+        have.set(key(b), r.id); draw(); btn.disabled = false; btn.textContent = 'Open'; toast(r.duplicate ? 'Already in your library' : 'Added to your library');
+      } catch (e) { el.querySelector('#rdae').textContent = e.message; btn.disabled = false; btn.innerHTML = icon('plus') + 'Try again'; }
+    };
+  };
+  const catalogues = () => {
+    const el = openSheet('Catalogues', 'Anything that speaks OPDS: Calibre-Web, Kavita, Komga, Standard Ebooks and others', '<div id="rdcl"></div>');
+    const list = async () => { sources = await rdApi('/store/sources'); chips(); el.querySelector('#rdcl').innerHTML = `<div class="list">${sources.map(x => `<div class="row"><span class="grow"><b>${esc(x.name)}</b><small>${x.builtin ? 'Built in' : 'Added by you'}</small></span>${x.builtin ? '' : `<button class="btn danger" data-d="${esc(x.id)}" style="min-height:36px">Remove</button>`}</div>`).join('')}</div>
+      <form id="rdcf"><label>Add a catalogue</label><input id="rdcn" placeholder="Name (for example My Calibre-Web)" maxlength="60" required><input id="rdcu" type="url" placeholder="Address, for example https://books.example.com/opds" required style="margin-top:8px">
+      <input id="rdcuser" placeholder="Username (if it needs one)" autocomplete="off" style="margin-top:8px"><input id="rdcp" type="password" placeholder="Password" autocomplete="new-password" style="margin-top:8px"><div class="err" id="rdce"></div>
+      <div class="bar"><button class="btn" style="flex:1;justify-content:center">Add catalogue</button></div></form>`;
+      el.querySelectorAll('[data-d]').forEach(b => b.onclick = async () => { try { await rdApi('/store/sources/' + b.dataset.d, { method: 'DELETE' }); if (S.source === b.dataset.d) S.source = 'all'; await list(); } catch (e) { toast(e.message); } });
+      el.querySelector('#rdcf').onsubmit = async e => { e.preventDefault(); const err = el.querySelector('#rdce'), btn = e.submitter; err.textContent = ''; if (btn) btn.disabled = true;
+        try { await rdApi('/store/sources', rdJson('POST', { name: el.querySelector('#rdcn').value, url: el.querySelector('#rdcu').value, user: el.querySelector('#rdcuser').value, pass: el.querySelector('#rdcp').value })); toast('Catalogue added'); await list(); } catch (x) { err.textContent = x.message; if (btn) btn.disabled = false; } };
+    };
+    list();
+  };
+  chips();
+  let t; $('#rdsq').oninput = e => { clearTimeout(t); t = setTimeout(() => { S.q = e.target.value.trim(); run(true); }, 450); };
+  $('#rdsq').onkeydown = e => { if (e.key === 'Enter') { clearTimeout(t); S.q = e.target.value.trim(); run(true); } };
+  if ($('#rdcat')) $('#rdcat').onclick = catalogues;
+  run(true);
+}
+
 // ======================================================= Reader
 let rdCur = null; // the open book's reader, so leaving the page can close it
 async function rdOpen(id) {
@@ -356,13 +422,15 @@ class RdEpub {
       doc.addEventListener('keydown', e => R.keys(e));
       doc.addEventListener('touchstart', e => { x0 = e.changedTouches[0].screenX; y0 = e.changedTouches[0].screenY; t0 = Date.now(); }, { passive: true });
       doc.addEventListener('touchend', e => { const dx = e.changedTouches[0].screenX - x0, dy = e.changedTouches[0].screenY - y0; if (this.R.P.flow === 'paged' && Math.abs(dx) > 50 && Math.abs(dx) > Math.abs(dy) * 1.5 && Date.now() - t0 < 700) dx < 0 ? this.next() : this.prev(); });
-      doc.addEventListener('click', e => { if (e.target.closest && e.target.closest('a[href]')) return; const sel = contents.window.getSelection(); if (sel && !sel.isCollapsed) return; const w = contents.window.innerWidth; if (this.R.P.flow !== 'paged') return R.toggleBars(); if (e.clientX < w * 0.22) this.prev(); else if (e.clientX > w * 0.78) this.next(); else R.toggleBars(); });
+      doc.addEventListener('click', e => { if (e.target.closest && e.target.closest('a[href]')) return; const sel = contents.window.getSelection(); if (sel && !sel.isCollapsed) return; if (this.R.P.flow !== 'paged') return R.toggleBars(); const x = this.tapX(contents, e.clientX); if (x < 0.22) this.prev(); else if (x > 0.78) this.next(); else R.toggleBars(); });
       doc.addEventListener('mousedown', () => R.hideSel());
     });
     const start = loc && !/^\d+$/.test(loc) ? loc : undefined;
     try { await this.rend.display(start); } catch { await this.rend.display(); }
     for (const n of R.notes) this.drawNote(n);
   }
+  /** Where a tap landed as a fraction of the reading area. The page frame is wider than the screen (the pages sit side by side inside it), so its own coordinates say nothing about the screen. */
+  tapX(contents, clientX) { const f = contents.window.frameElement.getBoundingClientRect(), s = this.stage.getBoundingClientRect(); return (f.left + clientX - s.left) / s.width; }
   restyle(k) {
     const R = this.R, P = R.P, t = rdTheme(P);
     if (k === 'flow') { const loc = this.at && this.at.start.cfi; return this.build(loc); }

public/style.css

@@ -874,3 +874,23 @@ kbd { display: inline-block; padding: 1px 7px; border: 1px solid var(--line); bo
 .rdseg { display: flex; gap: 6px; flex-wrap: wrap; } .rdseg button { flex: 1; min-height: 40px; border-radius: 10px; border: 1px solid var(--line); background: var(--panel2); color: var(--text); padding: 0 10px; white-space: nowrap; }
 .rdseg button.on { background: var(--accent); color: var(--accent-t); border-color: transparent; font-weight: 600; }
 .rdstep { display: flex; align-items: center; gap: 12px; } .rdstep span { min-width: 56px; text-align: center; font-weight: 600; } .rdstep .btn { min-width: 64px; justify-content: center; }
+
+/* Mail: swipe a message left to delete, right to archive */
+.mrow { position: relative; touch-action: pan-y; will-change: transform; background: var(--panel); }
+.mrow.swl { box-shadow: var(--sww, 100vw) 0 0 var(--danger); } .mrow.swr { box-shadow: calc(-1 * var(--sww, 100vw)) 0 0 #2f9e6e; }
+.mrow.swl.swgo { box-shadow: var(--sww, 100vw) 0 0 #b3261e; }
+.mrow.swl::after, .mrow.swr::after { content: attr(data-sw); position: absolute; top: 0; bottom: 0; display: flex; align-items: center; color: #fff; font-weight: 600; padding: 0 20px; white-space: nowrap; }
+.mrow.swl::after { left: 100%; } .mrow.swr::after { right: 100%; }
+
+/* Mail: choosing several messages */
+.mchk { width: 22px; height: 22px; border-radius: 50%; border: 2px solid var(--muted); flex: none; margin-top: 1px; position: relative; }
+.mchk.on { background: var(--accent); border-color: var(--accent); }
+.mchk.on::after { content: ''; position: absolute; left: 6px; top: 2px; width: 5px; height: 10px; border: solid var(--accent-t); border-width: 0 2px 2px 0; transform: rotate(45deg); }
+.mrow.picked { background: color-mix(in srgb, var(--accent) 12%, var(--panel)); }
+.mselbar { position: sticky; bottom: 8px; z-index: 4; background: var(--panel); border: 1px solid var(--line); border-radius: 14px; padding: 8px 10px; margin: 10px 0 0; box-shadow: 0 4px 16px rgba(0,0,0,.12); }
+.msbtop { display: flex; align-items: center; gap: 8px; margin-bottom: 6px; } .msbtop .btn { min-height: 36px; }
+.msbrow { display: flex; gap: 6px; flex-wrap: wrap; } .msb { min-height: 40px; flex: 1 1 auto; justify-content: center; padding: 0 10px; font-size: 14px; } .msb svg { width: 18px; height: 18px; margin-right: 4px; }
+.msb:disabled { opacity: .45; }
+body.mselecting .mrow { user-select: none; -webkit-user-select: none; -webkit-touch-callout: none; }
+.mflag { color: var(--danger); } .mflag svg { fill: currentColor; }
+.rdhave { position: absolute; left: 0; right: 0; bottom: 0; z-index: 1; background: rgba(47,158,110,.92); color: #fff; font: 600 11px/1 system-ui; text-align: center; padding: 5px 0; }

public/sw.js

@@ -1,5 +1,5 @@
 // App-shell cache only; Seafile/API traffic always goes to the network.
-const CACHE = 'hub-v34';
+const CACHE = 'hub-v37';
 const SHARE = 'hub-share'; // files shared into the app from other apps, held until the person picks a folder
 const SHELL = ['/icon.svg', '/manifest.webmanifest', '/icons/icon-192.png', '/icons/apple-touch-icon.png'];
 

store.js

@@ -0,0 +1,176 @@
+// Book store: free, DRM-free books from Project Gutenberg (Gutendex), Standard Ebooks (OPDS), the Internet Archive (public-domain EPUBs) and any OPDS catalogue an
+// admin adds (Calibre-Web, Kavita, ...). The hub server does all the talking to those sites: searching, covers and downloading a chosen book straight onto the
+// signed-in person's shelf (books.js), so the browser never contacts them and no catalogue sign-in reaches the browser.
+const fs = require('fs'), path = require('path'), crypto = require('crypto');
+const users = require('./users'), auth = require('./auth'), books = require('./books');
+
+const UA = 'NE-Hub/1.0 (self-hosted e-book reader)';
+const MAX_BOOK = 300 * 1024 * 1024, MAX_COVER = 3 * 1024 * 1024, MAX_TEXT = 4 * 1024 * 1024;
+const env = (k, d) => (process.env[k] || d).replace(/\/+$/, '');
+const BUILTIN = {
+  gutenberg: { name: 'Project Gutenberg', base: () => env('GUTENDEX_URL', 'https://gutendex.com'), hosts: ['gutendex.com', 'gutenberg.org'] },
+  standardebooks: { name: 'Standard Ebooks', base: () => env('STANDARD_EBOOKS_URL', 'https://standardebooks.org/feeds/opds'), hosts: ['standardebooks.org'], opds: true },
+  archive: { name: 'Internet Archive', base: () => env('ARCHIVE_URL', 'https://archive.org'), hosts: ['archive.org'], needsQuery: true },
+};
+const FORMAT_OF = { 'application/epub+zip': 'epub', 'application/pdf': 'pdf', 'application/x-mobipocket-ebook': 'mobi', 'application/vnd.amazon.mobi8-ebook': 'azw3', 'application/vnd.amazon.ebook': 'azw' };
+const PREFER = ['epub', 'azw3', 'mobi', 'azw', 'pdf'];
+
+const json = (res, code, o) => { res.writeHead(code, { 'content-type': 'application/json', 'cache-control': 'no-store' }); res.end(JSON.stringify(o)); };
+const fail = (code, message) => Object.assign(new Error(message), { code });
+const readBody = async (req, max = 1e5) => { const c = []; let n = 0; for await (const d of req) { n += d.length; if (n > max) throw fail(413, 'Too large'); c.push(d); } try { return JSON.parse(Buffer.concat(c).toString('utf8') || '{}'); } catch { throw fail(400, 'Bad request'); } };
+
+// ---- catalogues an admin added (shared by everyone who may use Books). Sign-ins are encrypted at rest with a key from the server secret.
+const DATA_DIR = process.env.DATA_DIR || path.join(__dirname, 'data'), SRC_FILE = () => path.join(DATA_DIR, 'store-sources.json');
+let _key; const key = () => _key || (_key = crypto.scryptSync(auth.secret, 'nehub-store', 32));
+const seal = v => { const iv = crypto.randomBytes(12), c = crypto.createCipheriv('aes-256-gcm', key(), iv), d = Buffer.concat([c.update(JSON.stringify(v), 'utf8'), c.final()]); return Buffer.concat([iv, c.getAuthTag(), d]).toString('base64'); };
+const unseal = s => { try { const b = Buffer.from(s, 'base64'), d = crypto.createDecipheriv('aes-256-gcm', key(), b.subarray(0, 12)); d.setAuthTag(b.subarray(12, 28)); return JSON.parse(Buffer.concat([d.update(b.subarray(28)), d.final()]).toString('utf8')); } catch { return null; } };
+const customs = () => { try { const l = JSON.parse(fs.readFileSync(SRC_FILE(), 'utf8')); return Array.isArray(l) ? l : []; } catch { return []; } };
+const saveCustoms = l => { const f = SRC_FILE(), tmp = f + '.tmp'; fs.mkdirSync(DATA_DIR, { recursive: true }); fs.writeFileSync(tmp, JSON.stringify(l), { mode: 0o600 }); fs.renameSync(tmp, f); };
+const sourceOf = id => {
+  if (BUILTIN[id]) { const b = BUILTIN[id]; return { id, name: b.name, base: b.base(), builtin: true, opds: !!b.opds, needsQuery: !!b.needsQuery, hosts: [...b.hosts, new URL(b.base()).hostname] }; }
+  const c = customs().find(x => x.id === id); if (!c) throw fail(404, 'Unknown catalogue');
+  const cred = c.auth ? unseal(c.auth) : null; return { id, name: c.name, base: c.url, builtin: false, opds: true, cred };
+};
+
+// ---- talking to a site: only its own addresses are ever fetched (redirects included), a size limit applies, and a catalogue's sign-in goes nowhere else
+const sameSite = (src, u) => { const h = u.hostname; return src.builtin ? src.hosts.some(x => h === x || h.endsWith('.' + x)) : u.origin === new URL(src.base).origin; };
+async function get(src, url, { accept = '*/*', timeout = 10000 } = {}) {
+  let u = new URL(url);
+  for (let hop = 0; hop < 6; hop++) {
+    if (!/^https?:$/.test(u.protocol) || !sameSite(src, u)) throw fail(502, 'That address is not part of ' + src.name);
+    const headers = { 'user-agent': UA, accept }; if (src.cred && u.origin === new URL(src.base).origin) headers.authorization = 'Basic ' + Buffer.from(src.cred.user + ':' + src.cred.pass).toString('base64');
+    const r = await fetch(u, { headers, redirect: 'manual', signal: AbortSignal.timeout(timeout) });
+    if ([301, 302, 303, 307, 308].includes(r.status) && r.headers.get('location')) { u = new URL(r.headers.get('location'), u); try { await r.body?.cancel(); } catch {} continue; }
+    if (r.status === 401 || r.status === 403) throw fail(502, src.name + (src.cred ? ' refused the saved sign-in' : ' needs a sign-in'));
+    if (!r.ok) throw fail(502, src.name + ' answered ' + r.status);
+    return r;
+  }
+  throw fail(502, 'Too many redirects from ' + src.name);
+}
+async function limited(r, max) {
+  const len = +r.headers.get('content-length') || 0; if (len > max) throw fail(413, 'That file is too large');
+  const chunks = []; let n = 0; for await (const c of r.body) { n += c.length; if (n > max) throw fail(413, 'That file is too large'); chunks.push(c); } return Buffer.concat(chunks);
+}
+const getText = async (src, url, accept) => (await limited(await get(src, url, { accept }), MAX_TEXT)).toString('utf8');
+const getJson = async (src, url) => { try { return JSON.parse(await getText(src, url, 'application/json')); } catch (e) { if (e.code) throw e; throw fail(502, src.name + ' sent something unexpected'); } };
+
+// ---- OPDS (Atom) feeds, read with a small parser (no XML library in the hub)
+const unent = s => String(s || '').replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1').replace(/&(#x[0-9a-f]+|#\d+|amp|lt|gt|quot|apos);/gi, (m, e) => ({ amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }[e.toLowerCase()] || (e[1] === 'x' || e[1] === 'X' ? String.fromCodePoint(parseInt(e.slice(2), 16)) : e[0] === '#' ? String.fromCodePoint(+e.slice(1)) : m)));
+const tag = (x, t) => { const m = new RegExp('<' + t + '(?:\\s[^>]*)?>([\\s\\S]*?)</' + t + '>', 'i').exec(x); return m ? unent(m[1]).replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim() : ''; };
+const attrs = s => { const o = {}; for (const m of s.matchAll(/([\w:-]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g)) o[m[1].toLowerCase()] = unent(m[2] != null ? m[2] : m[3]); return o; };
+const linksOf = (x, base) => [...x.matchAll(/<link\b([^>]*?)\/?>/gi)].map(m => attrs(m[1])).filter(a => a.href).map(a => ({ ...a, href: new URL(a.href, base).href }));
+function parseFeed(xml, base) {
+  const head = xml.replace(/<entry[\s>][\s\S]*?<\/entry>/gi, ''), feed = { links: linksOf(head, base), entries: [] };
+  for (const m of xml.matchAll(/<entry[\s>][\s\S]*?<\/entry>/gi)) {
+    const e = m[0], links = linksOf(e, base), acq = links.filter(l => /opds-spec\.org\/acquisition/.test(l.rel || '') && FORMAT_OF[(l.type || '').split(';')[0].trim()]).map(l => ({ href: l.href, format: FORMAT_OF[l.type.split(';')[0].trim()] })).sort((a, b) => PREFER.indexOf(a.format) - PREFER.indexOf(b.format));
+    const img = links.find(l => /opds-spec\.org\/image$/.test(l.rel || '')) || links.find(l => /opds-spec\.org\/image/.test(l.rel || ''));
+    const nav = links.find(l => /atom\+xml/.test(l.type || '') && !/opds-spec\.org\/acquisition/.test(l.rel || ''));
+    feed.entries.push({ title: tag(e, 'title'), author: tag((/<author[\s>][\s\S]*?<\/author>/i.exec(e) || [''])[0], 'name'), summary: tag(e, 'summary') || tag(e, 'content'), acq: acq[0] || null, cover: img ? img.href : '', nav: nav ? nav.href : '' });
+  }
+  return feed;
+}
+const unbrace = u => u.replace(/%7B/gi, '{').replace(/%7D/gi, '}');
+const rootCache = new Map();
+async function opdsRoot(src) { const c = rootCache.get(src.id + src.base); if (c && Date.now() - c.t < 600000) return c.v; const v = parseFeed(await getText(src, src.base, 'application/atom+xml, application/xml;q=0.9, */*;q=0.5'), src.base); rootCache.set(src.id + src.base, { t: Date.now(), v }); return v; }
+async function opdsSearchUrl(src, q) {
+  const root = await opdsRoot(src), enc = encodeURIComponent(q), fill = t => t.replace(/\{searchTerms\??\}/g, enc).replace(/\{[^}]*\?\}/g, '').replace(/\{[^}]*\}/g, '');
+  const atom = root.links.find(l => l.rel === 'search' && /atom\+xml/.test(l.type || '') && /\{searchTerms/.test(unbrace(l.href)));
+  if (atom) return fill(unbrace(atom.href));
+  const osd = root.links.find(l => l.rel === 'search' && /opensearchdescription/.test(l.type || ''));
+  if (osd) { const x = await getText(src, osd.href, 'application/xml'); for (const m of x.matchAll(/<Url\b([^>]*?)\/?>/gi)) { const a = attrs(m[1]); if (/atom\+xml/.test(a.type || '') && a.template) return fill(unbrace(new URL(a.template.replace(/\{/g, '%7B').replace(/\}/g, '%7D'), osd.href).href)); } }
+  if (src.id === 'standardebooks') return src.base + '/all?query=' + enc;
+  throw fail(502, src.name + ' does not offer a search');
+}
+async function opdsSearch(src, q, page) {
+  let url;
+  if (q) url = await opdsSearchUrl(src, q);
+  else { const root = await opdsRoot(src); const nav = root.entries.filter(e => e.nav); url = root.entries.some(e => e.acq) ? src.base : (nav.find(e => /new|recent|latest|popular/i.test(e.title)) || nav[0] || {}).nav || src.base; }
+  let feed = parseFeed(await getText(src, url, 'application/atom+xml'), url);
+  for (let i = 1; i < page; i++) { const next = feed.links.find(l => l.rel === 'next'); if (!next) return { items: [], more: false }; url = next.href; feed = parseFeed(await getText(src, url, 'application/atom+xml'), url); }
+  const items = feed.entries.filter(e => e.acq).map(e => ({ ref: e.acq.href, format: e.acq.format, title: e.title, author: e.author, summary: e.summary.slice(0, 600), cover: e.cover }));
+  return { items, more: !!feed.links.find(l => l.rel === 'next') };
+}
+
+// ---- Project Gutenberg through Gutendex, and the Internet Archive
+const tidyName = n => { const m = /^([^,]+),\s*(.+)$/.exec(n || ''); return m ? (m[2] + ' ' + m[1]).replace(/\s*\(.*?\)/g, '').trim() : String(n || ''); };
+async function gutenbergSearch(src, q, page) {
+  const j = await getJson(src, `${src.base}/books/?${q ? 'search=' + encodeURIComponent(q) + '&' : ''}page=${page}`);
+  const items = (j.results || []).filter(b => (b.formats || {})['application/epub+zip']).map(b => ({ ref: String(b.id), format: 'epub', title: b.title, author: (b.authors || []).map(a => tidyName(a.name)).join(', '), summary: (b.subjects || []).slice(0, 3).join(' · '), cover: (b.formats || {})['image/jpeg'] || '' }));
+  return { items, more: !!j.next };
+}
+async function gutenbergResolve(src, ref) {
+  if (!/^\d{1,9}$/.test(ref)) throw fail(400, 'Bad book');
+  const b = await getJson(src, `${src.base}/books/${ref}`), url = (b.formats || {})['application/epub+zip']; if (!url) throw fail(404, 'No EPUB for that book');
+  return { url, format: 'epub' };
+}
+async function archiveSearch(src, q, page) {
+  const query = `(${q}) AND mediatype:texts AND format:EPUB AND -access-restricted-item:true`;
+  const j = await getJson(src, `${src.base}/advancedsearch.php?q=${encodeURIComponent(query)}&fl[]=identifier&fl[]=title&fl[]=creator&rows=20&page=${page}&output=json`);
+  const docs = (j.response && j.response.docs) || [], total = (j.response && j.response.numFound) || 0;
+  return { items: docs.filter(d => /^[\w.-]+$/.test(d.identifier || '')).map(d => ({ ref: d.identifier, format: 'epub', title: [].concat(d.title || '')[0] || d.identifier, author: [].concat(d.creator || []).join(', '), summary: '', cover: `${src.base}/services/img/${d.identifier}` })), more: page * 20 < total };
+}
+async function archiveResolve(src, ref) {
+  if (!/^[\w.-]+$/.test(ref)) throw fail(400, 'Bad book');
+  const m = await getJson(src, `${src.base}/metadata/${ref}`);
+  if (m.metadata && String(m.metadata['access-restricted-item']) === 'true') throw fail(403, 'That book is borrow-only (it has DRM) and cannot be added');
+  const files = (m.files || []).filter(f => /\.epub$/i.test(f.name || '')); const f = files.find(x => /^EPUB$/i.test(x.format)) || files[0]; if (!f) throw fail(404, 'No EPUB file for that item');
+  return { url: `${src.base}/download/${ref}/${f.name.split('/').map(encodeURIComponent).join('/')}`, format: 'epub' };
+}
+
+async function searchOne(src, q, page) {
+  if (src.id === 'gutenberg') return gutenbergSearch(src, q, page);
+  if (src.id === 'archive') return q ? archiveSearch(src, q, page) : { items: [], more: false };
+  return opdsSearch(src, q, page);
+}
+const stamp = (src, it) => ({ ...it, source: src.id, sourceName: src.name, cover: it.cover ? `/api/read/store/cover?s=${encodeURIComponent(src.id)}&u=${encodeURIComponent(it.cover)}` : '', coverUrl: it.cover || '' });
+const listSources = () => [...Object.keys(BUILTIN), ...customs().map(c => c.id)].map(id => { const s = sourceOf(id); return { id, name: s.name, builtin: s.builtin, needsQuery: !!s.needsQuery }; });
+
+async function handle(req, res, p) {
+  const me = users.current();
+  try {
+    if (p === '/sources' && req.method === 'GET') return json(res, 200, listSources());
+    if (p === '/sources' && req.method === 'POST') { // add an OPDS catalogue (admin)
+      if (!me || me.role !== 'admin') throw fail(403, 'Only an admin can add a catalogue');
+      const b = await readBody(req), name = String(b.name || '').trim().slice(0, 60), url = String(b.url || '').trim();
+      if (!name || !/^https?:\/\/\S+$/i.test(url)) throw fail(400, 'Give the catalogue a name and its address (starting with http)');
+      const id = 'c' + crypto.randomBytes(5).toString('hex'), entry = { id, name, url, auth: b.user ? seal({ user: String(b.user), pass: String(b.pass || '') }) : '' };
+      const l = customs(); l.push(entry); saveCustoms(l);
+      try { const src = sourceOf(id), xml = await getText(src, url, 'application/atom+xml'); if (!/<feed[\s>]/i.test(xml)) throw fail(502, 'That does not look like an OPDS catalogue'); }
+      catch (e) { saveCustoms(customs().filter(x => x.id !== id)); throw fail(e.code || 502, e.code ? e.message : 'Could not read that catalogue: ' + e.message); }
+      return json(res, 200, { id, name });
+    }
+    let m;
+    if ((m = /^\/sources\/(\w+)$/.exec(p)) && req.method === 'DELETE') {
+      if (!me || me.role !== 'admin') throw fail(403, 'Only an admin can remove a catalogue');
+      saveCustoms(customs().filter(x => x.id !== m[1])); return json(res, 200, { removed: true });
+    }
+    if (p === '/search' && req.method === 'GET') {
+      const u = new URL(req.url, 'http://x'), q = (u.searchParams.get('q') || '').trim().slice(0, 120), page = Math.min(20, Math.max(1, +u.searchParams.get('page') || 1)), want = u.searchParams.get('source') || 'all';
+      const ids = want === 'all' ? listSources().map(s => s.id) : [want], lists = [], errors = {}; let more = false;
+      await Promise.all(ids.map(async id => { let src; try { src = sourceOf(id); const r = await searchOne(src, q, page); lists.push([id, r.items.map(it => stamp(src, it))]); more = more || r.more; } catch (e) { const nm = src ? src.name : id, msg = e.name === 'TimeoutError' ? 'did not answer in time' : e.message; errors[id] = msg.startsWith(nm) ? msg : nm + ': ' + msg; } }));
+      lists.sort((a, b) => ids.indexOf(a[0]) - ids.indexOf(b[0]));
+      const items = []; for (let i = 0; lists.some(l => l[1][i]); i++) for (const l of lists) if (l[1][i]) items.push(l[1][i]); // one from each catalogue in turn
+      return json(res, 200, { items, errors, more });
+    }
+    if (p === '/cover' && req.method === 'GET') {
+      const u = new URL(req.url, 'http://x'), src = sourceOf(u.searchParams.get('s') || ''), r = await get(src, u.searchParams.get('u') || '', { accept: 'image/*' }), type = (r.headers.get('content-type') || '').split(';')[0].trim().toLowerCase();
+      if (!/^image\/(jpeg|png|gif|webp)$/.test(type)) throw fail(502, 'Not a picture');
+      const buf = await limited(r, MAX_COVER); res.writeHead(200, { 'content-type': type, 'cache-control': 'private, max-age=604800' }); return res.end(buf);
+    }
+    if (p === '/add' && req.method === 'POST') {
+      const b = await readBody(req), src = sourceOf(String(b.source || '')), ref = String(b.ref || '');
+      let target;
+      if (src.id === 'gutenberg') target = await gutenbergResolve(src, ref);
+      else if (src.id === 'archive') target = await archiveResolve(src, ref);
+      else { if (!sameSite(src, new URL(ref, src.base))) throw fail(400, 'Bad book address'); target = { url: ref, format: String(b.format || 'epub') }; }
+      const ext = books.TYPES[target.format] ? target.format : 'epub';
+      const buf = await limited(await get(src, target.url, { accept: 'application/epub+zip, application/octet-stream, */*;q=0.5', timeout: 120000 }), MAX_BOOK);
+      let cover; const cu = String(b.cover || ''); if (cu) { try { const r = await get(src, cu, { accept: 'image/*' }), type = (r.headers.get('content-type') || '').split(';')[0].trim().toLowerCase(); cover = { type, data: await limited(r, MAX_COVER) }; } catch {} }
+      let book; try { book = books.addBook(buf, ext, { title: String(b.title || '').slice(0, 200), author: String(b.author || '').slice(0, 200) }, cover); } catch (e) { throw fail(400, e.message); }
+      return json(res, 200, book);
+    }
+    return json(res, 404, { error: 'Not found' });
+  } catch (e) { return json(res, e.code && e.code >= 400 && e.code < 600 ? e.code : 502, { error: e.name === 'TimeoutError' ? 'The site did not answer in time' : e.message }); }
+}
+
+module.exports = { handle, parseFeed };

tools/store-test.js

@@ -0,0 +1,76 @@
+// Book store: search across Gutenberg, Standard Ebooks, the Internet Archive and an OPDS catalogue; adding a book puts it on the shelf; sites only reach their own addresses.
+const http = require('http'), assert = require('assert'), fs = require('fs'), os = require('os'), path = require('path'), cp = require('child_process');
+process.env.DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'nehub-store-'));
+const fx = fs.mkdtempSync(path.join(os.tmpdir(), 'nehub-storefx-')); cp.execFileSync('python3', [path.join(__dirname, 'mkbooks.py'), fx]);
+const epub = fs.readFileSync(path.join(fx, 'test.epub')), pdf = fs.readFileSync(path.join(fx, 'test.pdf')), gif = Buffer.from('R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==', 'base64');
+const seen = []; let port;
+const feed = (body, links = '') => `<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom">${links}${body}</feed>`;
+const acq = (id, title, author, href, type = 'application/epub+zip') => `<entry><title>${title}</title><author><name>${author}</name></author><id>${id}</id><summary>A &amp; B</summary><link rel="http://opds-spec.org/image" href="/img/${id}.gif" type="image/gif"/><link rel="http://opds-spec.org/acquisition/open-access" href="${href}" type="${type}"/></entry>`;
+const fake = http.createServer((req, res) => {
+  const u = new URL(req.url, 'http://x'), p = u.pathname; seen.push(req.method + ' ' + p + u.search + (req.headers.authorization ? ' [auth]' : ''));
+  const J = o => { res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify(o)); }, X = x => { res.writeHead(200, { 'content-type': 'application/atom+xml' }); res.end(x); };
+  if (p === '/gut/books/' ) return J({ next: u.searchParams.get('page') === '1' ? 'x' : null, results: [{ id: 11, title: 'Alice', authors: [{ name: 'Carroll, Lewis' }], subjects: ['Fantasy'], formats: { 'application/epub+zip': `http://127.0.0.1:${port}/gut/file/11.epub`, 'image/jpeg': `http://127.0.0.1:${port}/img/11.gif` } }, { id: 12, title: 'No epub', authors: [], formats: {} }] });
+  if (p === '/gut/books/11') return J({ id: 11, formats: { 'application/epub+zip': `http://127.0.0.1:${port}/gut/file/11.epub` } });
+  if (p === '/gut/file/11.epub') { res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(epub); }
+  if (p === '/gut/books/66') return J({ id: 66, formats: { 'application/epub+zip': `http://127.0.0.1:${port}/gut/evil` } });
+  if (p === '/gut/evil') { res.writeHead(302, { location: `http://localhost:${port}/secret` }); return res.end(); }
+  if (p === '/img/11.gif' || p.startsWith('/img/')) { res.writeHead(200, { 'content-type': 'image/gif' }); return res.end(gif); }
+  if (p === '/ia/advancedsearch.php') return J({ response: { numFound: 1, docs: [{ identifier: 'moby1', title: 'Moby', creator: ['Melville'] }, { identifier: '../bad', title: 'x' }] } });
+  if (p === '/ia/metadata/moby1') return J({ metadata: {}, files: [{ name: 'moby1.pdf', format: 'Text PDF' }, { name: 'moby1.epub', format: 'EPUB' }] });
+  if (p === '/ia/metadata/locked') return J({ metadata: { 'access-restricted-item': 'true' }, files: [{ name: 'x.epub', format: 'EPUB' }] });
+  if (p === '/ia/download/moby1/moby1.epub') { res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(Buffer.concat([epub, Buffer.from('ia')])); }
+  if (p === '/se/opds') return X(feed('<entry><title>New releases</title><id>n</id><link rel="http://opds-spec.org/sort/new" href="/se/opds/new" type="application/atom+xml"/></entry>', '<link rel="search" href="/se/opds/all?query={searchTerms}" type="application/atom+xml"/>'));
+  if (p === '/se/opds/all') return X(feed(acq('se1', 'Dracula ' + u.searchParams.get('query'), 'Bram Stoker', '/se/dl/dracula.epub')));
+  if (p === '/se/opds/new') return X(feed(acq('se2', 'Newest Book', 'Someone', '/se/dl/new.epub'), '<link rel="next" href="/se/opds/new2" type="application/atom+xml"/>'));
+  if (p === '/se/opds/new2') return X(feed(acq('se3', 'Page Two', 'Someone', '/se/dl/new.epub')));
+  if (p === '/se/dl/dracula.epub' || p === '/se/dl/new.epub') { res.writeHead(200, { 'content-type': 'application/epub+zip' }); return res.end(epub); }
+  if (p.startsWith('/cal/')) { // Calibre-Web style: needs a sign-in; search is described by an OpenSearch file
+    if (req.headers.authorization !== 'Basic ' + Buffer.from('reader:pw').toString('base64')) { res.writeHead(401); return res.end(); }
+    if (p === '/cal/opds') return X(feed('', '<link rel="search" href="/cal/opds/osd" type="application/opensearchdescription+xml"/>'));
+    if (p === '/cal/opds/osd') return X('<OpenSearchDescription><Url type="application/atom+xml" template="/cal/opds/search/{searchTerms}"/></OpenSearchDescription>');
+    if (p.startsWith('/cal/opds/search/')) return X(feed(acq('c1', 'Own Book ' + decodeURIComponent(p.split('/').pop()), 'Me', '/cal/get/1.pdf', 'application/pdf')));
+    if (p === '/cal/get/1.pdf') { res.writeHead(200, { 'content-type': 'application/pdf' }); return res.end(pdf); }
+  }
+  res.writeHead(404); res.end('no');
+}).listen(0, async () => {
+  port = fake.address().port; const B = 'http://127.0.0.1:' + port;
+  Object.assign(process.env, { GUTENDEX_URL: B + '/gut', ARCHIVE_URL: B + '/ia', STANDARD_EBOOKS_URL: B + '/se/opds' });
+  const books = require('../books'), users = require('../users');
+  const people = { a: users.create({ name: 'Ann', role: 'admin', handle: 'YQ' }), b: users.create({ name: 'Bob', role: 'user', tabs: ['read'], handle: 'Yg' }) };
+  const hub = http.createServer((req, res) => users.runAs(req.headers['x-user'] === 'b' ? people.b : people.a, () => books.handle(req, res))).listen(0, async () => {
+    const base = 'http://127.0.0.1:' + hub.address().port + '/api/read/store', call = (p, o = {}) => fetch(base + p, o), post = (p, b, h = {}) => call(p, { method: 'POST', body: JSON.stringify(b), headers: h });
+    try {
+      let r = await call('/sources'); assert.deepEqual((await r.json()).map(s => s.id), ['gutenberg', 'standardebooks', 'archive']);
+      r = await call('/search?q=alice'); let j = await r.json();
+      assert.deepEqual(j.items.map(i => i.source), ['gutenberg', 'standardebooks', 'archive'], 'one result from each catalogue in turn, books with no EPUB left out');
+      assert.equal(j.items[0].author, 'Lewis Carroll'); assert.equal(j.items[1].summary, 'A & B'); assert.match(j.items[0].cover, /^\/api\/read\/store\/cover\?s=gutenberg/); assert.equal(j.more, true);
+      assert.ok(seen.includes('GET /se/opds/all?query=alice'), 'the search link in the catalogue is used'); assert.ok(!j.items.some(i => i.ref === '../bad'), 'odd identifiers are dropped');
+      r = await call('/search?q=&source=standardebooks&page=2'); j = await r.json(); assert.deepEqual(j.items.map(i => i.title), ['Page Two'], 'no search words: the newest list, and its next pages');
+      r = await call('/search?q=x&source=nope'); j = await r.json(); assert.ok(j.errors.nope, 'a broken catalogue is reported and does not stop the rest');
+      r = await call(j.items ? '/cover?s=gutenberg&u=' + encodeURIComponent(B + '/img/11.gif') : ''); assert.equal(r.headers.get('content-type'), 'image/gif');
+      assert.equal((await call('/cover?s=gutenberg&u=' + encodeURIComponent('http://localhost:' + port + '/img/11.gif'))).status, 502, 'covers only come from the catalogue\'s own site');
+      // adding
+      r = await post('/add', { source: 'gutenberg', ref: '11', title: 'Alice', author: 'Lewis Carroll', cover: B + '/img/11.gif' }); let book = await r.json(); assert.equal(r.status, 200); assert.equal(book.ext, 'epub'); assert.equal(book.author, 'Lewis Carroll'); assert.equal(book.cover, true);
+      r = await post('/add', { source: 'gutenberg', ref: '11', title: 'Alice' }); assert.equal((await r.json()).duplicate, true, 'adding the same book twice does not store it twice');
+      assert.equal((await (await fetch(base.replace('/store', '/list'))).json()).length, 1);
+      assert.equal((await post('/add', { source: 'gutenberg', ref: '11' }, { 'x-user': 'b' })).status, 200); assert.equal((await (await fetch(base.replace('/store', '/list'), { headers: { 'x-user': 'b' } })).json()).length, 1, "each person's shelf is their own");
+      r = await post('/add', { source: 'archive', ref: 'moby1', title: 'Moby', author: 'Melville' }); assert.equal((await r.json()).title, 'Moby'); assert.ok(seen.includes('GET /ia/download/moby1/moby1.epub'), 'the EPUB is chosen over the PDF');
+      r = await post('/add', { source: 'archive', ref: 'locked' }); assert.equal(r.status, 403); assert.match((await r.json()).error, /DRM/);
+      r = await post('/add', { source: 'gutenberg', ref: '66' }); assert.equal(r.status, 502, 'a download that redirects off the site is refused'); assert.ok(!seen.some(x => x.includes('/secret')));
+      r = await post('/add', { source: 'gutenberg', ref: '../x' }); assert.equal(r.status, 400);
+      r = await post('/add', { source: 'standardebooks', ref: 'http://localhost:' + port + '/se/dl/dracula.epub' }); assert.equal(r.status, 400, 'a catalogue book must live on the catalogue\'s own address');
+      r = await post('/add', { source: 'standardebooks', ref: B + '/se/dl/dracula.epub', title: 'Dracula', format: 'epub' }); assert.equal(r.status, 200);
+      // custom catalogue
+      assert.equal((await post('/sources', { name: 'Mine', url: B + '/cal/opds' }, { 'x-user': 'b' })).status, 403, 'only an admin adds catalogues');
+      assert.equal((await post('/sources', { name: 'Bad', url: B + '/cal/opds', user: 'reader', pass: 'wrong' })).status, 502, 'a catalogue that refuses the sign-in is not added');
+      assert.equal((await call('/sources').then(x => x.json())).length, 3);
+      r = await post('/sources', { name: 'Mine', url: B + '/cal/opds', user: 'reader', pass: 'pw' }); const mine = await r.json(); assert.equal(r.status, 200);
+      assert.ok(!JSON.stringify(await (await call('/sources')).json()).includes('pw'), 'sign-ins are never sent to the browser'); assert.ok(!fs.readFileSync(path.join(process.env.DATA_DIR, 'store-sources.json'), 'utf8').includes('"pw"'), 'and are encrypted on disk');
+      j = await (await call('/search?q=hello%20world&source=' + mine.id)).json(); assert.equal(j.items[0].title, 'Own Book hello world', 'search found through the OpenSearch description'); assert.equal(j.items[0].format, 'pdf');
+      r = await post('/add', { source: mine.id, ref: j.items[0].ref, title: j.items[0].title, format: 'pdf' }, { 'x-user': 'b' }); assert.equal((await r.json()).ext, 'pdf');
+      assert.equal((await call('/sources/' + mine.id, { method: 'DELETE', headers: { 'x-user': 'b' } })).status, 403); assert.equal((await call('/sources/' + mine.id, { method: 'DELETE' })).status, 200);
+      console.log('store test ok');
+    } catch (err) { console.error(err); process.exitCode = 1; }
+    hub.close(); fake.close();
+  });
+});