My Store: Add many books, with title, author, description and cover read out of each EPUB

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

README.md

@@ -163,6 +163,7 @@ One **Store** tab (turn it on per person) with sections for **Books** (the free
 
 **My Store** is your own catalogue, for everyone who has the Store tab. The admin adds items with **Add**: upload files, or copy a file from their Seafile (**From Files**), with an optional cover picture.
 - **Books** (EPUB, PDF, MOBI, AZW3, DOCX): *Add to my library* puts a copy on that person's shelf, or download it.
+  - **Add many books** (admin): pick a whole batch of book files at once. Each one is uploaded on its own and the server reads the **title, author, description and cover picture out of the EPUB itself** (`epubmeta.js`, no libraries); a book already in the store (same file) is skipped. PDF, MOBI, AZW3 and DOCX carry little usable metadata, so they are named after the file, with no author or cover. When it finishes it lists any book with no cover or no author so you can fix those with *Edit*.
 - **Music** (MP3, M4A, AAC, FLAC, OGG, OPUS, WAV): an album can have many tracks; they play in the hub's player and can be downloaded.
 - **Apps**: any files to download (an APK, an installer...), with an optional link and version. The hub does not run them.
 - **Sites**: any website as a card. It opens in a new tab, or inside the hub if you tick *Open inside the hub* (some sites refuse to be shown that way).

epubmeta.js

@@ -0,0 +1,46 @@
+// Reads the title, author, description and cover picture out of an EPUB (a zip with an OPF file inside), with no libraries.
+const zlib = require('zlib'), path = require('path');
+
+const MAX_PART = 8 * 1024 * 1024; // the biggest piece of an EPUB this reads (the OPF and the cover picture)
+const IMAGE = { jpg: 'image/jpeg', jpeg: 'image/jpeg', png: 'image/png', gif: 'image/gif', webp: 'image/webp' };
+
+/** The entries of a zip: name -> where to find it. */
+function entries(buf) {
+  let e = -1; for (let i = buf.length - 22; i >= Math.max(0, buf.length - 65557); i--) if (buf.readUInt32LE(i) === 0x06054b50) { e = i; break; }
+  if (e < 0) throw new Error('Not a zip');
+  const n = buf.readUInt16LE(e + 10); let p = buf.readUInt32LE(e + 16); const out = new Map();
+  for (let k = 0; k < n && p + 46 <= buf.length && buf.readUInt32LE(p) === 0x02014b50; k++) {
+    const method = buf.readUInt16LE(p + 10), csize = buf.readUInt32LE(p + 20), usize = buf.readUInt32LE(p + 24), nl = buf.readUInt16LE(p + 28), xl = buf.readUInt16LE(p + 30), cl = buf.readUInt16LE(p + 32), off = buf.readUInt32LE(p + 42);
+    out.set(buf.toString('utf8', p + 46, p + 46 + nl), { method, csize, usize, off }); p += 46 + nl + xl + cl;
+  }
+  return out;
+}
+function read(buf, ent) {
+  if (!ent || ent.usize > MAX_PART || buf.readUInt32LE(ent.off) !== 0x04034b50) return null;
+  const s = ent.off + 30 + buf.readUInt16LE(ent.off + 26) + buf.readUInt16LE(ent.off + 28), raw = buf.subarray(s, s + ent.csize);
+  return ent.method === 0 ? raw : ent.method === 8 ? zlib.inflateRawSync(raw, { maxOutputLength: MAX_PART }) : null;
+}
+
+const unxml = s => String(s).replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1').replace(/<[^>]+>/g, ' ').replace(/&(#x[0-9a-f]+|#\d+|amp|lt|gt|quot|apos);/gi, (_, c) =>
+  ({ amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" })[c.toLowerCase()] || String.fromCodePoint(c[1].toLowerCase() === 'x' ? parseInt(c.slice(2), 16) : +c.slice(1))).replace(/\s+/g, ' ').trim();
+const tagsOf = (xml, name) => [...xml.matchAll(new RegExp(`<(?:\\w+:)?${name}\\b([^>]*?)(?:/>|>([\\s\\S]*?)</(?:\\w+:)?${name}>)`, 'gi'))].map(m => ({ attrs: m[1] || '', text: m[2] == null ? '' : unxml(m[2]) }));
+const attr = (attrs, name) => { const m = new RegExp(`\\b${name}\\s*=\\s*("([^"]*)"|'([^']*)')`, 'i').exec(attrs); return m ? (m[2] != null ? m[2] : m[3]) : ''; };
+const resolve = (base, href) => { try { href = decodeURIComponent(href); } catch {} return path.posix.normalize(path.posix.join(path.posix.dirname(base), href.split('#')[0])).replace(/^\/+/, ''); };
+
+/** { title, author, description, cover: { type, data } | null } (any part may be missing). Throws if the file is not a readable EPUB. */
+function readEpub(buf) {
+  const z = entries(buf), container = read(buf, z.get('META-INF/container.xml')); if (!container) throw new Error('No container.xml');
+  const opfPath = attr((tagsOf(container.toString('utf8'), 'rootfile')[0] || {}).attrs || '', 'full-path'); if (!opfPath) throw new Error('No package file');
+  const opf = (read(buf, z.get(opfPath)) || Buffer.alloc(0)).toString('utf8');
+  const meta = { title: (tagsOf(opf, 'title')[0] || {}).text || '', author: tagsOf(opf, 'creator').map(c => c.text).filter(Boolean).join(', '), description: ((tagsOf(opf, 'description')[0] || {}).text || '').slice(0, 2000), cover: null };
+  // the cover picture: the item the package calls the cover-image, else the one <meta name="cover"> points at, else an image named "cover"
+  const items = tagsOf(opf, 'item').map(i => ({ id: attr(i.attrs, 'id'), href: attr(i.attrs, 'href'), type: attr(i.attrs, 'media-type').toLowerCase(), props: attr(i.attrs, 'properties') }));
+  const metaCover = tagsOf(opf, 'meta').find(m => attr(m.attrs, 'name').toLowerCase() === 'cover'), cid = metaCover && attr(metaCover.attrs, 'content');
+  const pick = items.find(i => /\bcover-image\b/.test(i.props)) || items.find(i => cid && i.id === cid) || items.find(i => /^image\//.test(i.type) && /cover/i.test(i.id + ' ' + i.href));
+  if (pick) {
+    const data = read(buf, z.get(resolve(opfPath, pick.href))), ext = path.extname(pick.href).slice(1).toLowerCase(), type = /^image\/(jpeg|png|gif|webp)$/.test(pick.type) ? pick.type : IMAGE[ext];
+    if (data && type && data.length > 500) meta.cover = { type, data };
+  }
+  return meta;
+}
+module.exports = { readEpub };

mystore.js

@@ -7,11 +7,13 @@ const { pipeline } = require('stream/promises');
 const atomicWrite = require('./atomic');
 const users = require('./users'), settings = require('./settings'), books = require('./books');
 const { readBody } = require('./body');
+const { readEpub } = require('./epubmeta');
 
 const DATA_DIR = process.env.DATA_DIR || path.join(__dirname, 'data');
 const ROOT = () => path.join(DATA_DIR, 'mystore'), ITEMS = () => path.join(ROOT(), 'items.json'), FILES = () => path.join(ROOT(), 'files');
 const SEAFILE = () => (process.env.SEAFILE_URL || 'https://seafile.necloud.us').replace(/\/$/, '');
 const MAX_FILE = 1024 * 1024 * 1024, MAX_COVER = 3 * 1024 * 1024, MAX_EXT = 2 * 1024 * 1024, MAX_STATE = 256 * 1024, MAX_FILES_PER_ITEM = 200;
+const MAX_BOOK_IMPORT = 300 * 1024 * 1024;
 const KINDS = ['book', 'music', 'app', 'extension', 'link'];
 const PERMS = ['tasks.read', 'tasks.write', 'contacts.read', 'contacts.write', 'calendar.read', 'calendar.write', 'notes.read', 'notes.write', 'recipes.read', 'music.read']; // what an extension may ask the person for
 const BOOK_EXT = ['epub', 'pdf', 'mobi', 'azw3', 'azw', 'docx'];
@@ -107,6 +109,21 @@ async function handle(req, res) {
       if (it.kind === 'link') it.embed = !!b.embed;
       if (!old) { if (l.length >= 2000) throw fail(400, 'That is plenty of items'); l.push(it); } saveItems(l); return json(res, 200, view(it));
     }
+    if (p === '/import' && req.method === 'POST') { // bulk add: the body is one book file (its name is in ?name=); the title, author, description and cover come out of the book itself (admin)
+      needAdmin(me); const name = safeName(u.searchParams.get('name')), ext = extOf(name);
+      if (!BOOK_EXT.includes(ext)) throw fail(400, 'A book must be an EPUB, PDF, MOBI, AZW3 or DOCX file');
+      const l = loadItems(); if (l.length >= 2000) throw fail(400, 'That is plenty of items');
+      const it = { id: newId(), kind: 'book', files: [], cover: false, added: Date.now() }, fid = newId(), file = fileOn(it, fid);
+      const size = await receive(req, file, MAX_BOOK_IMPORT), buf = fs.readFileSync(file), hash = crypto.createHash('sha1').update(buf).digest('hex');
+      const dup = l.find(x => x.kind === 'book' && x.files.some(f => f.hash === hash));
+      if (dup) { fs.rmSync(file, { force: true }); return json(res, 200, { duplicate: true, item: view(dup) }); }
+      let meta = {}; if (ext === 'epub') { try { meta = readEpub(buf); } catch { meta = {}; } }
+      const plain = path.basename(name, '.' + ext).replace(/_+/g, ' ').trim();
+      Object.assign(it, { title: str(meta.title, 120) || str(plain, 120) || 'Untitled', creator: str(meta.author, 120), description: str(meta.description, 2000), version: '', url: '', updated: Date.now() });
+      it.files.push({ id: fid, name, size, ext, hash });
+      if (meta.cover && meta.cover.data.length <= MAX_COVER) { fs.writeFileSync(coverOn(it), meta.cover.data, { mode: 0o600 }); it.cover = meta.cover.type; }
+      l.push(it); saveItems(l); return json(res, 200, { item: view(it), noAuthor: !it.creator, noCover: !it.cover });
+    }
     if ((m = /^\/items\/([0-9a-f]{12})$/.exec(p)) && req.method === 'DELETE') {
       needAdmin(me); const l = loadItems(), it = l.find(x => x.id === m[1]); if (!it) throw fail(404, 'Item not found');
       for (const f of it.files) fs.rmSync(fileOn(it, f.id), { force: true }); fs.rmSync(coverOn(it), { force: true }); saveItems(l.filter(x => x !== it)); return json(res, 200, { ok: true });

public/store-view.js

@@ -35,13 +35,14 @@ async function stMine(el) {
   const draw = () => {
     if (!$('#stbody')) return;
     const mine = stItems.filter(i => i.kind === 'extension' && stInstalled.includes(i.id)), shown = stItems.filter(i => stFilter === 'all' || i.kind === stFilter);
-    el.innerHTML = `<div class="bar" style="margin:0 0 8px;justify-content:flex-start"><p class="sub" style="margin:0;flex:1">Books, music, apps and extensions added just for this hub.</p>${stAdmin ? `<button class="btn" id="stadd">${icon('plus')}Add</button>` : ''}</div>
+    el.innerHTML = `<div class="bar" style="margin:0 0 8px;justify-content:flex-start"><p class="sub" style="margin:0;flex:1">Books, music, apps and extensions added just for this hub.</p>${stAdmin ? `<label class="btn ghost" style="cursor:pointer;margin:0 6px 0 0" title="Pick many EPUB, PDF, MOBI, AZW3 or DOCX files at once. Titles, authors and covers are read from the books.">${icon('plus')}Add many books<input type="file" id="stbulk" hidden multiple accept=".epub,.pdf,.mobi,.azw3,.azw,.docx"></label><button class="btn" id="stadd">${icon('plus')}Add</button>` : ''}</div>
       ${mine.length ? `<h2>My extensions</h2><div class="vgrid">${mine.map(i => stCard(i, true)).join('')}</div>` : ''}
       <div class="seg" id="stkinds"><a href="#" data-k="all" class="${stFilter === 'all' ? 'on' : ''}">All</a>${ST_KINDS.map(k => `<a href="#" data-k="${k[0]}" class="${stFilter === k[0] ? 'on' : ''}">${k[1]}</a>`).join('')}</div>
       ${shown.length ? `<div class="vgrid">${shown.map(i => stCard(i)).join('')}</div>` : `<p class="empty">${stItems.length ? 'Nothing of that kind yet.' : stAdmin ? 'Nothing here yet. Add your first book, album, app or extension.' : 'Nothing has been added yet.'}</p>`}`;
     el.querySelectorAll('#stkinds a').forEach(a => a.onclick = e => { e.preventDefault(); stFilter = a.dataset.k; draw(); });
     el.querySelectorAll('[data-it]').forEach(a => a.onclick = e => { e.preventDefault(); const it = stItems.find(x => x.id === a.dataset.it); if (a.dataset.run) location.hash = '#/store/ext/' + it.id; else stDetail(it, draw); });
     if ($('#stadd')) $('#stadd').onclick = () => stEdit(null, draw);
+    if ($('#stbulk')) $('#stbulk').onchange = e => { const fs = [...e.target.files]; e.target.value = ''; if (fs.length) stBulk(fs, draw); };
   };
   draw();
 }
@@ -246,3 +247,24 @@ async function stHomeExt(el) {
     return `<a class="tile" href="${esc(href)}" ${ext ? 'target="_blank" rel="noopener noreferrer"' : ''}><span class="ic" style="background:hsl(${stHue(i.title)} 55% 42%);font-weight:700;font-size:20px">${esc(i.icon || (i.title.match(/\p{L}|\p{N}/u) || ['?'])[0].toUpperCase())}</span><b>${esc(i.title)}</b><small>${i.kind === 'extension' ? 'Extension' : 'Site'}</small></a>`; };
   el.innerHTML = `<h2>My apps</h2><div class="grid">${mine.map(tileOf).join('')}</div>`;
 }
+
+// Bulk add books: each file goes up on its own and the server reads the title, author and cover out of the book.
+async function stBulk(files, redraw) {
+  let stop = false, done = 0; const added = [], dups = [], failed = [], noCover = [], noAuthor = [];
+  const el = openSheet('Adding books', '', '<p class="sub" id="sbn"></p><div style="height:6px;border-radius:3px;background:var(--line);overflow:hidden"><div id="sbp" style="height:100%;width:0;background:var(--accent)"></div></div><p class="sub" id="sbl" style="margin-top:10px;word-break:break-word"></p><div id="sbr"></div><div class="bar"><button class="btn ghost" id="sbx" style="flex:1;justify-content:center">Stop</button></div>');
+  const q = s => el.querySelector(s); q('#sbx').onclick = () => { if (done >= files.length || stop) closeSheet(); else { stop = true; q('#sbx').textContent = 'Stopping…'; } };
+  for (const f of files) {
+    if (stop) break;
+    q('#sbn').textContent = `Book ${done + 1} of ${files.length}`; q('#sbl').textContent = f.name;
+    try {
+      const r = await stApi('/import?name=' + encodeURIComponent(f.name), { method: 'POST', body: f, headers: { 'content-type': 'application/octet-stream' } });
+      if (r.duplicate) dups.push(f.name); else { added.push(r.item); if (r.noCover) noCover.push(r.item.title); if (r.noAuthor) noAuthor.push(r.item.title); }
+    } catch (x) { failed.push(f.name + ': ' + x.message); }
+    q('#sbp').style.width = (++done / files.length * 100) + '%';
+  }
+  const list = (t, a) => a.length ? `<details style="margin:6px 0"><summary>${t} (${a.length})</summary><div class="sub" style="max-height:25dvh;overflow:auto;word-break:break-word">${a.map(esc).join('<br>')}</div></details>` : '';
+  q('#sbn').textContent = `${stop ? 'Stopped. ' : ''}Added ${added.length} of ${files.length} books`; q('#sbl').textContent = '';
+  q('#sbr').innerHTML = list('Already in the store, skipped', dups) + list('No cover found', noCover) + list('No author found', noAuthor) + list('Could not be added', failed) + ((noCover.length || noAuthor.length) ? '<p class="sub">Open a book and tap Edit to fix its author or cover.</p>' : '');
+  q('#sbx').textContent = 'Done'; done = files.length;
+  try { await stLoad(); } catch {} redraw();
+}

tools/mystore-test.js

@@ -24,6 +24,17 @@ const hub = http.createServer((req, res) => users.runAs(req.headers['x-user'] ==
     r = await fetch(`${base}/api/store/my/items/${book.id}/files/${book.files[0].id}`, { headers: bob }); assert.equal(r.status, 200); assert.match(r.headers.get('content-disposition'), /attachment/); assert.equal(Buffer.from(await r.arrayBuffer()).length, epub.length);
     r = await fetch(`${B}/items/${book.id}/to-library`, { method: 'POST', headers: bob }); const sh = await r.json(); assert.equal(r.status, 200); assert.equal(sh.title, 'My Own Book'); assert.equal(sh.cover, true);
     assert.equal((await (await fetch(base + '/api/read/list', { headers: bob })).json()).length, 1, 'it is on Bob\'s shelf, not Ann\'s'); assert.equal((await (await fetch(base + '/api/read/list')).json()).length, 0);
+    // bulk import: title, author and cover are read out of the EPUB, a repeat is skipped, a plain file falls back to its name
+    const mk = (dir, name) => { cp.execFileSync('python3', ['-c', `import zipfile,sys;z=zipfile.ZipFile(sys.argv[1],'w');z.writestr('mimetype','application/epub+zip',zipfile.ZIP_STORED);z.writestr('META-INF/container.xml','<container><rootfiles><rootfile full-path="OEBPS/c.opf" media-type="application/oebps-package+xml"/></rootfiles></container>');z.writestr('OEBPS/c.opf','<package><metadata xmlns:dc="x"><dc:title>Caf&#233; &amp; Cream</dc:title><dc:creator>Ann Author</dc:creator><dc:creator>Bo Two</dc:creator><meta name="cover" content="img"/></metadata><manifest><item id="img" href="img/cv.png" media-type="image/png"/></manifest></package>');z.writestr('OEBPS/img/cv.png',b'\\x89PNG'+b'0'*900);z.close()`, path.join(dir, name)]); return fs.readFileSync(path.join(dir, name)); };
+    const withCover = mk(fx, 'cover.epub'), imp = (n, b, h = {}) => put('/import?name=' + encodeURIComponent(n), b, h);
+    assert.equal((await imp('a.epub', withCover, bob)).status, 403, 'only the admin imports');
+    assert.equal((await imp('a.exe', withCover)).status, 400);
+    let ir = await (await imp('whatever_file.epub', withCover)).json(); assert.equal(ir.item.title, 'Café & Cream'); assert.equal(ir.item.creator, 'Ann Author, Bo Two'); assert.equal(ir.item.cover, true); assert.equal(ir.noCover, false);
+    assert.equal((await fetch(`${B}/items/${ir.item.id}/cover`)).headers.get('content-type'), 'image/png');
+    assert.equal((await (await imp('again.epub', withCover)).json()).duplicate, true, 'the same book is not added twice');
+    ir = await (await imp('Plain_Name.epub', epub)).json(); assert.equal(ir.item.title, 'Rabbit Tales'); assert.equal(ir.noCover, true); assert.equal(ir.item.creator, 'A. Writer');
+    ir = await (await imp('Some_Pdf_Book.pdf', Buffer.from('%PDF-1.4 hello'))).json(); assert.equal(ir.item.title, 'Some Pdf Book'); assert.equal(ir.noAuthor, true);
+    for (const it of (await (await fetch(B + '/items')).json()).items.filter(x => x.id !== book.id && x.kind === 'book')) await fetch(`${B}/items/${it.id}`, { method: 'DELETE' });
     // music: several files, streamed with ranges
     let alb = await (await post('/items', { kind: 'music', title: 'Album', creator: 'Band' })).json(); const mp3 = Buffer.from('0123456789abcdefghij');
     alb = await (await put(`/items/${alb.id}/files?name=01.mp3`, mp3)).json(); alb = await (await put(`/items/${alb.id}/files?name=02.mp3`, mp3)).json(); assert.equal(alb.files.length, 2);