From 33764998051ae28a5bcab8238d62deb6ba12ea98 Mon Sep 17 00:00:00 2001 From: nmemmert <67961292+nmemmert@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:48:04 -0400 Subject: [PATCH] My Store: Add many books, with title, author, description and cover read out of each EPUB Co-Authored-By: Claude Sonnet 5.5 --- README.md | 1 + epubmeta.js | 46 +++++++++++++++++++++++++++++++++++++++++++ mystore.js | 17 ++++++++++++++++ public/store-view.js | 24 +++++++++++++++++++++- tools/mystore-test.js | 11 +++++++++++ 5 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 epubmeta.js diff --git a/README.md b/README.md index 681c659..9b81289 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,7 @@ One **Store** tab (turn it on per person) with sections for **Books** (the free **My Store** is your own catalogue, for everyone who has the Store tab. The admin adds items with **Add**: upload files, or copy a file from their Seafile (**From Files**), with an optional cover picture. - **Books** (EPUB, PDF, MOBI, AZW3, DOCX): *Add to my library* puts a copy on that person's shelf, or download it. + - **Add many books** (admin): pick a whole batch of book files at once. Each one is uploaded on its own and the server reads the **title, author, description and cover picture out of the EPUB itself** (`epubmeta.js`, no libraries); a book already in the store (same file) is skipped. PDF, MOBI, AZW3 and DOCX carry little usable metadata, so they are named after the file, with no author or cover. When it finishes it lists any book with no cover or no author so you can fix those with *Edit*. - **Music** (MP3, M4A, AAC, FLAC, OGG, OPUS, WAV): an album can have many tracks; they play in the hub's player and can be downloaded. - **Apps**: any files to download (an APK, an installer...), with an optional link and version. The hub does not run them. - **Sites**: any website as a card. It opens in a new tab, or inside the hub if you tick *Open inside the hub* (some sites refuse to be shown that way). diff --git a/epubmeta.js b/epubmeta.js new file mode 100644 index 0000000..03f59ff --- /dev/null +++ b/epubmeta.js @@ -0,0 +1,46 @@ +// Reads the title, author, description and cover picture out of an EPUB (a zip with an OPF file inside), with no libraries. +const zlib = require('zlib'), path = require('path'); + +const MAX_PART = 8 * 1024 * 1024; // the biggest piece of an EPUB this reads (the OPF and the cover picture) +const IMAGE = { jpg: 'image/jpeg', jpeg: 'image/jpeg', png: 'image/png', gif: 'image/gif', webp: 'image/webp' }; + +/** The entries of a zip: name -> where to find it. */ +function entries(buf) { + let e = -1; for (let i = buf.length - 22; i >= Math.max(0, buf.length - 65557); i--) if (buf.readUInt32LE(i) === 0x06054b50) { e = i; break; } + if (e < 0) throw new Error('Not a zip'); + const n = buf.readUInt16LE(e + 10); let p = buf.readUInt32LE(e + 16); const out = new Map(); + for (let k = 0; k < n && p + 46 <= buf.length && buf.readUInt32LE(p) === 0x02014b50; k++) { + const method = buf.readUInt16LE(p + 10), csize = buf.readUInt32LE(p + 20), usize = buf.readUInt32LE(p + 24), nl = buf.readUInt16LE(p + 28), xl = buf.readUInt16LE(p + 30), cl = buf.readUInt16LE(p + 32), off = buf.readUInt32LE(p + 42); + out.set(buf.toString('utf8', p + 46, p + 46 + nl), { method, csize, usize, off }); p += 46 + nl + xl + cl; + } + return out; +} +function read(buf, ent) { + if (!ent || ent.usize > MAX_PART || buf.readUInt32LE(ent.off) !== 0x04034b50) return null; + const s = ent.off + 30 + buf.readUInt16LE(ent.off + 26) + buf.readUInt16LE(ent.off + 28), raw = buf.subarray(s, s + ent.csize); + return ent.method === 0 ? raw : ent.method === 8 ? zlib.inflateRawSync(raw, { maxOutputLength: MAX_PART }) : null; +} + +const unxml = s => String(s).replace(//g, '$1').replace(/<[^>]+>/g, ' ').replace(/&(#x[0-9a-f]+|#\d+|amp|lt|gt|quot|apos);/gi, (_, c) => + ({ amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" })[c.toLowerCase()] || String.fromCodePoint(c[1].toLowerCase() === 'x' ? parseInt(c.slice(2), 16) : +c.slice(1))).replace(/\s+/g, ' ').trim(); +const tagsOf = (xml, name) => [...xml.matchAll(new RegExp(`<(?:\\w+:)?${name}\\b([^>]*?)(?:/>|>([\\s\\S]*?))`, 'gi'))].map(m => ({ attrs: m[1] || '', text: m[2] == null ? '' : unxml(m[2]) })); +const attr = (attrs, name) => { const m = new RegExp(`\\b${name}\\s*=\\s*("([^"]*)"|'([^']*)')`, 'i').exec(attrs); return m ? (m[2] != null ? m[2] : m[3]) : ''; }; +const resolve = (base, href) => { try { href = decodeURIComponent(href); } catch {} return path.posix.normalize(path.posix.join(path.posix.dirname(base), href.split('#')[0])).replace(/^\/+/, ''); }; + +/** { title, author, description, cover: { type, data } | null } (any part may be missing). Throws if the file is not a readable EPUB. */ +function readEpub(buf) { + const z = entries(buf), container = read(buf, z.get('META-INF/container.xml')); if (!container) throw new Error('No container.xml'); + const opfPath = attr((tagsOf(container.toString('utf8'), 'rootfile')[0] || {}).attrs || '', 'full-path'); if (!opfPath) throw new Error('No package file'); + const opf = (read(buf, z.get(opfPath)) || Buffer.alloc(0)).toString('utf8'); + const meta = { title: (tagsOf(opf, 'title')[0] || {}).text || '', author: tagsOf(opf, 'creator').map(c => c.text).filter(Boolean).join(', '), description: ((tagsOf(opf, 'description')[0] || {}).text || '').slice(0, 2000), cover: null }; + // the cover picture: the item the package calls the cover-image, else the one points at, else an image named "cover" + const items = tagsOf(opf, 'item').map(i => ({ id: attr(i.attrs, 'id'), href: attr(i.attrs, 'href'), type: attr(i.attrs, 'media-type').toLowerCase(), props: attr(i.attrs, 'properties') })); + const metaCover = tagsOf(opf, 'meta').find(m => attr(m.attrs, 'name').toLowerCase() === 'cover'), cid = metaCover && attr(metaCover.attrs, 'content'); + const pick = items.find(i => /\bcover-image\b/.test(i.props)) || items.find(i => cid && i.id === cid) || items.find(i => /^image\//.test(i.type) && /cover/i.test(i.id + ' ' + i.href)); + if (pick) { + const data = read(buf, z.get(resolve(opfPath, pick.href))), ext = path.extname(pick.href).slice(1).toLowerCase(), type = /^image\/(jpeg|png|gif|webp)$/.test(pick.type) ? pick.type : IMAGE[ext]; + if (data && type && data.length > 500) meta.cover = { type, data }; + } + return meta; +} +module.exports = { readEpub }; diff --git a/mystore.js b/mystore.js index e409879..9642e80 100644 --- a/mystore.js +++ b/mystore.js @@ -7,11 +7,13 @@ const { pipeline } = require('stream/promises'); const atomicWrite = require('./atomic'); const users = require('./users'), settings = require('./settings'), books = require('./books'); const { readBody } = require('./body'); +const { readEpub } = require('./epubmeta'); const DATA_DIR = process.env.DATA_DIR || path.join(__dirname, 'data'); const ROOT = () => path.join(DATA_DIR, 'mystore'), ITEMS = () => path.join(ROOT(), 'items.json'), FILES = () => path.join(ROOT(), 'files'); const SEAFILE = () => (process.env.SEAFILE_URL || 'https://seafile.necloud.us').replace(/\/$/, ''); const MAX_FILE = 1024 * 1024 * 1024, MAX_COVER = 3 * 1024 * 1024, MAX_EXT = 2 * 1024 * 1024, MAX_STATE = 256 * 1024, MAX_FILES_PER_ITEM = 200; +const MAX_BOOK_IMPORT = 300 * 1024 * 1024; const KINDS = ['book', 'music', 'app', 'extension', 'link']; const PERMS = ['tasks.read', 'tasks.write', 'contacts.read', 'contacts.write', 'calendar.read', 'calendar.write', 'notes.read', 'notes.write', 'recipes.read', 'music.read']; // what an extension may ask the person for const BOOK_EXT = ['epub', 'pdf', 'mobi', 'azw3', 'azw', 'docx']; @@ -107,6 +109,21 @@ async function handle(req, res) { if (it.kind === 'link') it.embed = !!b.embed; if (!old) { if (l.length >= 2000) throw fail(400, 'That is plenty of items'); l.push(it); } saveItems(l); return json(res, 200, view(it)); } + if (p === '/import' && req.method === 'POST') { // bulk add: the body is one book file (its name is in ?name=); the title, author, description and cover come out of the book itself (admin) + needAdmin(me); const name = safeName(u.searchParams.get('name')), ext = extOf(name); + if (!BOOK_EXT.includes(ext)) throw fail(400, 'A book must be an EPUB, PDF, MOBI, AZW3 or DOCX file'); + const l = loadItems(); if (l.length >= 2000) throw fail(400, 'That is plenty of items'); + const it = { id: newId(), kind: 'book', files: [], cover: false, added: Date.now() }, fid = newId(), file = fileOn(it, fid); + const size = await receive(req, file, MAX_BOOK_IMPORT), buf = fs.readFileSync(file), hash = crypto.createHash('sha1').update(buf).digest('hex'); + const dup = l.find(x => x.kind === 'book' && x.files.some(f => f.hash === hash)); + if (dup) { fs.rmSync(file, { force: true }); return json(res, 200, { duplicate: true, item: view(dup) }); } + let meta = {}; if (ext === 'epub') { try { meta = readEpub(buf); } catch { meta = {}; } } + const plain = path.basename(name, '.' + ext).replace(/_+/g, ' ').trim(); + Object.assign(it, { title: str(meta.title, 120) || str(plain, 120) || 'Untitled', creator: str(meta.author, 120), description: str(meta.description, 2000), version: '', url: '', updated: Date.now() }); + it.files.push({ id: fid, name, size, ext, hash }); + if (meta.cover && meta.cover.data.length <= MAX_COVER) { fs.writeFileSync(coverOn(it), meta.cover.data, { mode: 0o600 }); it.cover = meta.cover.type; } + l.push(it); saveItems(l); return json(res, 200, { item: view(it), noAuthor: !it.creator, noCover: !it.cover }); + } if ((m = /^\/items\/([0-9a-f]{12})$/.exec(p)) && req.method === 'DELETE') { needAdmin(me); const l = loadItems(), it = l.find(x => x.id === m[1]); if (!it) throw fail(404, 'Item not found'); for (const f of it.files) fs.rmSync(fileOn(it, f.id), { force: true }); fs.rmSync(coverOn(it), { force: true }); saveItems(l.filter(x => x !== it)); return json(res, 200, { ok: true }); diff --git a/public/store-view.js b/public/store-view.js index 7e20fb9..2785b3f 100644 --- a/public/store-view.js +++ b/public/store-view.js @@ -35,13 +35,14 @@ async function stMine(el) { const draw = () => { if (!$('#stbody')) return; const mine = stItems.filter(i => i.kind === 'extension' && stInstalled.includes(i.id)), shown = stItems.filter(i => stFilter === 'all' || i.kind === stFilter); - el.innerHTML = `

Books, music, apps and extensions added just for this hub.

${stAdmin ? `` : ''}
+ el.innerHTML = `

Books, music, apps and extensions added just for this hub.

${stAdmin ? `` : ''}
${mine.length ? `

My extensions

${mine.map(i => stCard(i, true)).join('')}
` : ''}
All${ST_KINDS.map(k => `${k[1]}`).join('')}
${shown.length ? `
${shown.map(i => stCard(i)).join('')}
` : `

${stItems.length ? 'Nothing of that kind yet.' : stAdmin ? 'Nothing here yet. Add your first book, album, app or extension.' : 'Nothing has been added yet.'}

`}`; el.querySelectorAll('#stkinds a').forEach(a => a.onclick = e => { e.preventDefault(); stFilter = a.dataset.k; draw(); }); el.querySelectorAll('[data-it]').forEach(a => a.onclick = e => { e.preventDefault(); const it = stItems.find(x => x.id === a.dataset.it); if (a.dataset.run) location.hash = '#/store/ext/' + it.id; else stDetail(it, draw); }); if ($('#stadd')) $('#stadd').onclick = () => stEdit(null, draw); + if ($('#stbulk')) $('#stbulk').onchange = e => { const fs = [...e.target.files]; e.target.value = ''; if (fs.length) stBulk(fs, draw); }; }; draw(); } @@ -246,3 +247,24 @@ async function stHomeExt(el) { return `${esc(i.icon || (i.title.match(/\p{L}|\p{N}/u) || ['?'])[0].toUpperCase())}${esc(i.title)}${i.kind === 'extension' ? 'Extension' : 'Site'}`; }; el.innerHTML = `

My apps

${mine.map(tileOf).join('')}
`; } + +// Bulk add books: each file goes up on its own and the server reads the title, author and cover out of the book. +async function stBulk(files, redraw) { + let stop = false, done = 0; const added = [], dups = [], failed = [], noCover = [], noAuthor = []; + const el = openSheet('Adding books', '', '

'); + const q = s => el.querySelector(s); q('#sbx').onclick = () => { if (done >= files.length || stop) closeSheet(); else { stop = true; q('#sbx').textContent = 'Stopping…'; } }; + for (const f of files) { + if (stop) break; + q('#sbn').textContent = `Book ${done + 1} of ${files.length}`; q('#sbl').textContent = f.name; + try { + const r = await stApi('/import?name=' + encodeURIComponent(f.name), { method: 'POST', body: f, headers: { 'content-type': 'application/octet-stream' } }); + if (r.duplicate) dups.push(f.name); else { added.push(r.item); if (r.noCover) noCover.push(r.item.title); if (r.noAuthor) noAuthor.push(r.item.title); } + } catch (x) { failed.push(f.name + ': ' + x.message); } + q('#sbp').style.width = (++done / files.length * 100) + '%'; + } + const list = (t, a) => a.length ? `
${t} (${a.length})
${a.map(esc).join('
')}
` : ''; + q('#sbn').textContent = `${stop ? 'Stopped. ' : ''}Added ${added.length} of ${files.length} books`; q('#sbl').textContent = ''; + q('#sbr').innerHTML = list('Already in the store, skipped', dups) + list('No cover found', noCover) + list('No author found', noAuthor) + list('Could not be added', failed) + ((noCover.length || noAuthor.length) ? '

Open a book and tap Edit to fix its author or cover.

' : ''); + q('#sbx').textContent = 'Done'; done = files.length; + try { await stLoad(); } catch {} redraw(); +} diff --git a/tools/mystore-test.js b/tools/mystore-test.js index 7046f25..d75463f 100644 --- a/tools/mystore-test.js +++ b/tools/mystore-test.js @@ -24,6 +24,17 @@ const hub = http.createServer((req, res) => users.runAs(req.headers['x-user'] == r = await fetch(`${base}/api/store/my/items/${book.id}/files/${book.files[0].id}`, { headers: bob }); assert.equal(r.status, 200); assert.match(r.headers.get('content-disposition'), /attachment/); assert.equal(Buffer.from(await r.arrayBuffer()).length, epub.length); r = await fetch(`${B}/items/${book.id}/to-library`, { method: 'POST', headers: bob }); const sh = await r.json(); assert.equal(r.status, 200); assert.equal(sh.title, 'My Own Book'); assert.equal(sh.cover, true); assert.equal((await (await fetch(base + '/api/read/list', { headers: bob })).json()).length, 1, 'it is on Bob\'s shelf, not Ann\'s'); assert.equal((await (await fetch(base + '/api/read/list')).json()).length, 0); + // bulk import: title, author and cover are read out of the EPUB, a repeat is skipped, a plain file falls back to its name + const mk = (dir, name) => { cp.execFileSync('python3', ['-c', `import zipfile,sys;z=zipfile.ZipFile(sys.argv[1],'w');z.writestr('mimetype','application/epub+zip',zipfile.ZIP_STORED);z.writestr('META-INF/container.xml','');z.writestr('OEBPS/c.opf','Café & CreamAnn AuthorBo Two');z.writestr('OEBPS/img/cv.png',b'\\x89PNG'+b'0'*900);z.close()`, path.join(dir, name)]); return fs.readFileSync(path.join(dir, name)); }; + const withCover = mk(fx, 'cover.epub'), imp = (n, b, h = {}) => put('/import?name=' + encodeURIComponent(n), b, h); + assert.equal((await imp('a.epub', withCover, bob)).status, 403, 'only the admin imports'); + assert.equal((await imp('a.exe', withCover)).status, 400); + let ir = await (await imp('whatever_file.epub', withCover)).json(); assert.equal(ir.item.title, 'CafĂ© & Cream'); assert.equal(ir.item.creator, 'Ann Author, Bo Two'); assert.equal(ir.item.cover, true); assert.equal(ir.noCover, false); + assert.equal((await fetch(`${B}/items/${ir.item.id}/cover`)).headers.get('content-type'), 'image/png'); + assert.equal((await (await imp('again.epub', withCover)).json()).duplicate, true, 'the same book is not added twice'); + ir = await (await imp('Plain_Name.epub', epub)).json(); assert.equal(ir.item.title, 'Rabbit Tales'); assert.equal(ir.noCover, true); assert.equal(ir.item.creator, 'A. Writer'); + ir = await (await imp('Some_Pdf_Book.pdf', Buffer.from('%PDF-1.4 hello'))).json(); assert.equal(ir.item.title, 'Some Pdf Book'); assert.equal(ir.noAuthor, true); + for (const it of (await (await fetch(B + '/items')).json()).items.filter(x => x.id !== book.id && x.kind === 'book')) await fetch(`${B}/items/${it.id}`, { method: 'DELETE' }); // music: several files, streamed with ranges let alb = await (await post('/items', { kind: 'music', title: 'Album', creator: 'Band' })).json(); const mp3 = Buffer.from('0123456789abcdefghij'); alb = await (await put(`/items/${alb.id}/files?name=01.mp3`, mp3)).json(); alb = await (await put(`/items/${alb.id}/files?name=02.mp3`, mp3)).json(); assert.equal(alb.files.length, 2);