nate/hub
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Open Library in the book store; podcast topic search and transcript search; Recipes (Mealie) with a shopping list synced to Tasks; Contacts linked to Mail and the Calendar
build-container / build (push) Successful in 7m21s
Details

Browse Source 
- Store: Open Library search, books fetched from the Internet Archive (free editions only)
- Podcast: Podcast Index discovery by name and topic (optional key); search what is said across transcripts and play from that moment
- Recipes: browse, import from a link, add ingredients to Mealie's shopping list, mirrored two ways with a Shopping list in Tasks
- Contacts: per-person address book, vCard import/export, recipient suggestions in Mail, birthdays in the Calendar

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
nmemmertandClaude Sonnet 5.5 committed 2026-10-26 08:55:32 -04:00
1 parent 0617664d97
commit 176b1b8169
28 files changed
+799 -46
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

transcripts.js
+44 -7
View File
@@ -110,11 +110,11 @@ async function fetchAudio(src, cookie, tmp) {
let n = 0; const body = Readable.fromWeb(r.body); body.on('data', c => { n += c.length; if (n > MAX_AUDIO) body.destroy(new Error('That audio file is too large to transcribe')); });
await pipeline(body, fs.createWriteStream(tmp, { mode: 0o600 }));
}
function startJob(key, src, cookie, getAudio) {
function startJob(key, src, cookie, getAudio, ep) {
jobs.set(key, { state: 'running', since: Date.now() });
chain = chain.then(async () => {
const tmp = path.join(os.tmpdir(), 'nehub-tr-' + key);
try { if (getAudio) await saveAudio(await getAudio(), tmp); else await fetchAudio(src, cookie, tmp); const r = await whisper(tmp, 'audio'); if (!r) throw new Error('No speech was found in that audio'); atomicWrite(fileOf(key), JSON.stringify(r)); jobs.delete(key); }
try { if (getAudio) await saveAudio(await getAudio(), tmp); else await fetchAudio(src, cookie, tmp); const r = await whisper(tmp, 'audio'); if (!r) throw new Error('No speech was found in that audio'); atomicWrite(fileOf(key), JSON.stringify(ep ? { ...r, ep } : r)); jobs.delete(key); }
catch (e) { jobs.set(key, { state: 'error', since: Date.now(), error: e.message }); setTimeout(() => jobs.delete(key), 10 * 60e3).unref(); }
finally { fs.rm(tmp, { force: true }, () => {}); }
});
@@ -132,8 +132,8 @@ async function autoTick() {
if (fs.existsSync(fileOf(key)) || jobs.has(key) || t.n >= 3 || Date.now() - t.at < 6 * 3600e3 * t.n) continue; // (a failing one is retried a few times, further apart each time)
auto.tried.set(key, { n: t.n + 1, at: Date.now() });
let r = { why: '' }; try { r = await nativeTranscript(ep); } catch {}
if (r.transcript) { atomicWrite(fileOf(key), JSON.stringify(r.transcript)); continue; } // (the show publishes one: no need to make one)
startJob(key, '', '', () => podcast.episodeAudio(ep.id)); queued++;
if (r.transcript) { atomicWrite(fileOf(key), JSON.stringify({ ...r.transcript, ep: { key: 'pod:' + ep.id, src: podSrc(ep.id), title: ep.title, artist: ep.show } })); continue; } // (the show publishes one: no need to make one)
startJob(key, '', '', () => podcast.episodeAudio(ep.id), { key: 'pod:' + ep.id, src: podSrc(ep.id), title: ep.title, artist: ep.show }); queued++;
}
}
function startAuto({ on, keep }) {
@@ -144,6 +144,7 @@ function startAuto({ on, keep }) {
}
function status() { let saved = 0; try { saved = fs.readdirSync(cfg.dir).filter(x => x.endsWith('.json')).length; } catch {} return { enabled: !!cfg.api, auto: auto.on, keep: auto.keep, saved, working: [...jobs.values()].filter(j => j.state === 'running').length, failed: [...jobs.values()].filter(j => j.state === 'error').length }; }
const metaOf = q => ({ key: String(q.get('key') || ''), src: String(q.get('src') || ''), title: String(q.get('title') || '').slice(0, 200), artist: String(q.get('artist') || '').slice(0, 200), thumb: String(q.get('thumb') || '').slice(0, 300) });
const SRC_OK = [/^\/api\/podcast\/podcastitems\/[\w-]+\/file$/, /^\/api\/ab\/books\/[^?#\s]+$/]; // only audio this hub already serves to the person
const nativeMiss = new Map(); // key -> { t, why }: a show with no transcript is not asked again for a day
async function handle(req, res) {
@@ -157,7 +158,7 @@ async function handle(req, res) {
const job = jobs.get(key); if (job) return json(res, 200, { state: job.state, since: job.since, error: job.error, ...base });
if (req.method === 'POST') {
if (!cfg.api) return json(res, 400, { error: 'Transcribing is not set up on this server (see the whisper service in docker-compose.yml)' });
startJob(key, src, req.headers.cookie); return json(res, 200, { state: 'running', since: Date.now(), ...base });
startJob(key, src, req.headers.cookie, null, metaOf(q)); return json(res, 200, { state: 'running', since: Date.now(), ...base });
}
let why = '';
if (kind === 'pod') {
@@ -165,11 +166,47 @@ async function handle(req, res) {
if (miss && miss.t > Date.now() - 6 * 3600e3) why = miss.why;
else {
let r; try { r = await nativeTranscript(await podcast.feedInfo(me, key0.slice(4))); } catch (e) { r = { why: 'Checking the show\'s feed failed (' + e.message + ').' }; }
if (r.transcript) { atomicWrite(fileOf(key), JSON.stringify(r.transcript)); return json(res, 200, { state: 'ready', ...base, transcript: r.transcript }); }
if (r.transcript) { atomicWrite(fileOf(key), JSON.stringify({ ...r.transcript, ep: metaOf(q) })); return json(res, 200, { state: 'ready', ...base, transcript: r.transcript }); }
why = r.why; nativeMiss.set(key, { t: Date.now(), why });
}
}
return json(res, 200, { state: 'none', why, ...base });
} catch (e) { json(res, e.status || 500, { error: e.message }); }
}
module.exports = { init, startAuto, status, handle, parseCues, parseJson, transcriptLinks, shape, whisper, getText, nativeTranscript, enabled: () => true };
// ---- searching what has been said: every transcript on disk, for the words typed. Each file remembers which episode it is (`ep`); older ones are matched up from the person's own episodes.
const idx = new Map(); // file -> { mtime, ep, lines }
const fold = t => String(t).toLowerCase().normalize('NFKD').replace(/[\u0300-\u036f]/g, '').replace(/[^\p{L}\p{N}]+/gu, ' ').trim();
function load(f) {
const st = fs.statSync(path.join(cfg.dir, f)), c = idx.get(f); if (c && c.mtime === st.mtimeMs) return c;
let j; try { j = JSON.parse(fs.readFileSync(path.join(cfg.dir, f), 'utf8')); } catch { return null; }
const v = { mtime: st.mtimeMs, ep: j.ep || null, lines: (j.lines || []).map(l => ({ t: l.t, text: l.text || '', f: fold(l.text || '') })).filter(l => l.f) };
idx.set(f, v); return v;
}
/** Episodes whose transcript says all of the words (an exact phrase ranks first), with the moments they say it. Podcasts: only those on the person's list. */
async function search(me, q) {
const words = fold(q).split(' ').filter(Boolean), phrase = words.join(' ');
if (!words.length || phrase.length < 2) return [];
const mine = new Map((await podcast.myEpisodes(me)).map(e => [e.id, e])), byKey = new Map();
for (const e of mine.values()) byKey.set(keyFor('pod', 'pod:' + e.id, podSrc(e.id)) + '.json', e);
const out = [];
for (const f of fs.readdirSync(cfg.dir).filter(x => x.endsWith('.json'))) {
const d = load(f); if (!d) continue;
let ep = d.ep, m = byKey.get(f);
if (m) ep = { key: 'pod:' + m.id, src: podSrc(m.id), title: ep && ep.title || m.title, artist: ep && ep.artist || m.show, thumb: m.thumb }; // (a podcast on your list: always allowed, and old files get their name here)
else if (!ep || !ep.key || !String(ep.key).startsWith('ab:')) continue; // (other people's podcasts, and files that cannot be told apart, are not shown)
// a line is also checked joined to the next one, so a phrase split across two lines is found
const hits = []; let all = 0;
for (let i = 0; i < d.lines.length; i++) {
const l = d.lines[i], two = l.f + ' ' + (d.lines[i + 1] ? d.lines[i + 1].f : ''), ok = words.every(w => l.f.includes(w)) || (words.length > 1 && words.every(w => two.includes(w)) && !words.every(w => (d.lines[i + 1] ? d.lines[i + 1].f : '').includes(w)));
if (ok) { all++; if (hits.length < 4) hits.push({ t: l.t, text: l.text, exact: l.f.includes(phrase) || two.includes(phrase) }); }
}
if (all) out.push({ key: ep.key, src: ep.src, title: ep.title || 'Episode', artist: ep.artist || '', thumb: ep.thumb || '', n: all, exact: hits.some(h => h.exact), hits });
}
return out.sort((a, b) => (b.exact - a.exact) || (b.n - a.n)).slice(0, 30);
}
async function searchHandle(req, res) {
const q = (new URL(req.url, 'http://x').searchParams.get('q') || '').trim().slice(0, 100), me = users.current();
try { json(res, 200, { results: me ? await search(me, q) : [] }); } catch (e) { json(res, 500, { error: e.message }); }
}
module.exports = { search, searchHandle, init, startAuto, status, handle, parseCues, parseJson, transcriptLinks, shape, whisper, getText, nativeTranscript, enabled: () => true };