From 0534bf7c4b29de29e8fbef2e5f3a858360c4df86 Mon Sep 17 00:00:00 2001 From: nmemmert <67961292+nmemmert@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:52:59 -0400 Subject: [PATCH] Transcripts for podcasts and audiobooks: the show's own, or speech-to-text on your server (whisper service) Co-Authored-By: Claude Sonnet 5.5 --- .gitea/workflows/build.yml | 1 + README.md | 7 +++ docker-compose.yml | 14 +++++ podcast.js | 11 +++- public/lyrics-view.js | 29 ++++++++- public/player.js | 4 +- public/sw.js | 2 +- server.js | 6 +- tools/transcript-test.js | 42 +++++++++++++ transcripts.js | 120 +++++++++++++++++++++++++++++++++++++ 10 files changed, 228 insertions(+), 8 deletions(-) create mode 100644 tools/transcript-test.js create mode 100644 transcripts.js diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index 46a3f8a..7dbb694 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -26,6 +26,7 @@ jobs: - run: node tools/plex-test.js # Plex sign-in stays signed in and repairs itself - run: node tools/player-test.js # music player: queue, shuffle/repeat, stall recovery - run: node tools/lyrics-test.js # lyrics: formats, wrong matches refused, remembered + - run: node tools/transcript-test.js # transcripts: SRT/VTT/JSON, a show's own transcript link, private addresses refused, speech-to-text - run: node tools/mail-test.js # Outlook.com mail through Microsoft Graph: connect, refresh, read, send, privacy - run: node tools/mail-imap-test.js # Gmail and iCloud mail over IMAP/SMTP: connect, read, send, reply, privacy - run: node tools/files-test.js # the Seafile pass-through: timeouts, retries, closed downloads, uploads diff --git a/README.md b/README.md index 70950bc..9a35f1c 100644 --- a/README.md +++ b/README.md @@ -109,3 +109,10 @@ On the server: cp .env.example .env # fill in secrets, PUBLIC_URL, MS_CLIENT_ID (for Mail), etc. docker compose pull && docker compose up -d docker compose logs hub | grep "Setup code" # first sign-in: create your passkey + + +## Transcripts for podcasts and audiobooks +The lyrics screen shows a **Transcript** for podcast episodes and audiobooks. If the show publishes its own (Podcast 2.0 ``: SRT, VTT or JSON) it is used +straight away. Otherwise a **Transcribe on this server** button makes one with speech-to-text: the bundled `whisper` service in `docker-compose.yml` (faster-whisper behind an +OpenAI-compatible API) is reached at `TRANSCRIBE_API_URL` (default `http://127.0.0.1:8000`), model `TRANSCRIBE_MODEL` (default `Systran/faster-whisper-small`, downloaded on first use). +CPU only: about a quarter of the running time or more. One job at a time; results are kept in `/data/transcripts`. Remove the service and the variable to switch the button off. diff --git a/docker-compose.yml b/docker-compose.yml index 884e12d..cf0570e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,6 +22,7 @@ services: MS_CLIENT_ID: ${MS_CLIENT_ID:-} # Outlook.com mail (Microsoft app registration, see README -> Mail); leave empty to switch Mail off MS_CLIENT_SECRET: ${MS_CLIENT_SECRET:-} LYRICS_API_URL: http://127.0.0.1:9999 # the bundled lyrics service (Lyrica, below, shares this container's network); remove this line and the service to skip it + TRANSCRIBE_API_URL: http://127.0.0.1:8000 # the bundled speech-to-text service (whisper, below): makes a transcript for a podcast or audiobook on request. Remove this line and the service to skip it SIGNAL_API_URL: http://127.0.0.1:8085 # the bundled Signal bridge (below, shares this container's network); remove this line and the service to skip Signal OPENAUDIBLE_URL: ${OPENAUDIBLE_URL:-} # audiobooks straight from OpenAudible (e.g. http://172.16.16.106:8000); the username and password in Settings -> Audiobooks sign-in are OpenAudible's. Leave empty to go through the audiobook player instead PODCAST_URL: ${PODCAST_URL:-https://podcast.necloud.us/} @@ -48,6 +49,17 @@ services: volumes: - signal-data:/home/.local/share/signal-cli + # Speech to text (the "Transcribe" button on podcasts and audiobooks). faster-whisper behind an OpenAI-compatible API (the speaches project); the model + # is downloaded the first time it is used. CPU only: a one-hour episode takes roughly 5-15 minutes with the small model. Nothing leaves your server. + whisper: + image: ghcr.io/speaches-ai/speaches:latest-cpu + restart: unless-stopped + network_mode: "service:hub" # reached by the hub at 127.0.0.1:8000 and by nothing else + depends_on: + - hub + volumes: + - whisper-models:/home/ubuntu/.cache/huggingface/hub + # Lyrics and karaoke for the music player. Lyrica looks songs up on public lyrics sources (LRCLIB, Musixmatch via Lrcmux, NetEase, Megalobiz); # the song's artist and title are sent to them. Plex's own lyrics are used first when the song has them. See README -> Music. lyrica: @@ -65,3 +77,5 @@ volumes: name: nehub-data # fixed name so the data survives redeploys and renamed stacks signal-data: name: nehub-signal + whisper-models: + name: nehub-whisper diff --git a/podcast.js b/podcast.js index bfae367..92662ab 100644 --- a/podcast.js +++ b/podcast.js @@ -113,4 +113,13 @@ async function handle(req, res) { } } -module.exports = { handle, enabled }; +/** For transcripts: where an episode came from (its feed, its id in that feed, its file). Anyone may ask about the shows on their own list only. */ +async function feedInfo(me, itemId) { + if (!BASE || !/^[\w-]+$/.test(String(itemId))) return null; + const f = (o, k) => o && (o[k] !== undefined ? o[k] : o[k.toLowerCase()]); + const it = await (await podgrab('/podcastitems/' + itemId)).json().catch(() => null); if (!it) return null; + const pid = f(it, 'PodcastID') || f(it, 'podcastId'); if (!pid || !(await mySubs(me)).includes(pid)) return null; + const sh = await (await podgrab('/podcasts/' + pid)).json().catch(() => null); + return { guid: f(it, 'GUID') || f(it, 'guid') || '', fileUrl: f(it, 'FileURL') || f(it, 'fileUrl') || '', feedUrl: (sh && (f(sh, 'URL') || f(sh, 'url'))) || '' }; +} +module.exports = { handle, enabled, feedInfo }; diff --git a/public/lyrics-view.js b/public/lyrics-view.js index 89cba73..0841b21 100644 --- a/public/lyrics-view.js +++ b/public/lyrics-view.js @@ -19,6 +19,8 @@ function lyrParseLrc(text) { lines.forEach((l, i) => { l.e = i + 1 < lines.length ? lines[i + 1].t : l.t + 5000; if (l.words) { const last = l.words[l.words.length - 1]; if (last.e == null) last.e = l.e; } }); return lines; } +const lyrSpokenKey = t => !!t && /^(pod|ab):/.test(String(t.key)); // a podcast episode or an audiobook: words from a transcript instead of lyrics +const lyrLabel = () => lyrSpokenKey(typeof cur === 'function' ? cur() : null) ? 'Transcript' : 'Lyrics'; const lyr = { open: false, cache: new Map(), data: null, key: null, idx: -2, raf: 0, el: null, loading: false }; const lyrPref = () => { try { return { size: 30, karaoke: false, ...JSON.parse(localStorage.getItem('lyr_pref')) }; } catch { return { size: 30, karaoke: false }; } }; const lyrSetPref = p => { try { localStorage.setItem('lyr_pref', JSON.stringify({ ...lyrPref(), ...p })); } catch {} }; @@ -38,6 +40,19 @@ async function lyrFromPlex(t) { // lyrics stored with the song (embedded, a .lrc const lines = text.split(/\r?\n/).map(x => ({ text: x.trim() })); return lines.some(l => l.text) ? { found: true, source: 'Plex', sync: 'none', lines } : null; } catch { return null; } } +/** A transcript: the show's own if it publishes one, else (on request) one made on this server. Polls while that is being made. */ +async function lyrSpoken(t, start) { + const q = new URLSearchParams({ key: t.key, src: t.src || '' }); + for (let n = 0; ; n++) { + if (!lyr.el || lyr.key !== t.key) return null; + let r; try { r = await jfetch('/api/transcript?' + q, start && n === 0 ? { method: 'POST' } : undefined); } catch (e) { return { found: false, error: e.message }; } + if (r.state === 'ready') { r.transcript.spoken = true; return r.transcript; } + if (r.state === 'error') return { found: false, error: r.error, canTranscribe: r.canTranscribe, none: true }; + if (r.state === 'none') return { found: false, none: true, canTranscribe: r.canTranscribe }; + const body = lyr.el.querySelector('#lyrbody'); if (body) body.innerHTML = `

Making a transcript on your server… ${Math.max(0, Math.round((Date.now() - r.since) / 60000))} min so far.
You can close this: it keeps going, and it is kept for next time.

`; + await new Promise(ok => setTimeout(ok, 4000)); + } +} async function lyrLoad(t, force) { if (!force && lyr.cache.has(t.key)) return lyr.cache.get(t.key); let res = force ? null : await lyrFromPlex(t); @@ -64,7 +79,7 @@ function lyrClose() { const lyrKey = e => { if (e.key === 'Escape' && !document.getElementById('sheet')) lyrClose(); }; function lyrShell() { const t = cur(), pref = lyrPref(); if (!t || !lyr.el) return; - lyr.el.className = pref.karaoke ? 'kar' : ''; + lyr.el.className = pref.karaoke && !lyrSpokenKey(t) ? 'kar' : ''; lyr.el.style.setProperty('--lf', pref.size + 'px'); lyr.el.innerHTML = `
${esc(t.title)}${esc(t.artist)}
@@ -76,6 +91,7 @@ function lyrShell() {
`; const $l = q => lyr.el.querySelector(q); + if (lyrSpokenKey(t)) { $l('#lyrk').hidden = true; $l('#lyrs').hidden = true; lyr.el.setAttribute('aria-label', 'Transcript'); } // (karaoke and "wrong lyrics" make no sense for speech) $l('#lyrx').onclick = lyrClose; $l('#lyrpv').onclick = prev; $l('#lyrnx').onclick = next; $l('#lyrt').onclick = toggle; $l('#lyrk').onclick = () => { lyrSetPref({ karaoke: !lyrPref().karaoke }); lyr.el.className = lyrPref().karaoke ? 'kar' : ''; $l('#lyrk').setAttribute('aria-pressed', lyrPref().karaoke); lyr.idx = -2; }; const size = d => { const v = Math.max(18, Math.min(64, lyrPref().size + d)); lyrSetPref({ size: v }); lyr.el.style.setProperty('--lf', v + 'px'); }; @@ -88,14 +104,21 @@ function lyrShell() { async function lyrReload(force) { const t = cur(); if (!t || !lyr.el) return; lyr.key = t.key; lyr.data = null; lyr.idx = -2; const body = lyr.el.querySelector('#lyrbody'); body.innerHTML = '

Looking for the lyrics…

'; - const res = await lyrLoad(t, force); if (!lyr.el || lyr.key !== t.key) return; + const res = lyrSpokenKey(t) ? await lyrSpoken(t, force === 'start') : await lyrLoad(t, force); if (!res || !lyr.el || lyr.key !== t.key) return; lyr.data = res; lyrDraw(); } function lyrDraw() { const t = cur(), d = lyr.data, body = lyr.el && lyr.el.querySelector('#lyrbody'); if (!body || !t) return; const src = lyr.el.querySelector('#lyrsrc'), nudge = lyr.el.querySelector('#lyrnudge'); nudge.hidden = !(d && d.found && d.sync !== 'none'); lyr.el.querySelector('#lyroff').textContent = (lyrOffset(t.key) / 1000).toFixed(1) + ' s'; - if (!d || !d.found) { src.textContent = ''; body.innerHTML = `

${d && d.error ? esc(d.error) : 'No lyrics found for this song.'}

`; return; } + if (!d || !d.found) { + src.textContent = ''; + if (lyrSpokenKey(t)) { // no transcript yet: offer to make one + body.innerHTML = `

${d && d.error ? esc(d.error) : 'This one has no published transcript.'}

${d && d.canTranscribe ? `

Takes roughly a quarter of the running time or more on a CPU, and a long audiobook can take hours. It is kept afterwards.

` : '

An admin can add the whisper service to turn on transcribing.

'}`; + const b = body.querySelector('#lyrtr'); if (b) b.onclick = () => lyrReload('start'); return; + } + body.innerHTML = `

${d && d.error ? esc(d.error) : 'No lyrics found for this song.'}

`; return; + } if (d.instrumental) { src.textContent = ''; body.innerHTML = '

This one is instrumental.

'; return; } src.textContent = (d.source ? d.source + ' · ' : '') + (d.sync === 'word' ? 'word-synced' : d.sync === 'line' ? 'line-synced' : 'not time-synced'); body.innerHTML = d.lines.map((l, i) => l.text || l.words ? `
${l.words ? l.words.map((w, j) => `${esc(w.text)}`).join('') : esc(l.text)}
` : '
').join(''); diff --git a/public/player.js b/public/player.js index 39747ea..053bb1a 100644 --- a/public/player.js +++ b/public/player.js @@ -247,7 +247,7 @@ function renderNpCard() {
0:000:00
-
`; + `; const sk = el.querySelector('#cnk'); sk.oninput = () => { sk.dataset.drag = 1; el.querySelector('#cnc').textContent = fmtDur(sk.value * 1000); }; sk.onchange = () => { pl.a.currentTime = +sk.value; delete sk.dataset.drag; }; @@ -271,7 +271,7 @@ function openNow() {
0:000:00
-
${castOk(pl.a) ? '' : ''}
`); +
${castOk(pl.a) ? '' : ''}
`); const sk = el.querySelector('#npk'); sk.oninput = () => { sk.dataset.drag = 1; el.querySelector('#npc').textContent = fmtDur(sk.value * 1000); }; sk.onchange = () => { pl.a.currentTime = +sk.value; delete sk.dataset.drag; }; diff --git a/public/sw.js b/public/sw.js index 7edc1dc..2ceea3b 100644 --- a/public/sw.js +++ b/public/sw.js @@ -1,5 +1,5 @@ // App-shell cache only; Seafile/API traffic always goes to the network. -const CACHE = 'hub-v28'; +const CACHE = 'hub-v29'; const SHARE = 'hub-share'; // files shared into the app from other apps, held until the person picks a folder const SHELL = ['/icon.svg', '/manifest.webmanifest', '/icons/icon-192.png', '/icons/apple-touch-icon.png']; diff --git a/server.js b/server.js index ae6af5d..6666405 100644 --- a/server.js +++ b/server.js @@ -35,6 +35,7 @@ const admin = require('./admin'); const backup = require('./backup'); const push = require('./push'); const mail = require('./mail'); // Outlook.com mail, through Microsoft Graph (MS_CLIENT_ID) +const transcripts = require('./transcripts'); // words for podcasts and audiobooks: the show's own transcript, or speech-to-text (TRANSCRIBE_API_URL) const lyrics = require('./lyrics'); // lyrics for the music player, through Lyrica (LYRICS_API_URL) const abapi = require('./abapi'); // audiobook player library, audio and progress (AUDIOBOOKS_URL) const podcast = require('./podcast'); // Podgrab podcasts (PODCAST_URL, PODGRAB_PASSWORD) @@ -499,6 +500,7 @@ function storageInfo() { } const STARTED = Date.now(); push.start(); +transcripts.init({ dataDir: path.resolve(process.env.DATA_DIR || path.join(__dirname, 'data')), apiUrl: process.env.TRANSCRIBE_API_URL || '', model: process.env.TRANSCRIBE_MODEL || '', port: PORT }); lyrics.init({ dataDir: path.resolve(process.env.DATA_DIR || path.join(__dirname, 'data')), apiUrl: process.env.LYRICS_API_URL || '' }); signal.init({ dataDir: path.resolve(process.env.DATA_DIR || path.join(__dirname, 'data')), apiUrl: process.env.SIGNAL_API_URL || '' }); { const i = storageInfo(); console.log(`[data] ${i.dir}: ${i.passkeys} passkeys, ${i.calendarLinks} calendar links, ${i.events} events; mounted: ${i.mounted === null ? 'unknown' : i.mounted ? (i.source.kind + ' ' + i.source.name) : 'NO (data is lost on redeploy)'}; writable: ${i.writable}; COOKIE_SECRET from env: ${i.secretFromEnv}`); @@ -539,6 +541,7 @@ const ACCESS = [ [/^\/api\/podcast\//, ['podcast']], [/^\/api\/signal\//, ['signal']], [/^\/api\/lyrics$/, ['music']], + [/^\/api\/transcript$/, ['podcast', 'audio']], [/^\/api\/mail\//, ['mail']], [/^\/api\/cal\//, ['calendar']], [/^\/api\/tasks/, ['tasks']], @@ -596,7 +599,7 @@ async function handleRequest(req, res) { const c = t => users.can(user, t); res.writeHead(200, { 'content-type': 'application/json' }); return res.end(JSON.stringify({ seafile: c('files') || c('office') ? SEAFILE_URL : '', onlyoffice: c('office') ? ONLYOFFICE_URL : '', audiobooks: c('audio') ? AUDIOBOOKS_URL : '', verse: c('verse') ? VERSE_URL : '', - podcast: c('podcast') ? PODCAST_URL : '', signal: c('signal') && signal.enabled() ? '1' : '', lyrics: c('music') && lyrics.enabled() ? '1' : '', mail: c('mail') ? '1' : '', outlook: c('mail') && mail.outlookOn() ? '1' : '', mailAdmin: user.role === 'admin' ? '1' : '', editor: (c('office') || c('files')) && OO_OK, jwt: !!JWT_SECRET, ooReady: OO_OK, build: BUILD })); + podcast: c('podcast') ? PODCAST_URL : '', transcribe: (c('podcast') || c('audio')) && process.env.TRANSCRIBE_API_URL ? '1' : '', signal: c('signal') && signal.enabled() ? '1' : '', lyrics: c('music') && lyrics.enabled() ? '1' : '', mail: c('mail') ? '1' : '', outlook: c('mail') && mail.outlookOn() ? '1' : '', mailAdmin: user.role === 'admin' ? '1' : '', editor: (c('office') || c('files')) && OO_OK, jwt: !!JWT_SECRET, ooReady: OO_OK, build: BUILD })); } if (req.method === 'POST' && req.url === '/api/oo/config') return OO_OK ? ooConfig(req, res) : forbidden(res, 'In-app editing needs ONLYOFFICE_JWT_SECRET to be set on the server'); if (pathname === '/api/backup') return user.role === 'admin' ? backup.handle(req, res, user.name) : forbidden(res, 'Only an admin can do this'); @@ -623,6 +626,7 @@ async function handleRequest(req, res) { const to = audiobooksLogin(user); res.writeHead(302, { location: to || AUDIOBOOKS_URL, 'cache-control': 'no-store', 'referrer-policy': 'no-referrer' }); return res.end(); } + if (pathname === '/api/transcript') return transcripts.handle(req, res); if (pathname === '/api/lyrics') return lyrics.enabled() ? lyrics.handle(req, res) : forbidden(res, 'Lyrics are not set up on this server'); if (pathname.startsWith('/api/ab/')) return abapi.handle(req, res); if (pathname.startsWith('/api/podcast/')) return podcast.handle(req, res); diff --git a/tools/transcript-test.js b/tools/transcript-test.js new file mode 100644 index 0000000..d00def4 --- /dev/null +++ b/tools/transcript-test.js @@ -0,0 +1,42 @@ +// Transcripts: reading SRT/VTT/JSON, finding an episode's transcript link in a feed, refusing private addresses, and speech-to-text through a fake Whisper server. +const http = require('http'), fs = require('fs'), os = require('os'), path = require('path'); +const tr = require('../transcripts'); +let bad = 0; const ok = (c, m) => { console.log((c ? 'PASS ' : 'FAIL ') + m); if (!c) bad++; }; + +(async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trt-')); + const srv = http.createServer((req, res) => { + const body = []; req.on('data', d => body.push(d)); req.on('end', () => { + const b = Buffer.concat(body).toString('latin1'); + if (req.url === '/v1/audio/transcriptions' && req.method === 'POST') { + if (!/response_format/.test(b) || !/faster-whisper-small/.test(b) || !/hello-audio/.test(b)) { res.writeHead(400); return res.end('bad form'); } + res.writeHead(200, { 'content-type': 'application/json' }); return res.end(JSON.stringify({ text: 'x', segments: [{ start: 0, end: 2.5, text: ' Welcome to the show. ' }, { start: 2.5, end: 6, text: ' Today we talk about bread.' }, { start: 6, end: 7, text: ' ' }] })); + } + res.writeHead(404); res.end(); + }); + }); + await new Promise(r => srv.listen(0, '127.0.0.1', r)); + tr.init({ dataDir: dir, apiUrl: 'http://127.0.0.1:' + srv.address().port, model: 'Systran/faster-whisper-small', port: 8080 }); + + const srt = '1\n00:00:01,500 --> 00:00:03,000\nHello there\n\n2\n00:01:02,000 --> 00:01:05,250\nSecond & last\n'; + let l = tr.parseCues(srt); ok(l.length === 2 && l[0].t === 1500 && l[0].text === 'Hello there' && l[1].t === 62000 && l[1].text === 'Second & last', 'SubRip cues are read, tags and entities removed'); + l = tr.parseCues('WEBVTT\n\n00:05.000 --> 00:07.000\nShort form\n\n1:00:00.000 --> 1:00:02.000 align:start\nAn hour in\n'); ok(l.length === 2 && l[0].t === 5000 && l[1].t === 3600000, 'WebVTT cues, short and hour times'); + l = tr.parseJson(JSON.stringify({ version: '1.0.0', segments: [{ startTime: 0, endTime: 4, speaker: 'Ann', body: 'Hi all' }, { startTime: 4.5, body: 'Next' }] })); ok(l.length === 2 && l[0].text === 'Ann: Hi all' && l[1].t === 4500, 'Podcast 2.0 JSON segments'); + ok(tr.parseJson('not json').length === 0, 'broken JSON gives nothing, not a crash'); + const shaped = tr.shape([{ t: 3000, text: 'b' }, { t: 1000, text: 'a' }], 'S'); ok(shaped.lines[0].text === 'a' && shaped.lines[0].e === 3000 && shaped.sync === 'line', 'lines are sorted and given ends'); + + const feed = `g1g2`; + ok(tr.transcriptLinks(feed, 'g1', '')[0].url === 'https://a/b.srt?x=1&y=2', 'a transcript link is found by the episode guid (and & unescaped)'); + ok(tr.transcriptLinks(feed, 'nope', 'https://f/2.mp3')[0].type === 'text/vtt', 'or by the audio file address'); + ok(tr.transcriptLinks(feed, 'zzz', '').length === 0, 'another episode has none'); + + for (const u of ['http://127.0.0.1:1/x', 'http://localhost/x', 'http://192.168.1.5/x', 'http://10.0.0.2/x', 'ftp://example.com/x', 'http://[::1]/x']) { + let e = null; try { await tr.getText(u); } catch (x) { e = x; } ok(e && /not allowed/.test(e.message), 'refuses ' + u); + } + + const audio = path.join(dir, 'a.bin'); fs.writeFileSync(audio, 'hello-audio'); + const r = await tr.whisper(audio, 'audio'); ok(r && r.found && r.lines.length === 2 && r.lines[0].text === 'Welcome to the show.' && r.lines[1].text === 'Today we talk about bread.' && r.source === 'Transcribed on your server', 'speech-to-text segments become timed lines (blank ones dropped, tags removed)'); + ok(r.lines[1].t === 2500 && r.lines[1].e === 2500 + 6000, 'times are in milliseconds'); + srv.close(); fs.rmSync(dir, { recursive: true, force: true }); + console.log(bad ? '\n' + bad + ' check(s) failed' : '\nAll transcript checks passed'); process.exit(bad ? 1 : 0); +})().catch(e => { console.error(e); process.exit(1); }); diff --git a/transcripts.js b/transcripts.js new file mode 100644 index 0000000..67218fa --- /dev/null +++ b/transcripts.js @@ -0,0 +1,120 @@ +// Words for spoken audio (podcasts, audiobooks), shown in the same screen as song lyrics. +// 1. A transcript the show itself publishes (the Podcast 2.0 tag in its feed: SRT, VTT or JSON), fetched on request. +// 2. Otherwise, speech-to-text on this server: an OpenAI-compatible Whisper service (TRANSCRIBE_API_URL, e.g. the bundled faster-whisper/speaches +// container). Only on request, one job at a time; the result is kept on disk, so an episode is transcribed once. +const fs = require('fs'), path = require('path'), os = require('os'), crypto = require('crypto'); +const { Readable } = require('stream'), { pipeline } = require('stream/promises'); +const atomicWrite = require('./atomic'); +const users = require('./users'); +const podcast = require('./podcast'); + +const cfg = { api: '', model: '', dir: '', port: 8080 }; +const MAX_AUDIO = 1.5 * 1024 ** 3, MAX_TEXT = 6 * 1024 * 1024; +function init({ dataDir, apiUrl, model, port }) { cfg.api = (apiUrl || '').replace(/\/$/, ''); cfg.model = model || 'Systran/faster-whisper-small'; cfg.dir = path.join(dataDir, 'transcripts'); cfg.port = port || 8080; fs.mkdirSync(cfg.dir, { recursive: true, mode: 0o700 }); } +const fail = (status, message) => Object.assign(new Error(message), { status }); +const json = (res, code, o) => { res.writeHead(code, { 'content-type': 'application/json', 'cache-control': 'no-store' }); res.end(JSON.stringify(o)); }; + +// ---- reading transcript files +const tms = (h, m, s, f) => (+(h || 0) * 3600 + +m * 60 + +s) * 1000 + Math.round(+('0.' + (f || '0')) * 1000); +const clean = t => String(t || '').replace(/<[^>]+>/g, '').replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"').replace(/'/g, "'").replace(/\s+/g, ' ').trim(); +/** SubRip and WebVTT: "00:01:02,500 --> 00:01:05,000" then the words. */ +function parseCues(text) { + const out = [], re = /(?:(\d+):)?(\d{1,2}):(\d{2})[.,](\d{1,3})\s*-->\s*(?:(\d+):)?(\d{1,2}):(\d{2})[.,](\d{1,3})[^\n]*\n([\s\S]*?)(?=\n\s*\n|$)/g; let m; + while ((m = re.exec(String(text).replace(/\r/g, '')))) { const body = clean(m[9]); if (body) out.push({ t: tms(m[1], m[2], m[3], m[4]), text: body }); } + return out; +} +/** Podcast 2.0 JSON: { segments: [{ startTime, endTime, speaker, body }] } */ +function parseJson(text) { + let j; try { j = JSON.parse(text); } catch { return []; } + return (j.segments || []).map(s => ({ t: Math.round(+s.startTime * 1000), text: (s.speaker ? s.speaker + ': ' : '') + clean(s.body) })).filter(x => x.text && Number.isFinite(x.t)); +} +const withEnds = lines => { lines.sort((a, b) => a.t - b.t); lines.forEach((l, i) => { l.e = i + 1 < lines.length ? lines[i + 1].t : l.t + 6000; }); return lines; }; +const shape = (lines, source) => lines.length ? { found: true, source, sync: 'line', lines: withEnds(lines) } : null; + +// ---- the show's own transcript +const PRIVATE = /^(localhost|.*\.local|.*\.internal|0\.0\.0\.0|127\.|10\.|192\.168\.|169\.254\.|172\.(1[6-9]|2\d|3[01])\.|\[?::1\]?$|\[?f[cd][0-9a-f]{2}:)/i; +async function getText(url, hops = 0) { // only public http(s) addresses; redirects are checked again + const u = new URL(url); if (!/^https?:$/.test(u.protocol) || PRIVATE.test(u.hostname)) throw fail(400, 'That address is not allowed'); + const r = await fetch(u, { redirect: 'manual', signal: AbortSignal.timeout(15000), headers: { 'user-agent': 'NEHub/1 (+transcripts)', accept: 'text/*, application/json, application/xml, */*' } }); + if (r.status >= 300 && r.status < 400 && r.headers.get('location') && hops < 3) return getText(new URL(r.headers.get('location'), u).href, hops + 1); + if (!r.ok) throw fail(502, 'Status ' + r.status); + const len = +r.headers.get('content-length') || 0; if (len > MAX_TEXT) throw fail(413, 'Too large'); + const text = await r.text(); if (text.length > MAX_TEXT) throw fail(413, 'Too large'); return text; +} +const attr = (s, k) => { const m = new RegExp(k + '\\s*=\\s*"([^"]*)"', 'i').exec(s); return m ? m[1].replace(/&/g, '&') : ''; }; +/** The transcript links an episode has in its feed. */ +function transcriptLinks(feedXml, guid, fileUrl) { + const items = String(feedXml).split(/]/i).slice(1); + const unwrap = x => String(x || '').replace(//g, '').trim(); + const item = items.find(b => { const g = /]*>([\s\S]*?)<\/guid>/i.exec(b); return g && guid && unwrap(g[1]) === guid; }) + || items.find(b => { const e = /]*>/i.exec(b); return e && fileUrl && attr(e[0], 'url') === fileUrl; }); + if (!item) return []; + return [...item.matchAll(/]*?)\/?>/gi)].map(m => ({ url: attr(m[1], 'url'), type: attr(m[1], 'type').toLowerCase() })).filter(x => x.url); +} +async function nativeTranscript(info) { + if (!info || !info.feedUrl) return null; + const links = transcriptLinks(await getText(info.feedUrl), info.guid, info.fileUrl); + const rank = l => /json/.test(l.type) ? 0 : /vtt/.test(l.type) ? 1 : /subrip|srt/.test(l.type) ? 2 : 9; + for (const l of links.sort((a, b) => rank(a) - rank(b))) { + if (rank(l) > 2) continue; // (plain HTML or text has no times: not worth showing as a transcript) + try { const t = await getText(l.url), lines = /json/.test(l.type) ? parseJson(t) : parseCues(t); const r = shape(lines, 'Published by the show'); if (r) return r; } catch {} + } + return null; +} + +// ---- speech to text on this server +const jobs = new Map(); // key -> { state: 'running' | 'error', since, error } +let chain = Promise.resolve(); // one at a time: it is heavy +const fileOf = key => path.join(cfg.dir, key + '.json'); +const keyFor = (kind, id, src) => crypto.createHash('sha1').update(kind + '|' + id + '|' + src).digest('hex'); +async function whisper(file, name) { + const form = new FormData(); + form.append('file', await fs.openAsBlob(file), name || 'audio'); + form.append('model', cfg.model); form.append('response_format', 'verbose_json'); form.append('timestamp_granularities[]', 'segment'); + const r = await fetch(cfg.api + '/v1/audio/transcriptions', { method: 'POST', body: form, signal: AbortSignal.timeout(6 * 3600e3) }); + if (!r.ok) throw new Error('The transcription service said ' + r.status + ': ' + (await r.text()).slice(0, 200)); + const j = await r.json(); + return shape((j.segments || []).map(s => ({ t: Math.round(+s.start * 1000), text: clean(s.text) })).filter(x => x.text), 'Transcribed on your server'); +} +/** Bring the audio over from this hub (as the person who asked, with their sign-in) into a temp file. */ +async function fetchAudio(src, cookie, tmp) { + const r = await fetch('http://127.0.0.1:' + cfg.port + src, { headers: { cookie: cookie || '' }, redirect: 'follow', signal: AbortSignal.timeout(2 * 3600e3) }); + if (!r.ok || !r.body) throw new Error('Could not read the audio (' + r.status + ')'); + if ((+r.headers.get('content-length') || 0) > MAX_AUDIO) throw new Error('That audio file is too large to transcribe'); + let n = 0; const body = Readable.fromWeb(r.body); body.on('data', c => { n += c.length; if (n > MAX_AUDIO) body.destroy(new Error('That audio file is too large to transcribe')); }); + await pipeline(body, fs.createWriteStream(tmp, { mode: 0o600 })); +} +function startJob(key, src, cookie) { + jobs.set(key, { state: 'running', since: Date.now() }); + chain = chain.then(async () => { + const tmp = path.join(os.tmpdir(), 'nehub-tr-' + key); + try { await fetchAudio(src, cookie, tmp); const r = await whisper(tmp, 'audio'); if (!r) throw new Error('No speech was found in that audio'); atomicWrite(fileOf(key), JSON.stringify(r)); jobs.delete(key); } + catch (e) { jobs.set(key, { state: 'error', since: Date.now(), error: e.message }); setTimeout(() => jobs.delete(key), 10 * 60e3).unref(); } + finally { fs.rm(tmp, { force: true }, () => {}); } + }); +} + +const SRC_OK = [/^\/api\/podcast\/podcastitems\/[\w-]+\/file$/, /^\/api\/ab\/books\/[^?#\s]+$/]; // only audio this hub already serves to the person +const nativeMiss = new Map(); // key -> time: a show with no transcript is not asked again for a day +async function handle(req, res) { + const u = new URL(req.url, 'http://x'), q = u.searchParams, key0 = String(q.get('key') || ''), src = String(q.get('src') || ''); + const kind = key0.startsWith('pod:') ? 'pod' : key0.startsWith('ab:') ? 'ab' : ''; + if (!kind || key0.length > 200 || !SRC_OK.some(re => re.test(src))) return json(res, 400, { error: 'That cannot be transcribed' }); + const key = keyFor(kind, key0, src), me = users.current(); + const base = { canTranscribe: !!cfg.api }; + try { + if (fs.existsSync(fileOf(key))) return json(res, 200, { state: 'ready', ...base, transcript: JSON.parse(fs.readFileSync(fileOf(key), 'utf8')) }); + const job = jobs.get(key); if (job) return json(res, 200, { state: job.state, since: job.since, error: job.error, ...base }); + if (req.method === 'POST') { + if (!cfg.api) return json(res, 400, { error: 'Transcribing is not set up on this server (see the whisper service in docker-compose.yml)' }); + startJob(key, src, req.headers.cookie); return json(res, 200, { state: 'running', since: Date.now(), ...base }); + } + if (kind === 'pod' && !(nativeMiss.get(key) > Date.now() - 864e5)) { + let r = null; try { r = await nativeTranscript(await podcast.feedInfo(me, key0.slice(4))); } catch {} + if (r) { atomicWrite(fileOf(key), JSON.stringify(r)); return json(res, 200, { state: 'ready', ...base, transcript: r }); } + nativeMiss.set(key, Date.now()); + } + return json(res, 200, { state: 'none', ...base }); + } catch (e) { json(res, e.status || 500, { error: e.message }); } +} +module.exports = { init, handle, parseCues, parseJson, transcriptLinks, shape, whisper, getText, nativeTranscript, enabled: () => true };