Transcripts for podcasts and audiobooks: the show's own, or speech-to-text on your server (whisper service)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

.gitea/workflows/build.yml

@@ -26,6 +26,7 @@ jobs:
       - run: node tools/plex-test.js    # Plex sign-in stays signed in and repairs itself
       - run: node tools/player-test.js  # music player: queue, shuffle/repeat, stall recovery
       - run: node tools/lyrics-test.js  # lyrics: formats, wrong matches refused, remembered
+      - run: node tools/transcript-test.js # transcripts: SRT/VTT/JSON, a show's own transcript link, private addresses refused, speech-to-text
       - run: node tools/mail-test.js    # Outlook.com mail through Microsoft Graph: connect, refresh, read, send, privacy
       - run: node tools/mail-imap-test.js # Gmail and iCloud mail over IMAP/SMTP: connect, read, send, reply, privacy
       - run: node tools/files-test.js   # the Seafile pass-through: timeouts, retries, closed downloads, uploads

README.md

@@ -109,3 +109,10 @@ On the server:
     cp .env.example .env                # fill in secrets, PUBLIC_URL, MS_CLIENT_ID (for Mail), etc.
     docker compose pull && docker compose up -d
     docker compose logs hub | grep "Setup code"   # first sign-in: create your passkey
+
+
+## Transcripts for podcasts and audiobooks
+The lyrics screen shows a **Transcript** for podcast episodes and audiobooks. If the show publishes its own (Podcast 2.0 `<podcast:transcript>`: SRT, VTT or JSON) it is used
+straight away. Otherwise a **Transcribe on this server** button makes one with speech-to-text: the bundled `whisper` service in `docker-compose.yml` (faster-whisper behind an
+OpenAI-compatible API) is reached at `TRANSCRIBE_API_URL` (default `http://127.0.0.1:8000`), model `TRANSCRIBE_MODEL` (default `Systran/faster-whisper-small`, downloaded on first use).
+CPU only: about a quarter of the running time or more. One job at a time; results are kept in `/data/transcripts`. Remove the service and the variable to switch the button off.

docker-compose.yml

@@ -22,6 +22,7 @@ services:
       MS_CLIENT_ID: ${MS_CLIENT_ID:-}                       # Outlook.com mail (Microsoft app registration, see README -> Mail); leave empty to switch Mail off
       MS_CLIENT_SECRET: ${MS_CLIENT_SECRET:-}
       LYRICS_API_URL: http://127.0.0.1:9999                  # the bundled lyrics service (Lyrica, below, shares this container's network); remove this line and the service to skip it
+      TRANSCRIBE_API_URL: http://127.0.0.1:8000              # the bundled speech-to-text service (whisper, below): makes a transcript for a podcast or audiobook on request. Remove this line and the service to skip it
       SIGNAL_API_URL: http://127.0.0.1:8085                  # the bundled Signal bridge (below, shares this container's network); remove this line and the service to skip Signal
       OPENAUDIBLE_URL: ${OPENAUDIBLE_URL:-}                # audiobooks straight from OpenAudible (e.g. http://172.16.16.106:8000); the username and password in Settings -> Audiobooks sign-in are OpenAudible's. Leave empty to go through the audiobook player instead
       PODCAST_URL: ${PODCAST_URL:-https://podcast.necloud.us/}
@@ -48,6 +49,17 @@ services:
     volumes:
       - signal-data:/home/.local/share/signal-cli
 
+  # Speech to text (the "Transcribe" button on podcasts and audiobooks). faster-whisper behind an OpenAI-compatible API (the speaches project); the model
+  # is downloaded the first time it is used. CPU only: a one-hour episode takes roughly 5-15 minutes with the small model. Nothing leaves your server.
+  whisper:
+    image: ghcr.io/speaches-ai/speaches:latest-cpu
+    restart: unless-stopped
+    network_mode: "service:hub"       # reached by the hub at 127.0.0.1:8000 and by nothing else
+    depends_on:
+      - hub
+    volumes:
+      - whisper-models:/home/ubuntu/.cache/huggingface/hub
+
   # Lyrics and karaoke for the music player. Lyrica looks songs up on public lyrics sources (LRCLIB, Musixmatch via Lrcmux, NetEase, Megalobiz);
   # the song's artist and title are sent to them. Plex's own lyrics are used first when the song has them. See README -> Music.
   lyrica:
@@ -65,3 +77,5 @@ volumes:
     name: nehub-data   # fixed name so the data survives redeploys and renamed stacks
   signal-data:
     name: nehub-signal
+  whisper-models:
+    name: nehub-whisper

podcast.js

@@ -113,4 +113,13 @@ async function handle(req, res) {
   }
 }
 
-module.exports = { handle, enabled };
+/** For transcripts: where an episode came from (its feed, its id in that feed, its file). Anyone may ask about the shows on their own list only. */
+async function feedInfo(me, itemId) {
+  if (!BASE || !/^[\w-]+$/.test(String(itemId))) return null;
+  const f = (o, k) => o && (o[k] !== undefined ? o[k] : o[k.toLowerCase()]);
+  const it = await (await podgrab('/podcastitems/' + itemId)).json().catch(() => null); if (!it) return null;
+  const pid = f(it, 'PodcastID') || f(it, 'podcastId'); if (!pid || !(await mySubs(me)).includes(pid)) return null;
+  const sh = await (await podgrab('/podcasts/' + pid)).json().catch(() => null);
+  return { guid: f(it, 'GUID') || f(it, 'guid') || '', fileUrl: f(it, 'FileURL') || f(it, 'fileUrl') || '', feedUrl: (sh && (f(sh, 'URL') || f(sh, 'url'))) || '' };
+}
+module.exports = { handle, enabled, feedInfo };

public/lyrics-view.js

@@ -19,6 +19,8 @@ function lyrParseLrc(text) {
   lines.forEach((l, i) => { l.e = i + 1 < lines.length ? lines[i + 1].t : l.t + 5000; if (l.words) { const last = l.words[l.words.length - 1]; if (last.e == null) last.e = l.e; } });
   return lines;
 }
+const lyrSpokenKey = t => !!t && /^(pod|ab):/.test(String(t.key)); // a podcast episode or an audiobook: words from a transcript instead of lyrics
+const lyrLabel = () => lyrSpokenKey(typeof cur === 'function' ? cur() : null) ? 'Transcript' : 'Lyrics';
 const lyr = { open: false, cache: new Map(), data: null, key: null, idx: -2, raf: 0, el: null, loading: false };
 const lyrPref = () => { try { return { size: 30, karaoke: false, ...JSON.parse(localStorage.getItem('lyr_pref')) }; } catch { return { size: 30, karaoke: false }; } };
 const lyrSetPref = p => { try { localStorage.setItem('lyr_pref', JSON.stringify({ ...lyrPref(), ...p })); } catch {} };
@@ -38,6 +40,19 @@ async function lyrFromPlex(t) { // lyrics stored with the song (embedded, a .lrc
     const lines = text.split(/\r?\n/).map(x => ({ text: x.trim() })); return lines.some(l => l.text) ? { found: true, source: 'Plex', sync: 'none', lines } : null;
   } catch { return null; }
 }
+/** A transcript: the show's own if it publishes one, else (on request) one made on this server. Polls while that is being made. */
+async function lyrSpoken(t, start) {
+  const q = new URLSearchParams({ key: t.key, src: t.src || '' });
+  for (let n = 0; ; n++) {
+    if (!lyr.el || lyr.key !== t.key) return null;
+    let r; try { r = await jfetch('/api/transcript?' + q, start && n === 0 ? { method: 'POST' } : undefined); } catch (e) { return { found: false, error: e.message }; }
+    if (r.state === 'ready') { r.transcript.spoken = true; return r.transcript; }
+    if (r.state === 'error') return { found: false, error: r.error, canTranscribe: r.canTranscribe, none: true };
+    if (r.state === 'none') return { found: false, none: true, canTranscribe: r.canTranscribe };
+    const body = lyr.el.querySelector('#lyrbody'); if (body) body.innerHTML = `<p class="empty">Making a transcript on your server… ${Math.max(0, Math.round((Date.now() - r.since) / 60000))} min so far.<br><small>You can close this: it keeps going, and it is kept for next time.</small></p>`;
+    await new Promise(ok => setTimeout(ok, 4000));
+  }
+}
 async function lyrLoad(t, force) {
   if (!force && lyr.cache.has(t.key)) return lyr.cache.get(t.key);
   let res = force ? null : await lyrFromPlex(t);
@@ -64,7 +79,7 @@ function lyrClose() {
 const lyrKey = e => { if (e.key === 'Escape' && !document.getElementById('sheet')) lyrClose(); };
 function lyrShell() {
   const t = cur(), pref = lyrPref(); if (!t || !lyr.el) return;
-  lyr.el.className = pref.karaoke ? 'kar' : '';
+  lyr.el.className = pref.karaoke && !lyrSpokenKey(t) ? 'kar' : '';
   lyr.el.style.setProperty('--lf', pref.size + 'px');
   lyr.el.innerHTML = `<div class="lyrbg" style="background-image:url('${px.art(t.thumb, 300)}')"></div>
     <div class="lyrtop"><img class="lyrart" alt="" src="${px.art(t.thumb, 120)}"><div class="grow"><b>${esc(t.title)}</b><small>${esc(t.artist)}</small><small id="lyrsrc"></small></div>
@@ -76,6 +91,7 @@ function lyrShell() {
       <span class="lyrnudge" id="lyrnudge" hidden><button class="ibtn" id="lyrm" aria-label="Lyrics earlier" title="Lyrics too late? Show them earlier">−</button><small id="lyroff">0.0 s</small><button class="ibtn" id="lyrp" aria-label="Lyrics later" title="Lyrics too early? Show them later">+</button></span>
       <button class="ibtn" id="lyrpv" aria-label="Previous">${icon('prev')}</button><button class="ibtn big" id="lyrt" aria-label="Play or pause"></button><button class="ibtn" id="lyrnx" aria-label="Next">${icon('next')}</button></div>`;
   const $l = q => lyr.el.querySelector(q);
+  if (lyrSpokenKey(t)) { $l('#lyrk').hidden = true; $l('#lyrs').hidden = true; lyr.el.setAttribute('aria-label', 'Transcript'); } // (karaoke and "wrong lyrics" make no sense for speech)
   $l('#lyrx').onclick = lyrClose; $l('#lyrpv').onclick = prev; $l('#lyrnx').onclick = next; $l('#lyrt').onclick = toggle;
   $l('#lyrk').onclick = () => { lyrSetPref({ karaoke: !lyrPref().karaoke }); lyr.el.className = lyrPref().karaoke ? 'kar' : ''; $l('#lyrk').setAttribute('aria-pressed', lyrPref().karaoke); lyr.idx = -2; };
   const size = d => { const v = Math.max(18, Math.min(64, lyrPref().size + d)); lyrSetPref({ size: v }); lyr.el.style.setProperty('--lf', v + 'px'); };
@@ -88,14 +104,21 @@ function lyrShell() {
 async function lyrReload(force) {
   const t = cur(); if (!t || !lyr.el) return; lyr.key = t.key; lyr.data = null; lyr.idx = -2;
   const body = lyr.el.querySelector('#lyrbody'); body.innerHTML = '<p class="empty">Looking for the lyrics…</p>';
-  const res = await lyrLoad(t, force); if (!lyr.el || lyr.key !== t.key) return;
+  const res = lyrSpokenKey(t) ? await lyrSpoken(t, force === 'start') : await lyrLoad(t, force); if (!res || !lyr.el || lyr.key !== t.key) return;
   lyr.data = res; lyrDraw();
 }
 function lyrDraw() {
   const t = cur(), d = lyr.data, body = lyr.el && lyr.el.querySelector('#lyrbody'); if (!body || !t) return;
   const src = lyr.el.querySelector('#lyrsrc'), nudge = lyr.el.querySelector('#lyrnudge');
   nudge.hidden = !(d && d.found && d.sync !== 'none'); lyr.el.querySelector('#lyroff').textContent = (lyrOffset(t.key) / 1000).toFixed(1) + ' s';
-  if (!d || !d.found) { src.textContent = ''; body.innerHTML = `<p class="empty">${d && d.error ? esc(d.error) : 'No lyrics found for this song.'}</p>`; return; }
+  if (!d || !d.found) {
+    src.textContent = '';
+    if (lyrSpokenKey(t)) { // no transcript yet: offer to make one
+      body.innerHTML = `<p class="empty">${d && d.error ? esc(d.error) : 'This one has no published transcript.'}</p>${d && d.canTranscribe ? `<div class="bar" style="justify-content:center"><button class="btn" id="lyrtr">Transcribe on this server</button></div><p class="empty"><small>Takes roughly a quarter of the running time or more on a CPU, and a long audiobook can take hours. It is kept afterwards.</small></p>` : '<p class="empty"><small>An admin can add the whisper service to turn on transcribing.</small></p>'}`;
+      const b = body.querySelector('#lyrtr'); if (b) b.onclick = () => lyrReload('start'); return;
+    }
+    body.innerHTML = `<p class="empty">${d && d.error ? esc(d.error) : 'No lyrics found for this song.'}</p>`; return;
+  }
   if (d.instrumental) { src.textContent = ''; body.innerHTML = '<p class="empty">This one is instrumental.</p>'; return; }
   src.textContent = (d.source ? d.source + ' · ' : '') + (d.sync === 'word' ? 'word-synced' : d.sync === 'line' ? 'line-synced' : 'not time-synced');
   body.innerHTML = d.lines.map((l, i) => l.text || l.words ? `<div class="ll" data-i="${i}"><span class="lt">${l.words ? l.words.map((w, j) => `<span class="w" data-j="${j}">${esc(w.text)}</span>`).join('') : esc(l.text)}</span></div>` : '<div class="lgap"></div>').join('');

public/player.js

@@ -247,7 +247,7 @@ function renderNpCard() {
     <div class="npc-seek"><span id="cnc">0:00</span><input type="range" id="cnk" min="0" step="1" value="0" aria-label="Seek"><span id="cnd">0:00</span></div>
     <div class="npc-ctl"><button class="ibtn" id="cns" aria-label="Shuffle">${icon('shuffle')}</button><button class="ibtn" id="cnp" aria-label="Previous">${icon('prev')}</button>
       <button class="ibtn big" id="cnt" aria-label="Play or pause"></button><button class="ibtn" id="cnn" aria-label="Next">${icon('next')}</button><button class="ibtn" id="cnr" aria-label="Repeat">${icon('repeat')}</button>
-      <button class="btn ghost npc-l" id="cnl">Lyrics</button><button class="btn ghost npc-q" id="cnq">Queue · ${Math.max(0, pl.q.length - pl.i - 1)} up next</button><button class="btn ghost npc-q" id="cnz">${esc(sleepLabel())}</button></div>`;
+      <button class="btn ghost npc-l" id="cnl">${lyrLabel()}</button><button class="btn ghost npc-q" id="cnq">Queue · ${Math.max(0, pl.q.length - pl.i - 1)} up next</button><button class="btn ghost npc-q" id="cnz">${esc(sleepLabel())}</button></div>`;
   const sk = el.querySelector('#cnk');
   sk.oninput = () => { sk.dataset.drag = 1; el.querySelector('#cnc').textContent = fmtDur(sk.value * 1000); };
   sk.onchange = () => { pl.a.currentTime = +sk.value; delete sk.dataset.drag; };
@@ -271,7 +271,7 @@ function openNow() {
     <input type="range" id="npk" min="0" step="1" value="0" aria-label="Seek"><div class="times"><span id="npc">0:00</span><span id="npd">0:00</span></div>
     <div class="ctl"><button class="ibtn" id="npp" aria-label="Previous">${icon('prev')}</button><button class="ibtn big" id="npt" aria-label="Play or pause"></button>
     <button class="ibtn" id="npn" aria-label="Next">${icon('next')}</button></div>
-    <div class="bar" style="justify-content:center"><button class="ibtn" id="nps" aria-label="Shuffle">${icon('shuffle')}</button><button class="btn ghost" id="npl">Lyrics</button>${castOk(pl.a) ? '<button class="btn ghost" id="npcast">Cast</button>' : ''}<button class="btn ghost" id="npq">Queue · ${pl.q.length - pl.i - 1} up next</button><button class="ibtn" id="npr" aria-label="Repeat">${icon('repeat')}</button></div></div>`);
+    <div class="bar" style="justify-content:center"><button class="ibtn" id="nps" aria-label="Shuffle">${icon('shuffle')}</button><button class="btn ghost" id="npl">${lyrLabel()}</button>${castOk(pl.a) ? '<button class="btn ghost" id="npcast">Cast</button>' : ''}<button class="btn ghost" id="npq">Queue · ${pl.q.length - pl.i - 1} up next</button><button class="ibtn" id="npr" aria-label="Repeat">${icon('repeat')}</button></div></div>`);
   const sk = el.querySelector('#npk');
   sk.oninput = () => { sk.dataset.drag = 1; el.querySelector('#npc').textContent = fmtDur(sk.value * 1000); };
   sk.onchange = () => { pl.a.currentTime = +sk.value; delete sk.dataset.drag; };

public/sw.js

@@ -1,5 +1,5 @@
 // App-shell cache only; Seafile/API traffic always goes to the network.
-const CACHE = 'hub-v28';
+const CACHE = 'hub-v29';
 const SHARE = 'hub-share'; // files shared into the app from other apps, held until the person picks a folder
 const SHELL = ['/icon.svg', '/manifest.webmanifest', '/icons/icon-192.png', '/icons/apple-touch-icon.png'];
 

server.js

@@ -35,6 +35,7 @@ const admin = require('./admin');
 const backup = require('./backup');
 const push = require('./push');
 const mail = require('./mail');  // Outlook.com mail, through Microsoft Graph (MS_CLIENT_ID)
+const transcripts = require('./transcripts'); // words for podcasts and audiobooks: the show's own transcript, or speech-to-text (TRANSCRIBE_API_URL)
 const lyrics = require('./lyrics');  // lyrics for the music player, through Lyrica (LYRICS_API_URL)
 const abapi = require('./abapi');  // audiobook player library, audio and progress (AUDIOBOOKS_URL)
 const podcast = require('./podcast');  // Podgrab podcasts (PODCAST_URL, PODGRAB_PASSWORD)
@@ -499,6 +500,7 @@ function storageInfo() {
 }
 const STARTED = Date.now();
 push.start();
+transcripts.init({ dataDir: path.resolve(process.env.DATA_DIR || path.join(__dirname, 'data')), apiUrl: process.env.TRANSCRIBE_API_URL || '', model: process.env.TRANSCRIBE_MODEL || '', port: PORT });
 lyrics.init({ dataDir: path.resolve(process.env.DATA_DIR || path.join(__dirname, 'data')), apiUrl: process.env.LYRICS_API_URL || '' });
 signal.init({ dataDir: path.resolve(process.env.DATA_DIR || path.join(__dirname, 'data')), apiUrl: process.env.SIGNAL_API_URL || '' });
 { const i = storageInfo(); console.log(`[data] ${i.dir}: ${i.passkeys} passkeys, ${i.calendarLinks} calendar links, ${i.events} events; mounted: ${i.mounted === null ? 'unknown' : i.mounted ? (i.source.kind + ' ' + i.source.name) : 'NO (data is lost on redeploy)'}; writable: ${i.writable}; COOKIE_SECRET from env: ${i.secretFromEnv}`);
@@ -539,6 +541,7 @@ const ACCESS = [
   [/^\/api\/podcast\//, ['podcast']],
   [/^\/api\/signal\//, ['signal']],
   [/^\/api\/lyrics$/, ['music']],
+  [/^\/api\/transcript$/, ['podcast', 'audio']],
   [/^\/api\/mail\//, ['mail']],
   [/^\/api\/cal\//, ['calendar']],
   [/^\/api\/tasks/, ['tasks']],
@@ -596,7 +599,7 @@ async function handleRequest(req, res) {
       const c = t => users.can(user, t);
       res.writeHead(200, { 'content-type': 'application/json' });
       return res.end(JSON.stringify({ seafile: c('files') || c('office') ? SEAFILE_URL : '', onlyoffice: c('office') ? ONLYOFFICE_URL : '', audiobooks: c('audio') ? AUDIOBOOKS_URL : '', verse: c('verse') ? VERSE_URL : '',
-        podcast: c('podcast') ? PODCAST_URL : '', signal: c('signal') && signal.enabled() ? '1' : '', lyrics: c('music') && lyrics.enabled() ? '1' : '', mail: c('mail') ? '1' : '', outlook: c('mail') && mail.outlookOn() ? '1' : '', mailAdmin: user.role === 'admin' ? '1' : '', editor: (c('office') || c('files')) && OO_OK, jwt: !!JWT_SECRET, ooReady: OO_OK, build: BUILD }));
+        podcast: c('podcast') ? PODCAST_URL : '', transcribe: (c('podcast') || c('audio')) && process.env.TRANSCRIBE_API_URL ? '1' : '', signal: c('signal') && signal.enabled() ? '1' : '', lyrics: c('music') && lyrics.enabled() ? '1' : '', mail: c('mail') ? '1' : '', outlook: c('mail') && mail.outlookOn() ? '1' : '', mailAdmin: user.role === 'admin' ? '1' : '', editor: (c('office') || c('files')) && OO_OK, jwt: !!JWT_SECRET, ooReady: OO_OK, build: BUILD }));
     }
     if (req.method === 'POST' && req.url === '/api/oo/config') return OO_OK ? ooConfig(req, res) : forbidden(res, 'In-app editing needs ONLYOFFICE_JWT_SECRET to be set on the server');
     if (pathname === '/api/backup') return user.role === 'admin' ? backup.handle(req, res, user.name) : forbidden(res, 'Only an admin can do this');
@@ -623,6 +626,7 @@ async function handleRequest(req, res) {
       const to = audiobooksLogin(user);
       res.writeHead(302, { location: to || AUDIOBOOKS_URL, 'cache-control': 'no-store', 'referrer-policy': 'no-referrer' }); return res.end();
     }
+    if (pathname === '/api/transcript') return transcripts.handle(req, res);
     if (pathname === '/api/lyrics') return lyrics.enabled() ? lyrics.handle(req, res) : forbidden(res, 'Lyrics are not set up on this server');
     if (pathname.startsWith('/api/ab/')) return abapi.handle(req, res);
     if (pathname.startsWith('/api/podcast/')) return podcast.handle(req, res);

tools/transcript-test.js

@@ -0,0 +1,42 @@
+// Transcripts: reading SRT/VTT/JSON, finding an episode's transcript link in a feed, refusing private addresses, and speech-to-text through a fake Whisper server.
+const http = require('http'), fs = require('fs'), os = require('os'), path = require('path');
+const tr = require('../transcripts');
+let bad = 0; const ok = (c, m) => { console.log((c ? 'PASS  ' : 'FAIL  ') + m); if (!c) bad++; };
+
+(async () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trt-'));
+  const srv = http.createServer((req, res) => {
+    const body = []; req.on('data', d => body.push(d)); req.on('end', () => {
+      const b = Buffer.concat(body).toString('latin1');
+      if (req.url === '/v1/audio/transcriptions' && req.method === 'POST') {
+        if (!/response_format/.test(b) || !/faster-whisper-small/.test(b) || !/hello-audio/.test(b)) { res.writeHead(400); return res.end('bad form'); }
+        res.writeHead(200, { 'content-type': 'application/json' }); return res.end(JSON.stringify({ text: 'x', segments: [{ start: 0, end: 2.5, text: ' Welcome to the show. ' }, { start: 2.5, end: 6, text: ' Today we talk about <b>bread</b>.' }, { start: 6, end: 7, text: '  ' }] }));
+      }
+      res.writeHead(404); res.end();
+    });
+  });
+  await new Promise(r => srv.listen(0, '127.0.0.1', r));
+  tr.init({ dataDir: dir, apiUrl: 'http://127.0.0.1:' + srv.address().port, model: 'Systran/faster-whisper-small', port: 8080 });
+
+  const srt = '1\n00:00:01,500 --> 00:00:03,000\nHello <i>there</i>\n\n2\n00:01:02,000 --> 00:01:05,250\nSecond &amp; last\n';
+  let l = tr.parseCues(srt); ok(l.length === 2 && l[0].t === 1500 && l[0].text === 'Hello there' && l[1].t === 62000 && l[1].text === 'Second & last', 'SubRip cues are read, tags and entities removed');
+  l = tr.parseCues('WEBVTT\n\n00:05.000 --> 00:07.000\nShort form\n\n1:00:00.000 --> 1:00:02.000 align:start\nAn hour in\n'); ok(l.length === 2 && l[0].t === 5000 && l[1].t === 3600000, 'WebVTT cues, short and hour times');
+  l = tr.parseJson(JSON.stringify({ version: '1.0.0', segments: [{ startTime: 0, endTime: 4, speaker: 'Ann', body: 'Hi all' }, { startTime: 4.5, body: 'Next' }] })); ok(l.length === 2 && l[0].text === 'Ann: Hi all' && l[1].t === 4500, 'Podcast 2.0 JSON segments');
+  ok(tr.parseJson('not json').length === 0, 'broken JSON gives nothing, not a crash');
+  const shaped = tr.shape([{ t: 3000, text: 'b' }, { t: 1000, text: 'a' }], 'S'); ok(shaped.lines[0].text === 'a' && shaped.lines[0].e === 3000 && shaped.sync === 'line', 'lines are sorted and given ends');
+
+  const feed = `<item><guid>g1</guid><podcast:transcript url="https://a/b.srt?x=1&amp;y=2" type="application/x-subrip"/></item><item><guid>g2</guid><enclosure url="https://f/2.mp3" length="1"/><podcast:transcript url="https://a/c.vtt" type="text/vtt"/></item>`;
+  ok(tr.transcriptLinks(feed, 'g1', '')[0].url === 'https://a/b.srt?x=1&y=2', 'a transcript link is found by the episode guid (and & unescaped)');
+  ok(tr.transcriptLinks(feed, 'nope', 'https://f/2.mp3')[0].type === 'text/vtt', 'or by the audio file address');
+  ok(tr.transcriptLinks(feed, 'zzz', '').length === 0, 'another episode has none');
+
+  for (const u of ['http://127.0.0.1:1/x', 'http://localhost/x', 'http://192.168.1.5/x', 'http://10.0.0.2/x', 'ftp://example.com/x', 'http://[::1]/x']) {
+    let e = null; try { await tr.getText(u); } catch (x) { e = x; } ok(e && /not allowed/.test(e.message), 'refuses ' + u);
+  }
+
+  const audio = path.join(dir, 'a.bin'); fs.writeFileSync(audio, 'hello-audio');
+  const r = await tr.whisper(audio, 'audio'); ok(r && r.found && r.lines.length === 2 && r.lines[0].text === 'Welcome to the show.' && r.lines[1].text === 'Today we talk about bread.' && r.source === 'Transcribed on your server', 'speech-to-text segments become timed lines (blank ones dropped, tags removed)');
+  ok(r.lines[1].t === 2500 && r.lines[1].e === 2500 + 6000, 'times are in milliseconds');
+  srv.close(); fs.rmSync(dir, { recursive: true, force: true });
+  console.log(bad ? '\n' + bad + ' check(s) failed' : '\nAll transcript checks passed'); process.exit(bad ? 1 : 0);
+})().catch(e => { console.error(e); process.exit(1); });

transcripts.js

@@ -0,0 +1,120 @@
+// Words for spoken audio (podcasts, audiobooks), shown in the same screen as song lyrics.
+// 1. A transcript the show itself publishes (the Podcast 2.0 <podcast:transcript> tag in its feed: SRT, VTT or JSON), fetched on request.
+// 2. Otherwise, speech-to-text on this server: an OpenAI-compatible Whisper service (TRANSCRIBE_API_URL, e.g. the bundled faster-whisper/speaches
+//    container). Only on request, one job at a time; the result is kept on disk, so an episode is transcribed once.
+const fs = require('fs'), path = require('path'), os = require('os'), crypto = require('crypto');
+const { Readable } = require('stream'), { pipeline } = require('stream/promises');
+const atomicWrite = require('./atomic');
+const users = require('./users');
+const podcast = require('./podcast');
+
+const cfg = { api: '', model: '', dir: '', port: 8080 };
+const MAX_AUDIO = 1.5 * 1024 ** 3, MAX_TEXT = 6 * 1024 * 1024;
+function init({ dataDir, apiUrl, model, port }) { cfg.api = (apiUrl || '').replace(/\/$/, ''); cfg.model = model || 'Systran/faster-whisper-small'; cfg.dir = path.join(dataDir, 'transcripts'); cfg.port = port || 8080; fs.mkdirSync(cfg.dir, { recursive: true, mode: 0o700 }); }
+const fail = (status, message) => Object.assign(new Error(message), { status });
+const json = (res, code, o) => { res.writeHead(code, { 'content-type': 'application/json', 'cache-control': 'no-store' }); res.end(JSON.stringify(o)); };
+
+// ---- reading transcript files
+const tms = (h, m, s, f) => (+(h || 0) * 3600 + +m * 60 + +s) * 1000 + Math.round(+('0.' + (f || '0')) * 1000);
+const clean = t => String(t || '').replace(/<[^>]+>/g, '').replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/\s+/g, ' ').trim();
+/** SubRip and WebVTT: "00:01:02,500 --> 00:01:05,000" then the words. */
+function parseCues(text) {
+  const out = [], re = /(?:(\d+):)?(\d{1,2}):(\d{2})[.,](\d{1,3})\s*-->\s*(?:(\d+):)?(\d{1,2}):(\d{2})[.,](\d{1,3})[^\n]*\n([\s\S]*?)(?=\n\s*\n|$)/g; let m;
+  while ((m = re.exec(String(text).replace(/\r/g, '')))) { const body = clean(m[9]); if (body) out.push({ t: tms(m[1], m[2], m[3], m[4]), text: body }); }
+  return out;
+}
+/** Podcast 2.0 JSON: { segments: [{ startTime, endTime, speaker, body }] } */
+function parseJson(text) {
+  let j; try { j = JSON.parse(text); } catch { return []; }
+  return (j.segments || []).map(s => ({ t: Math.round(+s.startTime * 1000), text: (s.speaker ? s.speaker + ': ' : '') + clean(s.body) })).filter(x => x.text && Number.isFinite(x.t));
+}
+const withEnds = lines => { lines.sort((a, b) => a.t - b.t); lines.forEach((l, i) => { l.e = i + 1 < lines.length ? lines[i + 1].t : l.t + 6000; }); return lines; };
+const shape = (lines, source) => lines.length ? { found: true, source, sync: 'line', lines: withEnds(lines) } : null;
+
+// ---- the show's own transcript
+const PRIVATE = /^(localhost|.*\.local|.*\.internal|0\.0\.0\.0|127\.|10\.|192\.168\.|169\.254\.|172\.(1[6-9]|2\d|3[01])\.|\[?::1\]?$|\[?f[cd][0-9a-f]{2}:)/i;
+async function getText(url, hops = 0) { // only public http(s) addresses; redirects are checked again
+  const u = new URL(url); if (!/^https?:$/.test(u.protocol) || PRIVATE.test(u.hostname)) throw fail(400, 'That address is not allowed');
+  const r = await fetch(u, { redirect: 'manual', signal: AbortSignal.timeout(15000), headers: { 'user-agent': 'NEHub/1 (+transcripts)', accept: 'text/*, application/json, application/xml, */*' } });
+  if (r.status >= 300 && r.status < 400 && r.headers.get('location') && hops < 3) return getText(new URL(r.headers.get('location'), u).href, hops + 1);
+  if (!r.ok) throw fail(502, 'Status ' + r.status);
+  const len = +r.headers.get('content-length') || 0; if (len > MAX_TEXT) throw fail(413, 'Too large');
+  const text = await r.text(); if (text.length > MAX_TEXT) throw fail(413, 'Too large'); return text;
+}
+const attr = (s, k) => { const m = new RegExp(k + '\\s*=\\s*"([^"]*)"', 'i').exec(s); return m ? m[1].replace(/&amp;/g, '&') : ''; };
+/** The transcript links an episode has in its feed. */
+function transcriptLinks(feedXml, guid, fileUrl) {
+  const items = String(feedXml).split(/<item[\s>]/i).slice(1);
+  const unwrap = x => String(x || '').replace(/<!\[CDATA\[|\]\]>/g, '').trim();
+  const item = items.find(b => { const g = /<guid[^>]*>([\s\S]*?)<\/guid>/i.exec(b); return g && guid && unwrap(g[1]) === guid; })
+    || items.find(b => { const e = /<enclosure[^>]*>/i.exec(b); return e && fileUrl && attr(e[0], 'url') === fileUrl; });
+  if (!item) return [];
+  return [...item.matchAll(/<podcast:transcript\s+([^>]*?)\/?>/gi)].map(m => ({ url: attr(m[1], 'url'), type: attr(m[1], 'type').toLowerCase() })).filter(x => x.url);
+}
+async function nativeTranscript(info) {
+  if (!info || !info.feedUrl) return null;
+  const links = transcriptLinks(await getText(info.feedUrl), info.guid, info.fileUrl);
+  const rank = l => /json/.test(l.type) ? 0 : /vtt/.test(l.type) ? 1 : /subrip|srt/.test(l.type) ? 2 : 9;
+  for (const l of links.sort((a, b) => rank(a) - rank(b))) {
+    if (rank(l) > 2) continue; // (plain HTML or text has no times: not worth showing as a transcript)
+    try { const t = await getText(l.url), lines = /json/.test(l.type) ? parseJson(t) : parseCues(t); const r = shape(lines, 'Published by the show'); if (r) return r; } catch {}
+  }
+  return null;
+}
+
+// ---- speech to text on this server
+const jobs = new Map(); // key -> { state: 'running' | 'error', since, error }
+let chain = Promise.resolve(); // one at a time: it is heavy
+const fileOf = key => path.join(cfg.dir, key + '.json');
+const keyFor = (kind, id, src) => crypto.createHash('sha1').update(kind + '|' + id + '|' + src).digest('hex');
+async function whisper(file, name) {
+  const form = new FormData();
+  form.append('file', await fs.openAsBlob(file), name || 'audio');
+  form.append('model', cfg.model); form.append('response_format', 'verbose_json'); form.append('timestamp_granularities[]', 'segment');
+  const r = await fetch(cfg.api + '/v1/audio/transcriptions', { method: 'POST', body: form, signal: AbortSignal.timeout(6 * 3600e3) });
+  if (!r.ok) throw new Error('The transcription service said ' + r.status + ': ' + (await r.text()).slice(0, 200));
+  const j = await r.json();
+  return shape((j.segments || []).map(s => ({ t: Math.round(+s.start * 1000), text: clean(s.text) })).filter(x => x.text), 'Transcribed on your server');
+}
+/** Bring the audio over from this hub (as the person who asked, with their sign-in) into a temp file. */
+async function fetchAudio(src, cookie, tmp) {
+  const r = await fetch('http://127.0.0.1:' + cfg.port + src, { headers: { cookie: cookie || '' }, redirect: 'follow', signal: AbortSignal.timeout(2 * 3600e3) });
+  if (!r.ok || !r.body) throw new Error('Could not read the audio (' + r.status + ')');
+  if ((+r.headers.get('content-length') || 0) > MAX_AUDIO) throw new Error('That audio file is too large to transcribe');
+  let n = 0; const body = Readable.fromWeb(r.body); body.on('data', c => { n += c.length; if (n > MAX_AUDIO) body.destroy(new Error('That audio file is too large to transcribe')); });
+  await pipeline(body, fs.createWriteStream(tmp, { mode: 0o600 }));
+}
+function startJob(key, src, cookie) {
+  jobs.set(key, { state: 'running', since: Date.now() });
+  chain = chain.then(async () => {
+    const tmp = path.join(os.tmpdir(), 'nehub-tr-' + key);
+    try { await fetchAudio(src, cookie, tmp); const r = await whisper(tmp, 'audio'); if (!r) throw new Error('No speech was found in that audio'); atomicWrite(fileOf(key), JSON.stringify(r)); jobs.delete(key); }
+    catch (e) { jobs.set(key, { state: 'error', since: Date.now(), error: e.message }); setTimeout(() => jobs.delete(key), 10 * 60e3).unref(); }
+    finally { fs.rm(tmp, { force: true }, () => {}); }
+  });
+}
+
+const SRC_OK = [/^\/api\/podcast\/podcastitems\/[\w-]+\/file$/, /^\/api\/ab\/books\/[^?#\s]+$/]; // only audio this hub already serves to the person
+const nativeMiss = new Map(); // key -> time: a show with no transcript is not asked again for a day
+async function handle(req, res) {
+  const u = new URL(req.url, 'http://x'), q = u.searchParams, key0 = String(q.get('key') || ''), src = String(q.get('src') || '');
+  const kind = key0.startsWith('pod:') ? 'pod' : key0.startsWith('ab:') ? 'ab' : '';
+  if (!kind || key0.length > 200 || !SRC_OK.some(re => re.test(src))) return json(res, 400, { error: 'That cannot be transcribed' });
+  const key = keyFor(kind, key0, src), me = users.current();
+  const base = { canTranscribe: !!cfg.api };
+  try {
+    if (fs.existsSync(fileOf(key))) return json(res, 200, { state: 'ready', ...base, transcript: JSON.parse(fs.readFileSync(fileOf(key), 'utf8')) });
+    const job = jobs.get(key); if (job) return json(res, 200, { state: job.state, since: job.since, error: job.error, ...base });
+    if (req.method === 'POST') {
+      if (!cfg.api) return json(res, 400, { error: 'Transcribing is not set up on this server (see the whisper service in docker-compose.yml)' });
+      startJob(key, src, req.headers.cookie); return json(res, 200, { state: 'running', since: Date.now(), ...base });
+    }
+    if (kind === 'pod' && !(nativeMiss.get(key) > Date.now() - 864e5)) {
+      let r = null; try { r = await nativeTranscript(await podcast.feedInfo(me, key0.slice(4))); } catch {}
+      if (r) { atomicWrite(fileOf(key), JSON.stringify(r)); return json(res, 200, { state: 'ready', ...base, transcript: r }); }
+      nativeMiss.set(key, Date.now());
+    }
+    return json(res, 200, { state: 'none', ...base });
+  } catch (e) { json(res, e.status || 500, { error: e.message }); }
+}
+module.exports = { init, handle, parseCues, parseJson, transcriptLinks, shape, whisper, getText, nativeTranscript, enabled: () => true };