hub/podcast.js

// Podcasts from Podgrab (https://github.com/akhilrex/podgrab). The hub talks to Podgrab's own web API on the person's behalf, so the
// Podgrab password (PODGRAB_PASSWORD, user "podgrab") stays on the server and the browser only ever talks to the hub.
const fs = require('fs'), path = require('path');
const users = require('./users');
const prog = require('./progress')('podcast-progress.json'); // where each person got to in each episode
const BASE = (process.env.PODCAST_URL || 'https://podcast.necloud.us/').replace(/\/$/, '');
const AUTH = process.env.PODGRAB_PASSWORD ? 'Basic ' + Buffer.from('podgrab:' + process.env.PODGRAB_PASSWORD).toString('base64') : '';

// Only these Podgrab calls are passed through: [method, path pattern]. Podgrab's page routes and its settings, backup and OPML routes are not.
const ALLOWED = [
  ['GET', /^\/podcasts$/], ['POST', /^\/podcasts$/],
  ['GET', /^\/podcasts\/[\w-]+$/], ['DELETE', /^\/podcasts\/[\w-]+$/],
  ['GET', /^\/podcasts\/[\w-]+\/(items|image|download|pause|unpause)$/], ['DELETE', /^\/podcasts\/[\w-]+\/items$/],
  ['GET', /^\/podcastitems$/], ['GET', /^\/podcastitems\/[\w-]+$/], ['PATCH', /^\/podcastitems\/[\w-]+$/],
  ['GET', /^\/podcastitems\/[\w-]+\/(image|file|markPlayed|markUnplayed|bookmark|unbookmark|download|delete)$/],
];

// Podgrab holds every show in one shared library; which of them are on a person's own list is kept here, per person. A new person starts with an empty
// list (the admin starts with everything already in Podgrab). Taking a show off your list leaves Podgrab alone unless nobody else follows it.
const subsFile = uid => path.join(users.dirOf(uid), 'podcast-subs.json');
const subsRead = uid => { try { const j = JSON.parse(fs.readFileSync(subsFile(uid), 'utf8')); return Array.isArray(j) ? j : null; } catch { return null; } };
const subsWrite = (uid, ids) => { const f = subsFile(uid), tmp = f + '.tmp'; fs.writeFileSync(tmp, JSON.stringify([...new Set(ids)]), { mode: 0o600 }); fs.renameSync(tmp, f); };

// ---- Podcast Index (https://podcastindex.org): a free, open directory searched by topic as well as name. Needs a free key and secret (PODCASTINDEX_KEY, PODCASTINDEX_SECRET).
const PI_URL = (process.env.PODCASTINDEX_URL || 'https://api.podcastindex.org/api/1.0').replace(/\/$/, '');
const PI_TOPICS = ['News', 'Technology', 'Business', 'Science', 'History', 'Health', 'Religion', 'Education', 'Comedy', 'Sports', 'Music', 'Society'];
const piOn = () => !!(process.env.PODCASTINDEX_KEY && process.env.PODCASTINDEX_SECRET);
async function piGet(path) {
  const date = String(Math.floor(Date.now() / 1000)), sig = require('crypto').createHash('sha1').update(process.env.PODCASTINDEX_KEY + process.env.PODCASTINDEX_SECRET + date).digest('hex');
  const r = await fetch(PI_URL + path, { headers: { 'user-agent': 'NE-Hub/1.0', 'x-auth-key': process.env.PODCASTINDEX_KEY, 'x-auth-date': date, authorization: sig }, signal: AbortSignal.timeout(10000) });
  if (!r.ok) throw new Error('status ' + r.status); return r.json();
}
const piShape = f => ({ title: f.title, author: f.author || f.ownerName || '', feedUrl: f.url, image: f.image || f.artwork || '' });
const piKeep = f => f && /^https?:\/\//i.test(f.url || '') && !f.dead;
const piSearch = async q => ((await piGet('/search/byterm?max=20&q=' + encodeURIComponent(q))).feeds || []).filter(piKeep).map(piShape);
const piTrending = async cat => ((await piGet('/podcasts/trending?max=20&lang=en&cat=' + encodeURIComponent(cat))).feeds || []).filter(piKeep).map(piShape);
async function appleSearch(q) {
  const r = await fetch((process.env.APPLE_PODCASTS_URL || 'https://itunes.apple.com/search') + '?' + new URLSearchParams({ media: 'podcast', limit: '15', term: q }), { signal: AbortSignal.timeout(10000) });
  if (!r.ok) throw new Error('status ' + r.status);
  return ((await r.json()).results || []).filter(x => /^https?:\/\//i.test(x.feedUrl || '')).map(x => ({ title: x.collectionName, author: x.artistName, feedUrl: x.feedUrl, image: x.artworkUrl100 }));
}
const idOf = o => o && (o.ID !== undefined ? o.ID : o.id);
async function podgrab(p, init = {}) {
  const r = await fetch(BASE + p, { ...init, headers: { accept: 'application/json', ...(AUTH ? { authorization: AUTH } : {}), ...init.headers }, redirect: 'manual', signal: AbortSignal.timeout(20000) });
  return r;
}
async function mySubs(me) {
  const have = subsRead(me.id); if (have) return have;
  let ids = [];
  if (me.role === 'admin') { try { const l = await (await podgrab('/podcasts')).json(); ids = (Array.isArray(l) ? l : []).map(idOf).filter(Boolean); } catch { return []; } }
  subsWrite(me.id, ids); return ids;
}
const json = (res, code, o) => { res.writeHead(code, { 'content-type': 'application/json', 'cache-control': 'no-store' }); res.end(JSON.stringify(o)); };
const enabled = () => !!BASE;

async function handle(req, res) {
  const u = new URL(req.url, 'http://x'), p = u.pathname.slice('/api/podcast'.length) || '/';
  if (p === '/discover') { // find new shows: Podcast Index (free, with a key) and Apple's free directory (the search words go to them; nothing else does)
    const q = (u.searchParams.get('q') || '').trim().slice(0, 100), topic = (u.searchParams.get('topic') || '').trim().slice(0, 40);
    if (topic && !piOn()) return json(res, 200, []);
    if (!topic && q.length < 2) return json(res, 200, []);
    const [pi, ap] = await Promise.allSettled([piOn() ? (topic ? piTrending(topic) : piSearch(q)) : Promise.resolve([]), topic ? Promise.resolve([]) : appleSearch(q)]);
    if (pi.status === 'rejected' && ap.status === 'rejected') return json(res, 502, { error: 'The podcast directory did not answer' });
    const seen = new Set(), out = [];
    for (const x of [...(pi.value || []), ...(ap.value || [])]) { const k = x.feedUrl.replace(/\/+$/, '').toLowerCase(); if (!seen.has(k)) { seen.add(k); out.push(x); } }
    return json(res, 200, out.slice(0, 30));
  }
  if (p === '/discover/topics') return json(res, 200, piOn() ? PI_TOPICS : []);
  if (p === '/progress') {
    if (req.method === 'GET') return json(res, 200, prog.load());
    if (req.method !== 'POST') return json(res, 405, { error: 'Method not allowed' });
    const chunks = []; let n = 0; for await (const c of req) { n += c.length; if (n > 1e6) return json(res, 413, { error: 'Too large' }); chunks.push(c); }
    let body; try { body = JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}'); } catch { return json(res, 400, { error: 'Bad request' }); }
    return json(res, 200, prog.merge(body));
  }
  if (!BASE) return json(res, 503, { error: 'Podcasts are not set up on this server' });
  if (p === '/status') return json(res, 200, { ok: true, host: new URL(BASE).host });
  if (!ALLOWED.some(([m, re]) => m === req.method && re.test(p))) return json(res, 404, { error: 'Not found' });
  const me = users.current(), mine = await mySubs(me);
  let m;
  try {
    if (req.method === 'GET' && p === '/podcasts') { // only the shows on this person's list
      const r = await podgrab('/podcasts' + u.search); const l = await r.json();
      return json(res, r.status, Array.isArray(l) ? l.filter(x => mine.includes(idOf(x))) : l);
    }
    if (req.method === 'POST' && p === '/podcasts') { // add to Podgrab (if it is not there yet) and to this person's list
      const chunks = []; let n = 0; for await (const c of req) { n += c.length; if (n > 1e5) return json(res, 413, { error: 'Too large' }); chunks.push(c); }
      let url = ''; try { url = String(JSON.parse(Buffer.concat(chunks).toString('utf8')).url || '').trim(); } catch {}
      if (!/^https?:\/\//i.test(url)) return json(res, 400, { error: 'Enter a feed address starting with http' });
      const r = await podgrab('/podcasts', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ url }) });
      let added = null; if (r.status === 200) { try { added = await r.json(); } catch {} }
      let id = idOf(added);
      if (!id) { // Podgrab already had it (409), or did not say which: find it by its feed address
        const l = await (await podgrab('/podcasts')).json(); const norm = x => String(x || '').replace(/\/+$/, '').toLowerCase();
        const hit = (Array.isArray(l) ? l : []).find(x => norm(x.URL || x.url) === norm(url)); id = idOf(hit); added = hit;
        if (!id) return json(res, r.status === 200 ? 502 : r.status, { error: 'Podgrab could not add that feed' });
      }
      subsWrite(me.id, [...mine, id]); return json(res, 200, added || { ID: id });
    }
    if (req.method === 'DELETE' && (m = /^\/podcasts\/([\w-]+)$/.exec(p))) { // off this person's list; out of Podgrab only when nobody else follows it
      subsWrite(me.id, mine.filter(x => x !== m[1]));
      const others = users.all().some(x => x.id !== me.id && (subsRead(x.id) || []).includes(m[1]));
      if (!others) await podgrab(p, { method: 'DELETE' }).catch(() => {});
      return json(res, 200, { ok: true, removed: !others });
    }
    if ((m = /^\/podcasts\/([\w-]+)(\/|$)/.exec(p)) && !mine.includes(m[1])) return json(res, 404, { error: 'That podcast is not on your list' });
  } catch (e) { return json(res, 502, { error: 'Podgrab did not answer' + (e.name === 'TimeoutError' ? ' in time' : '') }); }
  const hasBody = req.method === 'POST' || req.method === 'PATCH';
  let body;
  if (hasBody) { const chunks = []; let n = 0; for await (const c of req) { n += c.length; if (n > 1e5) return json(res, 413, { error: 'Too large' }); chunks.push(c); } body = Buffer.concat(chunks); }
  const h = { accept: req.headers.accept || '*/*', 'accept-encoding': 'identity' }; // so the length Podgrab reports is the length we pass on
  if (AUTH) h.authorization = AUTH;
  if (req.headers.range) h.range = req.headers.range;
  if (hasBody) h['content-type'] = 'application/json';
  const ctl = new AbortController(), streaming = p.endsWith('/file'), t = streaming ? null : setTimeout(() => ctl.abort(), 20000); // audio may take as long as it takes
  res.on('close', () => { clearTimeout(t); ctl.abort(); });
  try {
    const up = await fetch(BASE + p + u.search, { method: req.method, headers: h, body, signal: ctl.signal, redirect: 'manual' });
    const out = { 'cache-control': p.endsWith('/image') ? 'private, max-age=86400' : 'no-store' };
    for (const k of ['content-type', 'content-length', 'content-range', 'accept-ranges']) if (up.headers.get(k)) out[k] = up.headers.get(k);
    if (up.status >= 300 && up.status < 400) { // Podgrab sends you to the original file or picture when it has no copy of its own
      const loc = up.headers.get('location');
      if (loc && /^https?:\/\//i.test(loc)) { res.writeHead(302, { location: loc, 'referrer-policy': 'no-referrer' }); return res.end(); }
      return json(res, 502, { error: 'Podgrab sent an unexpected redirect' });
    }
    res.writeHead(up.status, out);
    if (!up.body) return res.end();
    const { Readable } = require('stream');
    Readable.fromWeb(up.body).on('error', () => res.destroy()).pipe(res);
  } catch (e) {
    if (res.headersSent) return res.destroy();
    json(res, 502, { error: 'Podgrab did not answer' + (e.name === 'AbortError' ? ' in time' : '') });
  }
}

/** For transcripts: where an episode came from (its feed, its id in that feed, its file). With a person, only the shows on their own list; without one (the server's own background work), any. */
async function feedInfo(me, itemId) {
  if (!BASE || !/^[\w-]+$/.test(String(itemId))) return null;
  const f = (o, k) => { if (!o) return undefined; const hit = Object.keys(o).find(x => x.toLowerCase() === k.toLowerCase()); return hit === undefined ? undefined : o[hit]; }; // (Podgrab's field names, whatever their capitals)
  const it = await (await podgrab('/podcastitems/' + itemId)).json().catch(() => null); if (!it) return null;
  const pid = f(it, 'PodcastID') || f(it, 'podcastId'); if (!pid || (me && !(await mySubs(me)).includes(pid))) return null;
  const sh = await (await podgrab('/podcasts/' + pid)).json().catch(() => null);
  return { guid: f(it, 'GUID') || f(it, 'guid') || '', fileUrl: f(it, 'FileURL') || f(it, 'fileUrl') || '', feedUrl: (sh && (f(sh, 'URL') || f(sh, 'url'))) || '' };
}
/** The newest `n` episodes of every show in Podgrab, newest first overall: [{ id, title, show, guid, fileUrl, feedUrl }]. */
async function recentEpisodes(n) {
  if (!BASE) return [];
  const f = (o, k) => { const hit = Object.keys(o || {}).find(x => x.toLowerCase() === k.toLowerCase()); return hit === undefined ? undefined : o[hit]; };
  const shows = await (await podgrab('/podcasts')).json().catch(() => []), out = [];
  for (const sh of Array.isArray(shows) ? shows : []) {
    const sid = idOf(sh); if (!sid) continue;
    const items = await (await podgrab('/podcasts/' + sid + '/items')).json().catch(() => []);
    for (const it of (Array.isArray(items) ? items : []).sort((x, y) => String(f(y, 'PubDate')).localeCompare(String(f(x, 'PubDate')))).slice(0, n))
      out.push({ id: String(idOf(it)), title: f(it, 'Title') || '', show: f(sh, 'Title') || '', date: String(f(it, 'PubDate') || ''), guid: f(it, 'GUID') || '', fileUrl: f(it, 'FileURL') || '', feedUrl: f(sh, 'URL') || '' });
  }
  return out.sort((x, y) => y.date.localeCompare(x.date));
}
/** The audio of one episode, straight from Podgrab (for the server's own background work). */
async function episodeAudio(itemId) {
  if (!BASE || !/^[\w-]+$/.test(String(itemId))) throw new Error('Bad episode');
  const r = await podgrab('/podcastitems/' + itemId + '/file'), loc = r.headers.get('location');
  if (r.status >= 300 && r.status < 400 && loc && /^https?:\/\//i.test(loc)) return fetch(loc, { signal: AbortSignal.timeout(2 * 3600e3) }); // (Podgrab has no copy of its own: the original)
  return r;
}
/** Every episode of the shows on this person's list: [{ id, title, show }], remembered for a few minutes (the transcript search uses it to know what they may see). */
const epsCache = new Map();
async function myEpisodes(me) {
  const c = epsCache.get(me.id); if (c && Date.now() - c.t < 300000) return c.v;
  const f = (o, k) => { const hit = Object.keys(o || {}).find(x => x.toLowerCase() === k.toLowerCase()); return hit === undefined ? undefined : o[hit]; }, v = [];
  if (BASE) for (const sid of await mySubs(me)) {
    const sh = await (await podgrab('/podcasts/' + sid)).json().catch(() => null), items = await (await podgrab('/podcasts/' + sid + '/items')).json().catch(() => []);
    for (const it of Array.isArray(items) ? items : []) v.push({ id: String(idOf(it)), title: f(it, 'Title') || '', show: (sh && f(sh, 'Title')) || '', thumb: '/api/podcast/podcasts/' + sid + '/image' });
  }
  epsCache.set(me.id, { t: Date.now(), v }); return v;
}
module.exports = { handle, myEpisodes, enabled, feedInfo, recentEpisodes, episodeAudio };