From bc2eb7307ff30506b6529d8bab1b295c805f3ed3 Mon Sep 17 00:00:00 2001
From: nmemmert <67961292+nmemmert@users.noreply.github.com>
Date: Tue, 18 Aug 2026 15:34:10 -0400
Subject: [PATCH] =?UTF-8?q?perf:=20indexes,=20query=20rewrites,=20caching?=
 =?UTF-8?q?=20=E2=80=94=20fix=20slow=20load=20at=2030k+=20photos?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Expression index on strftime('%m-%d', date_taken) makes Memories instant
- Composite indexes: (deleted, date_taken), (deleted, starred), (deleted, media_type),
  (deleted, place_city), (deleted, camera_model), (collection_id, photo_id)
- Albums: replace window-function queries with GROUP BY + MAX(id) CTE — avoids
  materialising 30k rows × 3 times just to pick cover photos
- Library stats + Albums: 30s/60s in-memory cache, invalidated on any write
- Videos: cursor-based pagination (100/page) with IntersectionObserver infinite scroll
- Fix build error: mixed ?? and || operators in MemoriesView track count display

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 client/src/api/photos.js               |   9 +-
 client/src/components/MemoriesView.jsx |   2 +-
 client/src/components/VideosView.jsx   |  71 +++++++++---
 server/index.js                        | 143 ++++++++++++++++++-------
 worker/db.js                           |  13 +++
 5 files changed, 180 insertions(+), 58 deletions(-)

diff --git a/client/src/api/photos.js b/client/src/api/photos.js
index aeba49d..e81aa15 100644
--- a/client/src/api/photos.js
+++ b/client/src/api/photos.js
@@ -115,11 +115,12 @@ export function fetchVideoStats() {
   return api('/api/videos/stats');
 }
 
-export function fetchVideos({ from, to, city } = {}) {
+export function fetchVideos({ from, to, city, cursor } = {}) {
   const p = new URLSearchParams();
-  if (from) p.set('from', from);
-  if (to)   p.set('to',   to);
-  if (city) p.set('city', city);
+  if (from)   p.set('from',   from);
+  if (to)     p.set('to',     to);
+  if (city)   p.set('city',   city);
+  if (cursor) p.set('cursor', cursor);
   const qs = p.toString();
   return api(`/api/videos${qs ? '?' + qs : ''}`);
 }
diff --git a/client/src/components/MemoriesView.jsx b/client/src/components/MemoriesView.jsx
index b5f4017..d00ccb3 100644
--- a/client/src/components/MemoriesView.jsx
+++ b/client/src/components/MemoriesView.jsx
@@ -347,7 +347,7 @@ export default function MemoriesView() {
                 {scanInfo && (
                   <div style={{ display: 'flex', alignItems: 'center', gap: 6 }}>
                     <span style={{ fontSize: 11, color: '#444' }}>
-                      {scanInfo.scanning ? '⟳ Scanning music…' : scanInfo.enabled ? `${(scanInfo.total ?? tracks.length || scanInfo.count).toLocaleString()} tracks` : ''}
+                      {scanInfo.scanning ? '⟳ Scanning music…' : scanInfo.enabled ? `${scanInfo.count.toLocaleString()} tracks` : ''}
                     </span>
                     {scanInfo.enabled && !scanInfo.scanning && (
                       <button
diff --git a/client/src/components/VideosView.jsx b/client/src/components/VideosView.jsx
index 83fa938..ad5a92b 100644
--- a/client/src/components/VideosView.jsx
+++ b/client/src/components/VideosView.jsx
@@ -1,4 +1,4 @@
-import { useState, useEffect } from 'react';
+import { useState, useEffect, useRef, useCallback } from 'react';
 import { fetchVideos } from '../api/photos.js';
 
 // ── Grouping (mirrors Timeline.jsx) ───────────────────────────────────────────
@@ -47,19 +47,52 @@ function fmtSize(b) {
 const SHORT_THRESHOLD = 2; // seconds
 
 export default function VideosView({ filters = {} }) {
-  const [videos,     setVideos]     = useState(null);
-  const [playing,    setPlaying]    = useState(null);
-  const [error,      setError]      = useState(null);
-  const [showShort,  setShowShort]  = useState(false);
+  const [videos,      setVideos]      = useState([]);
+  const [nextCursor,  setNextCursor]  = useState(undefined);
+  const [total,       setTotal]       = useState(null);
+  const [loading,     setLoading]     = useState(false);
+  const [playing,     setPlaying]     = useState(null);
+  const [error,       setError]       = useState(null);
+  const [showShort,   setShowShort]   = useState(false);
+  const loadingRef  = useRef(false);
+  const sentinelRef = useRef(null);
+
+  const loadPage = useCallback(async (cursor, activeFilters) => {
+    if (loadingRef.current) return;
+    loadingRef.current = true;
+    setLoading(true);
+    setError(null);
+    try {
+      const d = await fetchVideos({ ...activeFilters, cursor });
+      setVideos(prev => cursor ? [...prev, ...d.videos] : d.videos);
+      setNextCursor(d.nextCursor ?? null);
+      setTotal(d.total);
+    } catch (e) {
+      setError(e.message);
+    }
+    loadingRef.current = false;
+    setLoading(false);
+  }, []);
 
   useEffect(() => {
-    setVideos(null);
-    setError(null);
-    fetchVideos({ from: filters.from, to: filters.to, city: filters.city })
-      .then(d => setVideos(d.videos))
-      .catch(e => setError(e.message));
+    setVideos([]);
+    setNextCursor(undefined);
+    setTotal(null);
+    loadPage(undefined, { from: filters.from, to: filters.to, city: filters.city });
   }, [filters.from, filters.to, filters.city]);
 
+  useEffect(() => {
+    if (!sentinelRef.current || !nextCursor) return;
+    const el  = sentinelRef.current;
+    const cur = nextCursor;
+    const obs = new IntersectionObserver(
+      ([entry]) => { if (entry.isIntersecting) loadPage(cur, { from: filters.from, to: filters.to, city: filters.city }); },
+      { rootMargin: '600px' }
+    );
+    obs.observe(el);
+    return () => obs.disconnect();
+  }, [nextCursor, filters.from, filters.to, filters.city]);
+
   useEffect(() => {
     if (!playing) return;
     const onKey = (e) => { if (e.key === 'Escape') setPlaying(null); };
@@ -67,10 +100,10 @@ export default function VideosView({ filters = {} }) {
     return () => window.removeEventListener('keydown', onKey);
   }, [playing]);
 
-  if (error)   return <p style={{ color: '#f87171', padding: 24, fontSize: 14 }}>{error}</p>;
-  if (!videos) return <p style={{ color: '#777',    padding: 24, fontSize: 13 }}>Loading…</p>;
+  if (error) return <p style={{ color: '#f87171', padding: 24, fontSize: 14 }}>{error}</p>;
 
-  if (videos.length === 0) {
+  const isEmpty = !loading && videos.length === 0;
+  if (isEmpty) {
     return (
       <div style={{ textAlign: 'center', paddingTop: 60, color: '#555' }}>
         <div style={{ fontSize: 44, marginBottom: 18, opacity: 0.4 }}>🎬</div>
@@ -112,6 +145,9 @@ export default function VideosView({ filters = {} }) {
       )}
 
       <main style={{ padding: '0 3px' }}>
+        {loading && videos.length === 0 && (
+          <p style={{ padding: 24, color: '#555', fontSize: 13 }}>Loading…</p>
+        )}
         {groups.map(group => (
           <section key={group.key} style={{ marginBottom: 8 }}>
 
@@ -140,6 +176,15 @@ export default function VideosView({ filters = {} }) {
             </div>
           </section>
         ))}
+        <div ref={sentinelRef} style={{ height: 1 }} />
+        {loading && videos.length > 0 && (
+          <p style={{ textAlign: 'center', padding: 24, color: '#444', fontSize: 13 }}>Loading…</p>
+        )}
+        {nextCursor === null && videos.length > 0 && !loading && (
+          <p style={{ textAlign: 'center', padding: 24, color: '#555', fontSize: 12 }}>
+            — {total?.toLocaleString()} video{total === 1 ? '' : 's'} —
+          </p>
+        )}
       </main>
 
       {/* ── Player overlay ── */}
diff --git a/server/index.js b/server/index.js
index f849250..fff2a1b 100644
--- a/server/index.js
+++ b/server/index.js
@@ -26,6 +26,19 @@ if (MUSIC_DIR) {
 
 app.use(express.json());
 
+// ── In-memory cache (invalidated on any write) ────────────────────────────────
+const cache = {
+  _store: {},
+  get(key, ttlMs) {
+    const e = this._store[key];
+    if (!e || Date.now() - e.ts > ttlMs) return null;
+    return e.data;
+  },
+  set(key, data) { this._store[key] = { data, ts: Date.now() }; },
+  del(key) { delete this._store[key]; },
+  invalidate() { this._store = {}; },
+};
+
 // ── Photo Sync state ──────────────────────────────────────────────────────────
 
 const syncJobs = {
@@ -241,6 +254,10 @@ app.get('/api/ingest/stream', (req, res) => {
 // ── Library stats ─────────────────────────────────────────────────────────────
 
 app.get('/api/library/stats', (_req, res) => {
+  const cached = cache.get('library-stats', 30_000);
+  if (cached) return res.json(cached);
+
+  const total   = db.raw.prepare('SELECT COUNT(*) AS n FROM photos WHERE deleted = 0').get().n;
   const starred = db.raw.prepare('SELECT COUNT(*) AS n FROM photos WHERE starred = 1 AND deleted = 0').get().n;
   const years   = db.raw.prepare(
     "SELECT strftime('%Y', date_taken) AS year, COUNT(*) AS count FROM photos WHERE date_taken IS NOT NULL AND deleted = 0 GROUP BY year ORDER BY year DESC"
@@ -248,14 +265,9 @@ app.get('/api/library/stats', (_req, res) => {
   const cities  = db.raw.prepare(
     "SELECT place_city AS city, place_country AS country, COUNT(*) AS count FROM photos WHERE place_city IS NOT NULL AND deleted = 0 GROUP BY place_city ORDER BY count DESC LIMIT 100"
   ).all();
-  res.json({
-    total: db.raw.prepare('SELECT COUNT(*) AS n FROM photos WHERE deleted = 0').get().n,
-    starred,
-    years,
-    cities,
-    tags:        db.listAllTags(),
-    collections: db.listCollections(),
-  });
+  const data = { total, starred, years, cities, tags: db.listAllTags(), collections: db.listCollections() };
+  cache.set('library-stats', data);
+  res.json(data);
 });
 
 // ── Photos ────────────────────────────────────────────────────────────────────
@@ -348,16 +360,19 @@ app.get('/api/photos/:id', (req, res) => {
 
 app.delete('/api/photos/:id', (req, res) => {
   db.raw.prepare('UPDATE photos SET deleted = 1 WHERE id = ?').run(parseInt(req.params.id, 10));
+  cache.invalidate();
   res.sendStatus(204);
 });
 
 app.post('/api/photos/:id/restore', (req, res) => {
   db.raw.prepare('UPDATE photos SET deleted = 0 WHERE id = ?').run(parseInt(req.params.id, 10));
+  cache.invalidate();
   res.sendStatus(204);
 });
 
 app.delete('/api/photos/:id/permanent', (req, res) => {
   db.raw.prepare('DELETE FROM photos WHERE id = ?').run(parseInt(req.params.id, 10));
+  cache.invalidate();
   res.sendStatus(204);
 });
 
@@ -374,6 +389,7 @@ app.patch('/api/photos/bulk', (req, res) => {
   if ('starred' in req.body) {
     const ph = ids.map(() => '?').join(',');
     db.raw.prepare(`UPDATE photos SET starred = ? WHERE id IN (${ph})`).run(req.body.starred ? 1 : 0, ...ids);
+    cache.del('library-stats');
   }
   res.sendStatus(204);
 });
@@ -412,7 +428,9 @@ app.post('/api/collections', (req, res) => {
   const name = (req.body.name ?? '').trim().slice(0, 200);
   if (!name) return badRequest(res, 'name required');
   try {
-    res.status(201).json(db.createCollection(name, req.body.description));
+    const col = db.createCollection(name, req.body.description);
+    cache.invalidate();
+    res.status(201).json(col);
   } catch (e) {
     if (e.message.includes('UNIQUE')) return badRequest(res, 'collection already exists');
     throw e;
@@ -424,6 +442,7 @@ app.patch('/api/collections/:id', (req, res) => {
   if (!name) return badRequest(res, 'name required');
   try {
     db.raw.prepare('UPDATE collections SET name = ? WHERE id = ?').run(name, parseInt(req.params.id, 10));
+    cache.invalidate();
     res.sendStatus(204);
   } catch (e) {
     if (e.message.includes('UNIQUE')) return badRequest(res, 'collection already exists');
@@ -433,70 +452,90 @@ app.patch('/api/collections/:id', (req, res) => {
 
 app.delete('/api/collections/:id', (req, res) => {
   db.raw.prepare('DELETE FROM collections WHERE id = ?').run(parseInt(req.params.id, 10));
+  cache.invalidate();
   res.sendStatus(204);
 });
 
 // ── Albums (manual + smart auto-albums) ───────────────────────────────────────
 
 app.get('/api/albums', (_req, res) => {
+  const cached = cache.get('albums', 60_000);
+  if (cached) return res.json(cached);
+
+  // Collections: count from indexed join; cover via point-lookup on MAX(id) per collection
   const manual = db.raw.prepare(`
+    WITH cover AS (
+      SELECT collection_id, MAX(photo_id) AS cover_id
+      FROM photo_collections GROUP BY collection_id
+    )
     SELECT c.id, c.name, c.description, c.created_at,
       COUNT(pc.photo_id) AS photo_count,
-      (SELECT p.thumb_path FROM photo_collections pc2 JOIN photos p ON p.id = pc2.photo_id
-       WHERE pc2.collection_id = c.id ORDER BY p.date_taken DESC, p.id DESC LIMIT 1) AS cover_path
-    FROM collections c LEFT JOIN photo_collections pc ON pc.collection_id = c.id
+      p.thumb_path AS cover_path
+    FROM collections c
+    LEFT JOIN photo_collections pc ON pc.collection_id = c.id
+    LEFT JOIN cover cv ON cv.collection_id = c.id
+    LEFT JOIN photos p ON p.id = cv.cover_id
     GROUP BY c.id ORDER BY c.name
   `).all();
 
   const PHOTO_ONLY = "deleted = 0 AND (media_type IS NULL OR media_type = 'photo')";
 
+  // Use GROUP BY + MAX(id) per bucket instead of window functions (much faster at scale)
   const years = db.raw.prepare(`
-    SELECT year, count, cover_path FROM (
-      SELECT strftime('%Y', date_taken) AS year,
-        COUNT(*) OVER (PARTITION BY strftime('%Y', date_taken)) AS count,
-        thumb_path AS cover_path,
-        ROW_NUMBER() OVER (PARTITION BY strftime('%Y', date_taken) ORDER BY date_taken DESC, id DESC) AS rn
+    WITH agg AS (
+      SELECT strftime('%Y', date_taken) AS year, COUNT(*) AS count, MAX(id) AS cover_id
       FROM photos WHERE date_taken IS NOT NULL AND ${PHOTO_ONLY}
-    ) WHERE rn = 1 ORDER BY year DESC
+      GROUP BY strftime('%Y', date_taken)
+    )
+    SELECT a.year, a.count, p.thumb_path AS cover_path
+    FROM agg a JOIN photos p ON p.id = a.cover_id
+    ORDER BY a.year DESC
   `).all();
 
   const cities = db.raw.prepare(`
-    SELECT city, country, count, cover_path FROM (
+    WITH agg AS (
       SELECT place_city AS city, place_country AS country,
-        COUNT(*) OVER (PARTITION BY place_city) AS count,
-        thumb_path AS cover_path,
-        ROW_NUMBER() OVER (PARTITION BY place_city ORDER BY date_taken DESC, id DESC) AS rn
+        COUNT(*) AS count, MAX(id) AS cover_id
       FROM photos WHERE place_city IS NOT NULL AND ${PHOTO_ONLY}
-    ) WHERE rn = 1 ORDER BY count DESC LIMIT 50
+      GROUP BY place_city
+    )
+    SELECT a.city, a.country, a.count, p.thumb_path AS cover_path
+    FROM agg a JOIN photos p ON p.id = a.cover_id
+    ORDER BY a.count DESC LIMIT 50
   `).all();
 
   const cameras = db.raw.prepare(`
-    SELECT camera, count, cover_path FROM (
-      SELECT camera_model AS camera,
-        COUNT(*) OVER (PARTITION BY camera_model) AS count,
-        thumb_path AS cover_path,
-        ROW_NUMBER() OVER (PARTITION BY camera_model ORDER BY date_taken DESC, id DESC) AS rn
+    WITH agg AS (
+      SELECT camera_model AS camera, COUNT(*) AS count, MAX(id) AS cover_id
       FROM photos WHERE camera_model IS NOT NULL AND ${PHOTO_ONLY}
-    ) WHERE rn = 1 ORDER BY count DESC
+      GROUP BY camera_model
+    )
+    SELECT a.camera, a.count, p.thumb_path AS cover_path
+    FROM agg a JOIN photos p ON p.id = a.cover_id
+    ORDER BY a.count DESC
   `).all();
 
   const recent = [7, 30].map(days => {
     const since = new Date(Date.now() - days * 864e5).toISOString();
-    const count = db.raw.prepare(`SELECT COUNT(*) AS n FROM photos WHERE date_imported >= ? AND ${PHOTO_ONLY}`).get(since).n;
-    if (!count) return null;
-    const cover = db.raw.prepare(`SELECT thumb_path FROM photos WHERE date_imported >= ? AND ${PHOTO_ONLY} ORDER BY date_imported DESC LIMIT 1`).get(since);
-    return { days, label: days === 7 ? 'Last 7 days' : 'Last 30 days', count, cover_url: posterUrl(cover?.thumb_path) };
+    const row = db.raw.prepare(
+      `SELECT COUNT(*) AS count, MAX(id) AS cover_id FROM photos WHERE date_imported >= ? AND ${PHOTO_ONLY}`
+    ).get(since);
+    if (!row.count) return null;
+    const cover = db.raw.prepare('SELECT thumb_path FROM photos WHERE id = ?').get(row.cover_id);
+    return { days, label: days === 7 ? 'Last 7 days' : 'Last 30 days', count: row.count, cover_url: posterUrl(cover?.thumb_path) };
   }).filter(Boolean);
 
-  res.json({
-    manual:  manual.map(c => ({ ...c, cover_url: posterUrl(c.cover_path), cover_path: undefined })),
+  const data = {
+    manual: manual.map(c => ({ ...c, cover_url: posterUrl(c.cover_path), cover_path: undefined })),
     auto: {
       recent,
       years:   years.map(r   => ({ year:   r.year,   count: r.count,   cover_url: posterUrl(r.cover_path) })),
       cities:  cities.map(r  => ({ city:   r.city,   country: r.country, count: r.count,  cover_url: posterUrl(r.cover_path) })),
       cameras: cameras.map(r => ({ camera: r.camera, count: r.count,   cover_url: posterUrl(r.cover_path) })),
     },
-  });
+  };
+  cache.set('albums', data);
+  res.json(data);
 });
 
 app.post('/api/photos/bulk/collections', (req, res) => {
@@ -671,17 +710,41 @@ app.get('/api/videos/stats', (_req, res) => {
 });
 
 app.get('/api/videos', (req, res) => {
-  const { from, to, city } = req.query;
+  const { from, to, city, cursor } = req.query;
+  const limit = 100;
   const conditions = ["p.media_type = 'video'", "p.deleted = 0"];
   const params = [];
   if (from) { conditions.push('p.date_taken >= ?'); params.push(from); }
   if (to)   { conditions.push('p.date_taken <= ?'); params.push(to); }
   if (city) { conditions.push('p.place_city = ?');  params.push(city); }
+
+  let cursorSql = '';
+  const cursorParams = [];
+  if (cursor) {
+    const pipeAt = cursor.lastIndexOf('|');
+    const cursorDate = cursor.slice(0, pipeAt);
+    const cursorId   = parseInt(cursor.slice(pipeAt + 1), 10);
+    cursorSql = cursorDate
+      ? `AND (p.date_taken < ? OR (p.date_taken = ? AND p.id < ?) OR p.date_taken IS NULL)`
+      : `AND (p.date_taken IS NULL AND p.id < ?)`;
+    cursorDate ? cursorParams.push(cursorDate, cursorDate, cursorId) : cursorParams.push(cursorId);
+  }
+
   const where = 'WHERE ' + conditions.join(' AND ');
   const rows = db.raw.prepare(
-    `SELECT ${SELECT_COLS} FROM photos p ${where} ORDER BY p.date_taken DESC, p.id DESC`
-  ).all(...params);
-  res.json({ videos: rows.map(mapPhoto), total: rows.length });
+    `SELECT ${SELECT_COLS} FROM photos p ${where} ${cursorSql} ORDER BY p.date_taken DESC, p.id DESC LIMIT ?`
+  ).all(...params, ...cursorParams, limit + 1);
+
+  const total = db.raw.prepare(
+    `SELECT COUNT(*) AS n FROM photos p ${where}`
+  ).get(...params).n;
+
+  const hasMore    = rows.length > limit;
+  const page       = rows.slice(0, limit);
+  const last       = page[page.length - 1];
+  const nextCursor = hasMore && last ? `${last.date_taken ?? ''}|${last.id}` : null;
+
+  res.json({ videos: page.map(mapPhoto), nextCursor, total });
 });
 
 app.get('/api/photos/:id/video', (req, res) => {
diff --git a/worker/db.js b/worker/db.js
index 1e50711..cbe5109 100644
--- a/worker/db.js
+++ b/worker/db.js
@@ -126,6 +126,19 @@ export function openDb(dbPath) {
   db.exec('CREATE INDEX IF NOT EXISTS idx_photos_place_city  ON photos(place_city)');
   db.exec('CREATE INDEX IF NOT EXISTS idx_photos_media_type  ON photos(media_type)');
 
+  // Performance indexes for 30k+ libraries
+  // Composite covering index for the main library query (deleted + date_taken for sorted pagination)
+  db.exec('CREATE INDEX IF NOT EXISTS idx_photos_deleted_date ON photos(deleted, date_taken DESC, id DESC)');
+  // Composite for stats queries
+  db.exec('CREATE INDEX IF NOT EXISTS idx_photos_deleted_starred ON photos(deleted, starred)');
+  db.exec('CREATE INDEX IF NOT EXISTS idx_photos_deleted_type    ON photos(deleted, media_type)');
+  db.exec('CREATE INDEX IF NOT EXISTS idx_photos_deleted_city    ON photos(deleted, place_city)');
+  db.exec('CREATE INDEX IF NOT EXISTS idx_photos_deleted_camera  ON photos(deleted, camera_model)');
+  // Expression index for Memories ("on this day") — makes strftime('%m-%d') queries instant
+  db.exec("CREATE INDEX IF NOT EXISTS idx_photos_month_day ON photos(strftime('%m-%d', date_taken)) WHERE date_taken IS NOT NULL");
+  // Collections join performance
+  db.exec('CREATE INDEX IF NOT EXISTS idx_photo_collections_col ON photo_collections(collection_id, photo_id)');
+
   // Seed default settings
   const seedSetting = db.prepare('INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)');
   for (const [k, v] of Object.entries(SETTING_DEFAULTS)) seedSetting.run(k, v);