From 8c5c36857f2e2706adf89064e0ee003ad1c71c73 Mon Sep 17 00:00:00 2001
From: nmemmert <67961292+nmemmert@users.noreply.github.com>
Date: Tue, 18 Aug 2026 07:53:42 -0400
Subject: [PATCH] perf: skip already-indexed files before queuing on worker
 startup
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Load all known filepaths into an in-memory Set on startup (one bulk query).
Chokidar's initial 'add' burst now hits an O(1) Set lookup instead of going
through the serial drain queue with a per-file DB round-trip — a 100k-photo
library restarts in seconds rather than re-processing every file.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 worker/index.js | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/worker/index.js b/worker/index.js
index f478e65..b4592da 100644
--- a/worker/index.js
+++ b/worker/index.js
@@ -10,8 +10,15 @@ const POSTER_DIR = process.env.POSTER_DIR ?? './data/posters';
 
 const db = openDb(DB_PATH);
 
+// Load all known paths into memory once so chokidar's initial 'add' burst
+// for an already-indexed library costs one bulk query instead of N serial
+// DB lookups through the drain queue.
+const indexed = new Set(
+  db.raw.prepare('SELECT filepath FROM photos').all().map(r => r.filepath)
+);
+
 console.log(`[worker] watching ${path.resolve(PHOTO_DIR)}`);
-console.log(`[worker] db at ${path.resolve(DB_PATH)} (${db.count()} photos indexed)`);
+console.log(`[worker] db at ${path.resolve(DB_PATH)} (${indexed.size} photos indexed)`);
 
 const queue = [];
 let busy    = false;
@@ -28,6 +35,7 @@ async function drain() {
     const fp = queue.shift();
     try {
       await ingestFile(fp, db, POSTER_DIR);
+      indexed.add(fp);
       scanned++;
       db.setIngestStatus({ active: true, current: path.basename(fp), scanned, errors });
     } catch (err) {
@@ -50,12 +58,16 @@ const watcher = chokidar.watch(PHOTO_DIR, {
 
 watcher.on('add', (fp) => {
   if (!isSupported(fp)) return;
-  queue.push(path.resolve(fp));
+  const resolved = path.resolve(fp);
+  if (indexed.has(resolved)) return;   // already in DB — skip entirely
+  queue.push(resolved);
   drain();
 });
 
 watcher.on('unlink', (fp) => {
-  db.removePhoto(path.resolve(fp));
+  const resolved = path.resolve(fp);
+  indexed.delete(resolved);
+  db.removePhoto(resolved);
   console.log(`[worker] removed ${path.basename(fp)}`);
 });