perf: skip already-indexed files before queuing on worker startup
Load all known filepaths into an in-memory Set on startup (one bulk query). Chokidar's initial 'add' burst now hits an O(1) Set lookup instead of going through the serial drain queue with a per-file DB round-trip — a 100k-photo library restarts in seconds rather than re-processing every file. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
f0e237ab2b
commit
8c5c36857f
1 file changed
+15
-3
+15
-3
@@ -10,8 +10,15 @@ const POSTER_DIR = process.env.POSTER_DIR ?? './data/posters';
|
|||||||
|
|
||||||
const db = openDb(DB_PATH);
|
const db = openDb(DB_PATH);
|
||||||
|
|
||||||
|
// Load all known paths into memory once so chokidar's initial 'add' burst
|
||||||
|
// for an already-indexed library costs one bulk query instead of N serial
|
||||||
|
// DB lookups through the drain queue.
|
||||||
|
const indexed = new Set(
|
||||||
|
db.raw.prepare('SELECT filepath FROM photos').all().map(r => r.filepath)
|
||||||
|
);
|
||||||
|
|
||||||
console.log(`[worker] watching ${path.resolve(PHOTO_DIR)}`);
|
console.log(`[worker] watching ${path.resolve(PHOTO_DIR)}`);
|
||||||
console.log(`[worker] db at ${path.resolve(DB_PATH)} (${db.count()} photos indexed)`);
|
console.log(`[worker] db at ${path.resolve(DB_PATH)} (${indexed.size} photos indexed)`);
|
||||||
|
|
||||||
const queue = [];
|
const queue = [];
|
||||||
let busy = false;
|
let busy = false;
|
||||||
@@ -28,6 +35,7 @@ async function drain() {
|
|||||||
const fp = queue.shift();
|
const fp = queue.shift();
|
||||||
try {
|
try {
|
||||||
await ingestFile(fp, db, POSTER_DIR);
|
await ingestFile(fp, db, POSTER_DIR);
|
||||||
|
indexed.add(fp);
|
||||||
scanned++;
|
scanned++;
|
||||||
db.setIngestStatus({ active: true, current: path.basename(fp), scanned, errors });
|
db.setIngestStatus({ active: true, current: path.basename(fp), scanned, errors });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -50,12 +58,16 @@ const watcher = chokidar.watch(PHOTO_DIR, {
|
|||||||
|
|
||||||
watcher.on('add', (fp) => {
|
watcher.on('add', (fp) => {
|
||||||
if (!isSupported(fp)) return;
|
if (!isSupported(fp)) return;
|
||||||
queue.push(path.resolve(fp));
|
const resolved = path.resolve(fp);
|
||||||
|
if (indexed.has(resolved)) return; // already in DB — skip entirely
|
||||||
|
queue.push(resolved);
|
||||||
drain();
|
drain();
|
||||||
});
|
});
|
||||||
|
|
||||||
watcher.on('unlink', (fp) => {
|
watcher.on('unlink', (fp) => {
|
||||||
db.removePhoto(path.resolve(fp));
|
const resolved = path.resolve(fp);
|
||||||
|
indexed.delete(resolved);
|
||||||
|
db.removePhoto(resolved);
|
||||||
console.log(`[worker] removed ${path.basename(fp)}`);
|
console.log(`[worker] removed ${path.basename(fp)}`);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user