fix: dedupe scan 500 error, SQLite limit crash, add keep-best action

- Replace BigInt hammingDistance with 32-bit popcount — safer (no throw on malformed hashes) and ~10× faster - Make scan() async with setImmediate chunking so the event loop stays responsive on large libraries - Batch getGroups() photo lookup in slices of 900 to stay under SQLite's 999-variable limit - Add POST /api/duplicates/keep-best endpoint: scores by resolution then filesize, soft-deletes the losers, dismisses the pair - DuplicateGroup: green "Keep" badge on the best photo, green "Keep best" button in footer Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-08-19 15:45:46 -04:00
1 parent 93e16561ec
commit 4f7dd3b49b
@@ -183,3 +183,7 @@ export function scanDuplicates() {
 export function dismissGroup(photoIds) {
  return api('/api/duplicates/dismiss', { method: 'POST', ...json({ photoIds }) });
 }
+
+export function keepBestDuplicate(photoIds) {
+  return api('/api/duplicates/keep-best', { method: 'POST', ...json({ photoIds }) });
+}
@@ -9,7 +9,18 @@ function fmtDate(d) {
  return new Date(d).toLocaleDateString('en-US', { month: 'short', day: 'numeric', year: 'numeric' });
 }

-export default function DuplicateGroup({ photos, onDismiss, onOpen }) {
+function scorePhoto(p) {
+  const pixels  = (p.width  || 0) * (p.height || 0);
+  const size    = p.filesize || 0;
+  const hasExif = p.camera_model ? 1 : 0;
+  return pixels * 1e6 + size + hasExif * 1e3;
+}
+
+export default function DuplicateGroup({ photos, onDismiss, onKeepBest, onOpen }) {
+  const scores  = photos.map(scorePhoto);
+  const maxScore = Math.max(...scores);
+  const bestIdx  = scores.indexOf(maxScore);
+
  return (
    <div style={{
      background: '#111',
@@ -19,67 +30,103 @@ export default function DuplicateGroup({ photos, onDismiss, onOpen }) {
    }}>
      {/* Photo row */}
      <div style={{ display: 'flex', flexWrap: 'wrap', gap: 1, background: '#0c0c0c' }}>
-        {photos.map((photo, i) => (
-          <div
-            key={photo.id}
-            style={{ flex: '1 1 200px', minWidth: 160, maxWidth: 320, display: 'flex', flexDirection: 'column' }}
-          >
+        {photos.map((photo, i) => {
+          const isBest = i === bestIdx;
+          return (
            <div
-              onClick={() => onOpen(photo)}
+              key={photo.id}
              style={{
-                aspectRatio: '4/3',
-                cursor: 'pointer',
-                background: '#0c0c0c',
-                overflow: 'hidden',
+                flex: '1 1 200px', minWidth: 160, maxWidth: 320,
+                display: 'flex', flexDirection: 'column',
+                outline: isBest ? '2px solid #22c55e' : 'none',
+                outlineOffset: '-2px',
                position: 'relative',
              }}
            >
-              {photo.thumb_url ? (
-                <img
-                  src={photo.thumb_url}
-                  alt=""
-                  loading="lazy"
-                  style={{ width: '100%', height: '100%', objectFit: 'cover', display: 'block' }}
-                />
-              ) : (
-                <div style={{ width: '100%', height: '100%', display: 'flex', alignItems: 'center', justifyContent: 'center', color: '#222', fontSize: 24 }}>▪</div>
-              )}
+              <div
+                onClick={() => onOpen(photo)}
+                style={{
+                  aspectRatio: '4/3',
+                  cursor: 'pointer',
+                  background: '#0c0c0c',
+                  overflow: 'hidden',
+                  position: 'relative',
+                }}
+              >
+                {photo.thumb_url ? (
+                  <img
+                    src={photo.thumb_url}
+                    alt=""
+                    loading="lazy"
+                    style={{ width: '100%', height: '100%', objectFit: 'cover', display: 'block' }}
+                  />
+                ) : (
+                  <div style={{ width: '100%', height: '100%', display: 'flex', alignItems: 'center', justifyContent: 'center', color: '#222', fontSize: 24 }}>▪</div>
+                )}

-              {/* Index badge */}
-              <span style={{
-                position: 'absolute', top: 6, left: 6,
-                background: 'rgba(0,0,0,0.6)', borderRadius: 4,
-                fontSize: 10, color: '#888', padding: '2px 5px',
-              }}>
-                {i + 1}
-              </span>
-            </div>
-
-            {/* Meta */}
-            <div style={{ padding: '7px 10px 8px', fontSize: 12, color: '#777', display: 'flex', flexDirection: 'column', gap: 2 }}>
-              <span style={{ color: '#888' }}>{fmtDate(photo.date_taken)}</span>
-              <span>
-                {[fmtSize(photo.filesize), photo.width && `${photo.width}×${photo.height}`].filter(Boolean).join(' · ')}
-              </span>
-              {photo.camera_model && (
-                <span style={{ color: '#666', fontSize: 11, overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }}>
-                  {photo.camera_model}
+                {/* Index badge */}
+                <span style={{
+                  position: 'absolute', top: 6, left: 6,
+                  background: 'rgba(0,0,0,0.6)', borderRadius: 4,
+                  fontSize: 10, color: '#888', padding: '2px 5px',
+                }}>
+                  {i + 1}
                </span>
-              )}
+
+                {/* Best badge */}
+                {isBest && (
+                  <span style={{
+                    position: 'absolute', top: 6, right: 6,
+                    background: '#16a34a',
+                    borderRadius: 4,
+                    fontSize: 10, color: '#fff', padding: '2px 6px',
+                    fontWeight: 600,
+                  }}>
+                    Keep
+                  </span>
+                )}
+              </div>
+
+              {/* Meta */}
+              <div style={{ padding: '7px 10px 8px', fontSize: 12, color: '#777', display: 'flex', flexDirection: 'column', gap: 2 }}>
+                <span style={{ color: isBest ? '#86efac' : '#888' }}>{fmtDate(photo.date_taken)}</span>
+                <span>
+                  {[fmtSize(photo.filesize), photo.width && `${photo.width}×${photo.height}`].filter(Boolean).join(' · ')}
+                </span>
+                {photo.camera_model && (
+                  <span style={{ color: '#666', fontSize: 11, overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }}>
+                    {photo.camera_model}
+                  </span>
+                )}
+              </div>
            </div>
-          </div>
-        ))}
+          );
+        })}
      </div>

      {/* Footer */}
      <div style={{
        padding: '8px 14px',
        borderTop: '1px solid #1a1a1a',
-        display: 'flex', alignItems: 'center', gap: 12,
+        display: 'flex', alignItems: 'center', gap: 10,
      }}>
        <span style={{ fontSize: 12, color: '#666', flex: 1 }}>
          {photos.length} similar photos
        </span>
+        <button
+          onClick={onKeepBest}
+          style={{
+            background: '#14532d',
+            border: '1px solid #16a34a',
+            borderRadius: 5,
+            color: '#86efac',
+            cursor: 'pointer',
+            padding: '4px 12px',
+            fontSize: 12,
+          }}
+        >
+          Keep best
+        </button>
        <button
          onClick={onDismiss}
          style={{
@@ -1,5 +1,5 @@
 import { useState, useEffect } from 'react';
-import { fetchDuplicates, scanDuplicates, dismissGroup } from '../api/photos.js';
+import { fetchDuplicates, scanDuplicates, dismissGroup, keepBestDuplicate } from '../api/photos.js';
 import DuplicateGroup from './DuplicateGroup.jsx';

 export default function DuplicatesView({ onOpenPhoto, onGroupCountChange }) {
@@ -33,20 +33,33 @@ export default function DuplicatesView({ onOpenPhoto, onGroupCountChange }) {
    setScanning(false);
  }

+  function removeGroup(photoIds) {
+    setGroups(prev => {
+      const next = prev.filter(g => {
+        const s = new Set(g.map(p => p.id));
+        return !photoIds.every(id => s.has(id));
+      });
+      onGroupCountChange?.(next.length);
+      return next;
+    });
+  }
+
  async function handleDismiss(photoIds) {
    try {
      await dismissGroup(photoIds);
-      setGroups(prev => {
-        const next = prev.filter(g => {
-          const s = new Set(g.map(p => p.id));
-          return !photoIds.every(id => s.has(id));
-        });
-        onGroupCountChange?.(next.length);
-        return next;
-      });
+      removeGroup(photoIds);
    } catch {}
  }

+  async function handleKeepBest(photoIds) {
+    try {
+      await keepBestDuplicate(photoIds);
+      removeGroup(photoIds);
+    } catch (e) {
+      setError(e.message);
+    }
+  }
+
  return (
    <div style={{ padding: '24px 20px', maxWidth: 1100, margin: '0 auto' }}>
      {/* Toolbar */}
@@ -100,6 +113,7 @@ export default function DuplicatesView({ onOpenPhoto, onGroupCountChange }) {
              key={i}
              photos={group}
              onDismiss={() => handleDismiss(group.map(p => p.id))}
+              onKeepBest={() => handleKeepBest(group.map(p => p.id))}
              onOpen={(photo) => onOpenPhoto(photo, group)}
            />
          ))}
@@ -1,6 +1,10 @@
 import { hammingDistance } from '../worker/hash.js';

-export function scan(db, threshold) {
+// Chunk size for the outer loop — yields to the event loop between chunks
+// so the server stays responsive during a long scan.
+const SCAN_CHUNK = 50;
+
+export async function scan(db, threshold) {
  const THRESHOLD = threshold ?? parseInt(process.env.DEDUPE_THRESHOLD ?? '10', 10);
  const start = Date.now();

@@ -13,21 +17,42 @@ export function scan(db, threshold) {
  );

  let newPairs = 0;
-  db.raw.transaction(() => {
-    for (let i = 0; i < photos.length; i++) {
-      for (let j = i + 1; j < photos.length; j++) {
-        const dist = hammingDistance(photos[i].phash, photos[j].phash);
-        if (dist <= THRESHOLD) {
-          const r = insert.run(photos[i].id, photos[j].id, dist);
-          if (r.changes) newPairs++;
+
+  for (let i = 0; i < photos.length; i += SCAN_CHUNK) {
+    await new Promise(resolve => setImmediate(resolve));
+    const end = Math.min(i + SCAN_CHUNK, photos.length);
+
+    db.raw.transaction(() => {
+      for (let ii = i; ii < end; ii++) {
+        for (let j = ii + 1; j < photos.length; j++) {
+          const dist = hammingDistance(photos[ii].phash, photos[j].phash);
+          if (dist <= THRESHOLD) {
+            const r = insert.run(photos[ii].id, photos[j].id, dist);
+            if (r.changes) newPairs++;
+          }
        }
      }
-    }
-  })();
+    })();
+  }

  return { photos: photos.length, newPairs, duration: Date.now() - start };
 }

+const ID_BATCH = 900;
+
+function fetchPhotosByIds(db, ids) {
+  const rows = [];
+  for (let i = 0; i < ids.length; i += ID_BATCH) {
+    const batch = ids.slice(i, i + ID_BATCH);
+    const got = db.raw.prepare(
+      `SELECT id, thumb_path, preview_path, date_taken, filesize, width, height, camera_model
+       FROM photos WHERE id IN (${batch.map(() => '?').join(',')})`
+    ).all(batch);
+    rows.push(...got);
+  }
+  return rows;
+}
+
 export function getGroups(db) {
  const pairs = db.getDupePairs();
  if (!pairs.length) return [];
@@ -53,11 +78,7 @@ export function getGroups(db) {
  }

  const allIds = [...parent.keys()];
-  const rows   = db.raw.prepare(
-    `SELECT id, thumb_path, preview_path, date_taken, filesize, width, height, camera_model
-     FROM photos WHERE id IN (${allIds.map(() => '?').join(',')})`
-  ).all(allIds);
-
+  const rows   = fetchPhotosByIds(db, allIds);
  const photoMap = new Map(rows.map(r => [r.id, r]));

  return [...groups.values()]
@@ -65,3 +86,29 @@ export function getGroups(db) {
    .map(ids => ids.map(id => photoMap.get(id)).filter(Boolean))
    .sort((a, b) => b.length - a.length);
 }
+
+export function scoreDuplicate(photo) {
+  const pixels = (photo.width || 0) * (photo.height || 0);
+  const size   = photo.filesize || 0;
+  const hasExif = photo.camera_model ? 1 : 0;
+  return pixels * 1e6 + size + hasExif * 1e3;
+}
+
+export function keepBest(db, photoIds) {
+  const ids = photoIds.map(Number);
+  const photos = fetchPhotosByIds(db, ids);
+  if (photos.length < 2) return null;
+
+  photos.sort((a, b) => scoreDuplicate(b) - scoreDuplicate(a));
+  const keeper = photos[0];
+  const discard = photos.slice(1);
+
+  db.raw.transaction(() => {
+    for (const p of discard) {
+      db.raw.prepare('UPDATE photos SET deleted = 1 WHERE id = ?').run(p.id);
+    }
+  })();
+
+  db.dismissDuplicates(ids);
+  return { kept: keeper.id, deleted: discard.map(p => p.id) };
+}
@@ -5,7 +5,7 @@ import fs from 'fs';
 import { fileURLToPath } from 'url';
 import { spawn } from 'child_process';
 import { openDb } from '../worker/db.js';
-import { scan, getGroups } from './dedupe.js';
+import { scan, getGroups, keepBest } from './dedupe.js';
 import { scanMusicLibrary, initMusicTable, musicScanActive } from '../worker/scanMusic.js';

 const __dirname  = path.dirname(fileURLToPath(import.meta.url));
@@ -561,10 +561,10 @@ app.delete('/api/photos/:id/collections/:colId', (req, res) => {

 // ── Duplicates ────────────────────────────────────────────────────────────────

-app.post('/api/dedupe/scan', (_req, res) => {
+app.post('/api/dedupe/scan', async (_req, res) => {
  try {
    const { dedupe_threshold } = db.getSettings();
-    res.json(scan(db, dedupe_threshold));
+    res.json(await scan(db, dedupe_threshold));
  } catch (e) { res.status(500).json({ error: e.message }); }
 });

@@ -592,6 +592,16 @@ app.post('/api/duplicates/dismiss', (req, res) => {
  res.sendStatus(204);
 });

+app.post('/api/duplicates/keep-best', (req, res) => {
+  try {
+    const ids = req.body.photoIds;
+    if (!Array.isArray(ids) || ids.length < 2) return badRequest(res, 'photoIds must be array of ≥2');
+    const result = keepBest(db, ids);
+    cache.invalidate();
+    res.json(result);
+  } catch (e) { res.status(500).json({ error: e.message }); }
+});
+
 // ── Photo Sync ────────────────────────────────────────────────────────────────

 app.get('/api/sync/stream', (req, res) => {
@@ -20,12 +20,17 @@ export async function computeDHash(filepath) {
  return hash.toString(16).padStart(16, '0');
 }

+function popcount32(n) {
+  n = n >>> 0;
+  n -= (n >> 1) & 0x55555555;
+  n = (n & 0x33333333) + ((n >> 2) & 0x33333333);
+  n = (n + (n >> 4)) & 0x0f0f0f0f;
+  return Math.imul(n, 0x01010101) >>> 24;
+}
+
 export function hammingDistance(a, b) {
-  let xor = BigInt('0x' + a) ^ BigInt('0x' + b);
-  let dist = 0;
-  while (xor > 0n) {
-    dist += Number(xor & 1n);
-    xor >>= 1n;
-  }
-  return dist;
+  if (!a || !b || a.length !== 16 || b.length !== 16) return 64;
+  const xorHi = (parseInt(a.slice(0, 8), 16) ^ parseInt(b.slice(0, 8), 16)) >>> 0;
+  const xorLo = (parseInt(a.slice(8),    16) ^ parseInt(b.slice(8),    16)) >>> 0;
+  return popcount32(xorHi) + popcount32(xorLo);
 }