From e58e6de1250550aecbc86a274493363cf1a9cc63 Mon Sep 17 00:00:00 2001
From: nmemmert <67961292+nmemmert@users.noreply.github.com>
Date: Wed, 19 Aug 2026 16:20:33 -0400
Subject: [PATCH] fix: eliminate false-positive duplicate mega-groups
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The union-find algorithm caused transitivity chains: A≈B and B≈C merged
A, B, C into one group even if A and C were completely different photos,
producing the 5000+ "duplicate" groups that were wrong.

- getGroups now returns individual pairs (2 photos each) — what was
  actually flagged by the pHash comparison, no chaining
- scan() clears all existing pairs on each run so the result exactly
  matches the current threshold (no stale loose pairs from prior scans)
- Default dedupe_threshold lowered from 10 → 5 (fewer false positives;
  distance ≤5 means the images are genuinely near-identical)

Run "Scan library" once to replace the old results with clean pairs.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 server/dedupe.js | 40 ++++++++++++++--------------------------
 worker/db.js     |  2 +-
 2 files changed, 15 insertions(+), 27 deletions(-)

diff --git a/server/dedupe.js b/server/dedupe.js
index d9ee081..fd40e7d 100644
--- a/server/dedupe.js
+++ b/server/dedupe.js
@@ -5,9 +5,13 @@ import { hammingDistance } from '../worker/hash.js';
 const SCAN_CHUNK = 50;
 
 export async function scan(db, threshold) {
-  const THRESHOLD = threshold ?? parseInt(process.env.DEDUPE_THRESHOLD ?? '10', 10);
+  const THRESHOLD = threshold ?? parseInt(process.env.DEDUPE_THRESHOLD ?? '5', 10);
   const start = Date.now();
 
+  // Clear all existing pairs so the result exactly matches the current threshold.
+  // Dismissed pairs are also cleared — a fresh scan is a clean slate.
+  db.raw.prepare('DELETE FROM duplicate_pairs').run();
+
   const photos = db.raw.prepare(
     "SELECT id, phash FROM photos WHERE phash IS NOT NULL AND phash != '0000000000000000'"
   ).all();
@@ -57,34 +61,18 @@ export function getGroups(db) {
   const pairs = db.getDupePairs();
   if (!pairs.length) return [];
 
-  const parent = new Map();
-
-  function find(x) {
-    if (!parent.has(x)) parent.set(x, x);
-    if (parent.get(x) !== x) parent.set(x, find(parent.get(x)));
-    return parent.get(x);
-  }
-
-  function union(a, b) { parent.set(find(a), find(b)); }
-
-  for (const { photo_id_a, photo_id_b } of pairs) union(photo_id_a, photo_id_b);
-  for (const id of [...parent.keys()]) find(id);
-
-  const groups = new Map();
-  for (const id of parent.keys()) {
-    const root = find(id);
-    if (!groups.has(root)) groups.set(root, []);
-    groups.get(root).push(id);
-  }
-
-  const allIds = [...parent.keys()];
+  const allIds = [...new Set(pairs.flatMap(p => [p.photo_id_a, p.photo_id_b]))];
   const rows   = fetchPhotosByIds(db, allIds);
   const photoMap = new Map(rows.map(r => [r.id, r]));
 
-  return [...groups.values()]
-    .filter(ids => ids.length >= 2)
-    .map(ids => ids.map(id => photoMap.get(id)).filter(Boolean))
-    .sort((a, b) => b.length - a.length);
+  // Return each pair as its own 2-photo group.
+  // Union-find chaining caused false "5000-photo" mega-groups through
+  // transitivity (A≈B and B≈C ≠ A≈C). Individual pairs are always accurate.
+  return pairs
+    .map(({ photo_id_a, photo_id_b }) =>
+      [photoMap.get(photo_id_a), photoMap.get(photo_id_b)].filter(Boolean)
+    )
+    .filter(g => g.length === 2);
 }
 
 export function scoreDuplicate(photo) {
diff --git a/worker/db.js b/worker/db.js
index cbe5109..f6a75ce 100644
--- a/worker/db.js
+++ b/worker/db.js
@@ -7,7 +7,7 @@ const SETTING_DEFAULTS = {
   thumb_quality:     '80',
   preview_size:      '1600',
   preview_quality:   '85',
-  dedupe_threshold:  '10',
+  dedupe_threshold:  '5',
   // iCloud Photos sync
   icloud_enabled:    '0',
   icloud_apple_id:   '',