fix: dedupe scan 500 error, SQLite limit crash, add keep-best action

- Replace BigInt hammingDistance with 32-bit popcount — safer (no throw
  on malformed hashes) and ~10× faster
- Make scan() async with setImmediate chunking so the event loop stays
  responsive on large libraries
- Batch getGroups() photo lookup in slices of 900 to stay under SQLite's
  999-variable limit
- Add POST /api/duplicates/keep-best endpoint: scores by resolution then
  filesize, soft-deletes the losers, dismisses the pair
- DuplicateGroup: green "Keep" badge on the best photo, green "Keep best"
  button in footer

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
nmemmertandClaude Sonnet 4.6 committed 2026-08-19 15:45:46 -04:00
1 parent 93e16561ec
commit 4f7dd3b49b
6 files changed
+205 -78

No files matched your search

+62 -15
View File
@@ -1,6 +1,10 @@
import { hammingDistance } from '../worker/hash.js';
export function scan(db, threshold) {
// Chunk size for the outer loop — yields to the event loop between chunks
// so the server stays responsive during a long scan.
const SCAN_CHUNK = 50;
export async function scan(db, threshold) {
const THRESHOLD = threshold ?? parseInt(process.env.DEDUPE_THRESHOLD ?? '10', 10);
const start = Date.now();
@@ -13,21 +17,42 @@ export function scan(db, threshold) {
);
let newPairs = 0;
db.raw.transaction(() => {
for (let i = 0; i < photos.length; i++) {
for (let j = i + 1; j < photos.length; j++) {
const dist = hammingDistance(photos[i].phash, photos[j].phash);
if (dist <= THRESHOLD) {
const r = insert.run(photos[i].id, photos[j].id, dist);
if (r.changes) newPairs++;
for (let i = 0; i < photos.length; i += SCAN_CHUNK) {
await new Promise(resolve => setImmediate(resolve));
const end = Math.min(i + SCAN_CHUNK, photos.length);
db.raw.transaction(() => {
for (let ii = i; ii < end; ii++) {
for (let j = ii + 1; j < photos.length; j++) {
const dist = hammingDistance(photos[ii].phash, photos[j].phash);
if (dist <= THRESHOLD) {
const r = insert.run(photos[ii].id, photos[j].id, dist);
if (r.changes) newPairs++;
}
}
}
}
})();
})();
}
return { photos: photos.length, newPairs, duration: Date.now() - start };
}
const ID_BATCH = 900;
function fetchPhotosByIds(db, ids) {
const rows = [];
for (let i = 0; i < ids.length; i += ID_BATCH) {
const batch = ids.slice(i, i + ID_BATCH);
const got = db.raw.prepare(
`SELECT id, thumb_path, preview_path, date_taken, filesize, width, height, camera_model
FROM photos WHERE id IN (${batch.map(() => '?').join(',')})`
).all(batch);
rows.push(...got);
}
return rows;
}
export function getGroups(db) {
const pairs = db.getDupePairs();
if (!pairs.length) return [];
@@ -53,11 +78,7 @@ export function getGroups(db) {
}
const allIds = [...parent.keys()];
const rows = db.raw.prepare(
`SELECT id, thumb_path, preview_path, date_taken, filesize, width, height, camera_model
FROM photos WHERE id IN (${allIds.map(() => '?').join(',')})`
).all(allIds);
const rows = fetchPhotosByIds(db, allIds);
const photoMap = new Map(rows.map(r => [r.id, r]));
return [...groups.values()]
@@ -65,3 +86,29 @@ export function getGroups(db) {
.map(ids => ids.map(id => photoMap.get(id)).filter(Boolean))
.sort((a, b) => b.length - a.length);
}
export function scoreDuplicate(photo) {
const pixels = (photo.width || 0) * (photo.height || 0);
const size = photo.filesize || 0;
const hasExif = photo.camera_model ? 1 : 0;
return pixels * 1e6 + size + hasExif * 1e3;
}
export function keepBest(db, photoIds) {
const ids = photoIds.map(Number);
const photos = fetchPhotosByIds(db, ids);
if (photos.length < 2) return null;
photos.sort((a, b) => scoreDuplicate(b) - scoreDuplicate(a));
const keeper = photos[0];
const discard = photos.slice(1);
db.raw.transaction(() => {
for (const p of discard) {
db.raw.prepare('UPDATE photos SET deleted = 1 WHERE id = ?').run(p.id);
}
})();
db.dismissDuplicates(ids);
return { kept: keeper.id, deleted: discard.map(p => p.id) };
}