fix: dedupe scan 500 error, SQLite limit crash, add keep-best action
- Replace BigInt hammingDistance with 32-bit popcount — safer (no throw on malformed hashes) and ~10× faster - Make scan() async with setImmediate chunking so the event loop stays responsive on large libraries - Batch getGroups() photo lookup in slices of 900 to stay under SQLite's 999-variable limit - Add POST /api/duplicates/keep-best endpoint: scores by resolution then filesize, soft-deletes the losers, dismisses the pair - DuplicateGroup: green "Keep" badge on the best photo, green "Keep best" button in footer Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
93e16561ec
commit
4f7dd3b49b
6 files changed
+205
-78
No files matched your search
+62
-15
@@ -1,6 +1,10 @@
|
||||
import { hammingDistance } from '../worker/hash.js';
|
||||
|
||||
export function scan(db, threshold) {
|
||||
// Chunk size for the outer loop — yields to the event loop between chunks
|
||||
// so the server stays responsive during a long scan.
|
||||
const SCAN_CHUNK = 50;
|
||||
|
||||
export async function scan(db, threshold) {
|
||||
const THRESHOLD = threshold ?? parseInt(process.env.DEDUPE_THRESHOLD ?? '10', 10);
|
||||
const start = Date.now();
|
||||
|
||||
@@ -13,21 +17,42 @@ export function scan(db, threshold) {
|
||||
);
|
||||
|
||||
let newPairs = 0;
|
||||
db.raw.transaction(() => {
|
||||
for (let i = 0; i < photos.length; i++) {
|
||||
for (let j = i + 1; j < photos.length; j++) {
|
||||
const dist = hammingDistance(photos[i].phash, photos[j].phash);
|
||||
if (dist <= THRESHOLD) {
|
||||
const r = insert.run(photos[i].id, photos[j].id, dist);
|
||||
if (r.changes) newPairs++;
|
||||
|
||||
for (let i = 0; i < photos.length; i += SCAN_CHUNK) {
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
const end = Math.min(i + SCAN_CHUNK, photos.length);
|
||||
|
||||
db.raw.transaction(() => {
|
||||
for (let ii = i; ii < end; ii++) {
|
||||
for (let j = ii + 1; j < photos.length; j++) {
|
||||
const dist = hammingDistance(photos[ii].phash, photos[j].phash);
|
||||
if (dist <= THRESHOLD) {
|
||||
const r = insert.run(photos[ii].id, photos[j].id, dist);
|
||||
if (r.changes) newPairs++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})();
|
||||
})();
|
||||
}
|
||||
|
||||
return { photos: photos.length, newPairs, duration: Date.now() - start };
|
||||
}
|
||||
|
||||
const ID_BATCH = 900;
|
||||
|
||||
function fetchPhotosByIds(db, ids) {
|
||||
const rows = [];
|
||||
for (let i = 0; i < ids.length; i += ID_BATCH) {
|
||||
const batch = ids.slice(i, i + ID_BATCH);
|
||||
const got = db.raw.prepare(
|
||||
`SELECT id, thumb_path, preview_path, date_taken, filesize, width, height, camera_model
|
||||
FROM photos WHERE id IN (${batch.map(() => '?').join(',')})`
|
||||
).all(batch);
|
||||
rows.push(...got);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
export function getGroups(db) {
|
||||
const pairs = db.getDupePairs();
|
||||
if (!pairs.length) return [];
|
||||
@@ -53,11 +78,7 @@ export function getGroups(db) {
|
||||
}
|
||||
|
||||
const allIds = [...parent.keys()];
|
||||
const rows = db.raw.prepare(
|
||||
`SELECT id, thumb_path, preview_path, date_taken, filesize, width, height, camera_model
|
||||
FROM photos WHERE id IN (${allIds.map(() => '?').join(',')})`
|
||||
).all(allIds);
|
||||
|
||||
const rows = fetchPhotosByIds(db, allIds);
|
||||
const photoMap = new Map(rows.map(r => [r.id, r]));
|
||||
|
||||
return [...groups.values()]
|
||||
@@ -65,3 +86,29 @@ export function getGroups(db) {
|
||||
.map(ids => ids.map(id => photoMap.get(id)).filter(Boolean))
|
||||
.sort((a, b) => b.length - a.length);
|
||||
}
|
||||
|
||||
export function scoreDuplicate(photo) {
|
||||
const pixels = (photo.width || 0) * (photo.height || 0);
|
||||
const size = photo.filesize || 0;
|
||||
const hasExif = photo.camera_model ? 1 : 0;
|
||||
return pixels * 1e6 + size + hasExif * 1e3;
|
||||
}
|
||||
|
||||
export function keepBest(db, photoIds) {
|
||||
const ids = photoIds.map(Number);
|
||||
const photos = fetchPhotosByIds(db, ids);
|
||||
if (photos.length < 2) return null;
|
||||
|
||||
photos.sort((a, b) => scoreDuplicate(b) - scoreDuplicate(a));
|
||||
const keeper = photos[0];
|
||||
const discard = photos.slice(1);
|
||||
|
||||
db.raw.transaction(() => {
|
||||
for (const p of discard) {
|
||||
db.raw.prepare('UPDATE photos SET deleted = 1 WHERE id = ?').run(p.id);
|
||||
}
|
||||
})();
|
||||
|
||||
db.dismissDuplicates(ids);
|
||||
return { kept: keeper.id, deleted: discard.map(p => p.id) };
|
||||
}
|
||||
Reference in new issue
Block a user