feat: initial Luma implementation
Full self-hosted photo manager with: - Ingestion worker (chokidar + sharp + exifr, dHash perceptual hashing) - SQLite schema with WAL mode (photos, tags, collections, duplicate_pairs) - Express API with keyset pagination, tag/collection/search/date filters - React + Vite UI: timeline, lightbox, detail panel, tag/collection editors - Duplicate detection (Union-Find grouping) with review + dismiss UI - Map view (Leaflet + OpenStreetMap, marker clustering) - Docker multi-stage build + Compose with inline env config - GitHub Actions: CI build check + ghcr.io image publish on main/tags Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
nmemmertandClaude Sonnet 4.6 committed
1 parent
03e6d6b628
commit
e0f4e8ff0d
32 files changed
+4845
No files matched your search
+161
@@ -0,0 +1,161 @@
|
||||
import Database from 'better-sqlite3';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
|
||||
export function openDb(dbPath) {
|
||||
fs.mkdirSync(path.dirname(path.resolve(dbPath)), { recursive: true });
|
||||
|
||||
const db = new Database(dbPath);
|
||||
db.pragma('journal_mode = WAL');
|
||||
db.pragma('foreign_keys = ON');
|
||||
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS photos (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
filepath TEXT NOT NULL UNIQUE,
|
||||
thumb_path TEXT,
|
||||
preview_path TEXT,
|
||||
date_taken TEXT,
|
||||
date_imported TEXT NOT NULL,
|
||||
camera_model TEXT,
|
||||
gps_lat REAL,
|
||||
gps_lon REAL,
|
||||
phash TEXT,
|
||||
width INTEGER,
|
||||
height INTEGER,
|
||||
filesize INTEGER
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_photos_date_taken ON photos(date_taken);
|
||||
CREATE INDEX IF NOT EXISTS idx_photos_phash ON photos(phash);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS collections (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
description TEXT,
|
||||
poster_path TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS photo_collections (
|
||||
photo_id INTEGER NOT NULL REFERENCES photos(id) ON DELETE CASCADE,
|
||||
collection_id INTEGER NOT NULL REFERENCES collections(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (photo_id, collection_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tags (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS photo_tags (
|
||||
photo_id INTEGER NOT NULL REFERENCES photos(id) ON DELETE CASCADE,
|
||||
tag_id INTEGER NOT NULL REFERENCES tags(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (photo_id, tag_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS duplicate_pairs (
|
||||
photo_id_a INTEGER NOT NULL REFERENCES photos(id) ON DELETE CASCADE,
|
||||
photo_id_b INTEGER NOT NULL REFERENCES photos(id) ON DELETE CASCADE,
|
||||
distance INTEGER NOT NULL,
|
||||
dismissed INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (photo_id_a, photo_id_b),
|
||||
CHECK (photo_id_a < photo_id_b)
|
||||
);
|
||||
`);
|
||||
|
||||
const s = {
|
||||
// Photos
|
||||
insert: db.prepare(`
|
||||
INSERT OR IGNORE INTO photos
|
||||
(filepath, thumb_path, preview_path, date_taken, date_imported,
|
||||
camera_model, gps_lat, gps_lon, phash, width, height, filesize)
|
||||
VALUES
|
||||
(@filepath, @thumb_path, @preview_path, @date_taken, @date_imported,
|
||||
@camera_model, @gps_lat, @gps_lon, @phash, @width, @height, @filesize)
|
||||
`),
|
||||
exists: db.prepare('SELECT 1 FROM photos WHERE filepath = ? LIMIT 1'),
|
||||
remove: db.prepare('DELETE FROM photos WHERE filepath = ?'),
|
||||
count: db.prepare('SELECT COUNT(*) AS n FROM photos'),
|
||||
|
||||
// Tags
|
||||
tagByName: db.prepare('SELECT id, name FROM tags WHERE name = ?'),
|
||||
tagInsert: db.prepare('INSERT OR IGNORE INTO tags (name) VALUES (?)'),
|
||||
tagList: db.prepare(`
|
||||
SELECT t.id, t.name, COUNT(pt.photo_id) AS count
|
||||
FROM tags t LEFT JOIN photo_tags pt ON pt.tag_id = t.id
|
||||
GROUP BY t.id ORDER BY t.name
|
||||
`),
|
||||
tagOfPhoto: db.prepare(
|
||||
'SELECT t.id, t.name FROM tags t JOIN photo_tags pt ON pt.tag_id = t.id WHERE pt.photo_id = ? ORDER BY t.name'
|
||||
),
|
||||
tagAttach: db.prepare('INSERT OR IGNORE INTO photo_tags (photo_id, tag_id) VALUES (?, ?)'),
|
||||
tagDetach: db.prepare('DELETE FROM photo_tags WHERE photo_id = ? AND tag_id = ?'),
|
||||
|
||||
// Collections
|
||||
colList: db.prepare(`
|
||||
SELECT c.id, c.name, c.description, c.created_at,
|
||||
COUNT(pc.photo_id) AS photo_count
|
||||
FROM collections c
|
||||
LEFT JOIN photo_collections pc ON pc.collection_id = c.id
|
||||
GROUP BY c.id ORDER BY c.name
|
||||
`),
|
||||
colInsert: db.prepare('INSERT INTO collections (name, description) VALUES (?, ?)'),
|
||||
colOfPhoto: db.prepare(
|
||||
'SELECT c.id, c.name FROM collections c JOIN photo_collections pc ON pc.collection_id = c.id WHERE pc.photo_id = ? ORDER BY c.name'
|
||||
),
|
||||
colAdd: db.prepare('INSERT OR IGNORE INTO photo_collections (photo_id, collection_id) VALUES (?, ?)'),
|
||||
colRemove: db.prepare('DELETE FROM photo_collections WHERE photo_id = ? AND collection_id = ?'),
|
||||
|
||||
// Duplicates
|
||||
dupePairs: db.prepare(
|
||||
'SELECT photo_id_a, photo_id_b, distance FROM duplicate_pairs WHERE dismissed = 0'
|
||||
),
|
||||
dupePairCount: db.prepare(
|
||||
'SELECT COUNT(*) AS n FROM duplicate_pairs WHERE dismissed = 0'
|
||||
),
|
||||
};
|
||||
|
||||
const addTagTx = db.transaction((photoId, tagName) => {
|
||||
s.tagInsert.run(tagName);
|
||||
const tag = s.tagByName.get(tagName);
|
||||
s.tagAttach.run(photoId, tag.id);
|
||||
return tag;
|
||||
});
|
||||
|
||||
return {
|
||||
raw: db,
|
||||
|
||||
// Photos
|
||||
insertPhoto: (row) => s.insert.run(row),
|
||||
hasPhoto: (fp) => !!s.exists.get(fp),
|
||||
removePhoto: (fp) => s.remove.run(fp),
|
||||
count: () => s.count.get().n,
|
||||
|
||||
// Tags
|
||||
addTagToPhoto: (photoId, tagName) => addTagTx(photoId, tagName),
|
||||
removeTagFromPhoto: (photoId, tagId) => s.tagDetach.run(photoId, tagId),
|
||||
getPhotoTags: (photoId) => s.tagOfPhoto.all(photoId),
|
||||
listAllTags: () => s.tagList.all(),
|
||||
|
||||
// Collections
|
||||
listCollections: () => s.colList.all(),
|
||||
createCollection: (name, desc) => {
|
||||
const info = s.colInsert.run(name, desc ?? null);
|
||||
return { id: info.lastInsertRowid, name, description: desc ?? null };
|
||||
},
|
||||
addPhotoToCollection: (photoId, colId) => s.colAdd.run(photoId, colId),
|
||||
removePhotoFromCollection:(photoId, colId) => s.colRemove.run(photoId, colId),
|
||||
getPhotoCollections: (photoId) => s.colOfPhoto.all(photoId),
|
||||
|
||||
// Duplicates
|
||||
getDupePairs: () => s.dupePairs.all(),
|
||||
dismissDuplicates: (ids) => {
|
||||
const ph = ids.map(() => '?').join(',');
|
||||
db.prepare(
|
||||
`UPDATE duplicate_pairs SET dismissed = 1
|
||||
WHERE photo_id_a IN (${ph}) AND photo_id_b IN (${ph})`
|
||||
).run(...ids, ...ids);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import sharp from 'sharp';
|
||||
|
||||
// dHash (difference hash): 9×8 → 64-bit comparison → 16-char hex string.
|
||||
// Two images with Hamming distance ≤ 10 are near-duplicates.
|
||||
export async function computeDHash(filepath) {
|
||||
const { data } = await sharp(filepath)
|
||||
.resize(9, 8, { fit: 'fill' })
|
||||
.greyscale()
|
||||
.raw()
|
||||
.toBuffer({ resolveWithObject: true });
|
||||
|
||||
let hash = 0n;
|
||||
for (let row = 0; row < 8; row++) {
|
||||
for (let col = 0; col < 8; col++) {
|
||||
const left = data[row * 9 + col];
|
||||
const right = data[row * 9 + col + 1];
|
||||
hash = (hash << 1n) | (left < right ? 1n : 0n);
|
||||
}
|
||||
}
|
||||
return hash.toString(16).padStart(16, '0');
|
||||
}
|
||||
|
||||
export function hammingDistance(a, b) {
|
||||
let xor = BigInt('0x' + a) ^ BigInt('0x' + b);
|
||||
let dist = 0;
|
||||
while (xor > 0n) {
|
||||
dist += Number(xor & 1n);
|
||||
xor >>= 1n;
|
||||
}
|
||||
return dist;
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import 'dotenv/config';
|
||||
import chokidar from 'chokidar';
|
||||
import path from 'path';
|
||||
import { openDb } from './db.js';
|
||||
import { ingestFile, isSupported } from './ingest.js';
|
||||
|
||||
const PHOTO_DIR = process.env.PHOTO_DIR ?? './photos';
|
||||
const DB_PATH = process.env.DB_PATH ?? './data/library.db';
|
||||
const POSTER_DIR = process.env.POSTER_DIR ?? './data/posters';
|
||||
|
||||
const db = openDb(DB_PATH);
|
||||
|
||||
console.log(`[worker] watching ${path.resolve(PHOTO_DIR)}`);
|
||||
console.log(`[worker] db at ${path.resolve(DB_PATH)} (${db.count()} photos indexed)`);
|
||||
|
||||
// Serial queue: chokidar fires many 'add' events at once on startup;
|
||||
// process them one at a time to keep memory and I/O pressure low.
|
||||
const queue = [];
|
||||
let busy = false;
|
||||
|
||||
async function drain() {
|
||||
if (busy) return;
|
||||
busy = true;
|
||||
while (queue.length > 0) {
|
||||
const fp = queue.shift();
|
||||
try {
|
||||
await ingestFile(fp, db, POSTER_DIR);
|
||||
} catch (err) {
|
||||
console.error(`[worker] ${path.basename(fp)}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
busy = false;
|
||||
}
|
||||
|
||||
const watcher = chokidar.watch(PHOTO_DIR, {
|
||||
persistent: true,
|
||||
ignoreInitial: false,
|
||||
awaitWriteFinish: { stabilityThreshold: 2000, pollInterval: 100 },
|
||||
ignored: /(^|[/\\])\../, // skip dotfiles
|
||||
});
|
||||
|
||||
watcher.on('add', (fp) => {
|
||||
if (!isSupported(fp)) return;
|
||||
queue.push(path.resolve(fp));
|
||||
drain();
|
||||
});
|
||||
|
||||
watcher.on('unlink', (fp) => {
|
||||
const resolved = path.resolve(fp);
|
||||
db.removePhoto(resolved);
|
||||
console.log(`[worker] removed ${path.basename(fp)}`);
|
||||
});
|
||||
|
||||
watcher.on('error', (err) => console.error('[worker] watcher error:', err));
|
||||
|
||||
process.on('SIGINT', () => {
|
||||
console.log('\n[worker] shutting down');
|
||||
watcher.close();
|
||||
process.exit(0);
|
||||
});
|
||||
@@ -0,0 +1,94 @@
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import crypto from 'crypto';
|
||||
import sharp from 'sharp';
|
||||
import exifr from 'exifr';
|
||||
import { computeDHash } from './hash.js';
|
||||
|
||||
const SUPPORTED = new Set([
|
||||
'.jpg', '.jpeg', '.png', '.webp', '.heic', '.heif',
|
||||
'.tiff', '.tif', '.gif', '.avif',
|
||||
]);
|
||||
|
||||
export function isSupported(filepath) {
|
||||
return SUPPORTED.has(path.extname(filepath).toLowerCase());
|
||||
}
|
||||
|
||||
export async function ingestFile(filepath, db, posterDir) {
|
||||
if (db.hasPhoto(filepath)) return;
|
||||
|
||||
const stat = fs.statSync(filepath);
|
||||
|
||||
let exif = {};
|
||||
try {
|
||||
exif = await exifr.parse(filepath, {
|
||||
pick: ['DateTimeOriginal', 'Make', 'Model', 'Orientation'],
|
||||
gps: true,
|
||||
}) ?? {};
|
||||
} catch {}
|
||||
|
||||
const dateTaken = exif.DateTimeOriginal ?? null;
|
||||
const cameraModel = [exif.Make, exif.Model].filter(Boolean).join(' ') || null;
|
||||
const gpsLat = exif.latitude ?? null;
|
||||
const gpsLon = exif.longitude ?? null;
|
||||
|
||||
const { dir, thumb, preview } = posterPaths(filepath, posterDir, dateTaken ?? stat.mtime);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
|
||||
let width = null, height = null;
|
||||
|
||||
try {
|
||||
const meta = await sharp(filepath, { failOn: 'none' }).metadata();
|
||||
width = meta.width;
|
||||
height = meta.height;
|
||||
|
||||
await sharp(filepath, { failOn: 'none' })
|
||||
.rotate()
|
||||
.resize(400, 400, { fit: 'inside', withoutEnlargement: true })
|
||||
.jpeg({ quality: 80 })
|
||||
.toFile(thumb);
|
||||
|
||||
await sharp(filepath, { failOn: 'none' })
|
||||
.rotate()
|
||||
.resize(1600, 1600, { fit: 'inside', withoutEnlargement: true })
|
||||
.jpeg({ quality: 85 })
|
||||
.toFile(preview);
|
||||
} catch (err) {
|
||||
console.error(`[ingest] poster failed ${path.basename(filepath)}: ${err.message}`);
|
||||
return;
|
||||
}
|
||||
|
||||
let phash = null;
|
||||
try { phash = await computeDHash(filepath); } catch {}
|
||||
|
||||
db.insertPhoto({
|
||||
filepath,
|
||||
thumb_path: thumb,
|
||||
preview_path: preview,
|
||||
date_taken: dateTaken ? new Date(dateTaken).toISOString() : null,
|
||||
date_imported: new Date().toISOString(),
|
||||
camera_model: cameraModel,
|
||||
gps_lat: gpsLat,
|
||||
gps_lon: gpsLon,
|
||||
phash,
|
||||
width,
|
||||
height,
|
||||
filesize: stat.size,
|
||||
});
|
||||
|
||||
console.log(`[ingest] ${path.basename(filepath)}`);
|
||||
}
|
||||
|
||||
function posterPaths(filepath, posterDir, date) {
|
||||
const id = crypto.createHash('sha1').update(filepath).digest('hex').slice(0, 10);
|
||||
const stem = path.basename(filepath, path.extname(filepath));
|
||||
const name = `${id}_${stem}`;
|
||||
const d = new Date(date);
|
||||
const folder = `${d.getFullYear()}/${String(d.getMonth() + 1).padStart(2, '0')}`;
|
||||
const dir = path.join(posterDir, folder);
|
||||
return {
|
||||
dir,
|
||||
thumb: path.join(dir, `${name}_thumb.jpg`),
|
||||
preview: path.join(dir, `${name}_preview.jpg`),
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user