Persistent station records, backups, and iPhone-friendly dashboard
- Keep per-day extremes in data/records.json (merge-only) and derive all-time, notable-day, monthly and yearly records from it; rebuild from raw files at startup - Use station maxdailygust for true gust extremes - Add /api/station/backup.jsonl, records-backup.json and records/rebuild - Dashboard: safe-area insets, 44px touch targets, scrollable nav/range bars, chart touch-action, standalone PWA manifest + apple meta tags Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Nate EmmertandClaude Sonnet 5.5 committed
1 parent
4c2c619fb6
commit
a97d104d14
4 files changed
+315
-60
No files matched your search
+205
-56
@@ -148,6 +148,9 @@ function saveStationReading(obs) {
|
||||
if (seen.has(ts)) return false;
|
||||
fs.appendFileSync(stationDayFile(day), JSON.stringify(obs) + '\n');
|
||||
seen.add(ts);
|
||||
if (typeof updateRecordsFromReading === 'function') {
|
||||
try { updateRecordsFromReading(obs); } catch (e) { console.error('[records] update error:', e.message); }
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -191,6 +194,173 @@ function dedupeStationFiles() {
|
||||
}
|
||||
dedupeStationFiles();
|
||||
|
||||
// ── Persistent all-time records ───────────────────────────────────────────────
|
||||
// data/records.json keeps one small summary per local day (extremes + the time they
|
||||
// happened). All-time / monthly / yearly records are derived from it, so they survive
|
||||
// even if raw readings are pruned or lost. Merging only ever widens an extreme.
|
||||
const RECORDS_FILE = path.join(DATA_DIR, 'records.json');
|
||||
let recordsStore = { version: 1, days: {} };
|
||||
try {
|
||||
const loaded = JSON.parse(fs.readFileSync(RECORDS_FILE, 'utf8'));
|
||||
if (loaded && loaded.days) recordsStore = loaded;
|
||||
} catch {}
|
||||
let recordsDirty = false;
|
||||
|
||||
// [field, source key(s), 'max' | 'min']
|
||||
const DAY_FIELDS = [
|
||||
['hi', ['tempf'], 'max'], ['lo', ['tempf'], 'min'],
|
||||
['gust', ['windgustmph', 'maxdailygust'], 'max'],
|
||||
['rate', ['rainratein'], 'max'],
|
||||
['uv', ['uv'], 'max'], ['solar', ['solarradiation'], 'max'],
|
||||
['pmax', ['baromrelin'], 'max'], ['pmin', ['baromrelin'], 'min'],
|
||||
['hmax', ['humidity'], 'max'], ['hmin', ['humidity'], 'min'],
|
||||
];
|
||||
|
||||
function mergeExtreme(day, field, kind, val, ts) {
|
||||
if (!Number.isFinite(val)) return false;
|
||||
const cur = day[field];
|
||||
if (!cur || (kind === 'max' ? val > cur[0] : val < cur[0])) { day[field] = [val, ts]; return true; }
|
||||
return false;
|
||||
}
|
||||
|
||||
function applyReadingToRecords(obs) {
|
||||
const dayKey = stationLocalDay(obs);
|
||||
const day = recordsStore.days[dayKey] || (recordsStore.days[dayKey] = { n: 0 });
|
||||
let changed = false;
|
||||
for (const [field, keys, kind] of DAY_FIELDS) {
|
||||
for (const k of keys) {
|
||||
if (obs[k] != null && mergeExtreme(day, field, kind, Number(obs[k]), obs.dateutc)) changed = true;
|
||||
}
|
||||
}
|
||||
if (obs.dailyrainin != null && Number.isFinite(Number(obs.dailyrainin)) && !(Number(obs.dailyrainin) <= (day.rain || 0))) {
|
||||
day.rain = Number(obs.dailyrainin); changed = true;
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
|
||||
function saveRecordsStore() {
|
||||
if (!recordsDirty) return;
|
||||
try {
|
||||
fs.writeFileSync(RECORDS_FILE + '.tmp', JSON.stringify(recordsStore));
|
||||
fs.renameSync(RECORDS_FILE + '.tmp', RECORDS_FILE);
|
||||
recordsDirty = false;
|
||||
} catch (e) { console.error('[records] save error:', e.message); }
|
||||
}
|
||||
|
||||
function updateRecordsFromReading(obs) {
|
||||
const before = (recordsStore.days[stationLocalDay(obs)] || {}).n || 0;
|
||||
applyReadingToRecords(obs);
|
||||
recordsStore.days[stationLocalDay(obs)].n = before + 1;
|
||||
recordsDirty = true;
|
||||
}
|
||||
|
||||
// Re-scan raw readings and merge into the store (never lowers an existing record).
|
||||
function rebuildRecordsFromFiles() {
|
||||
let files;
|
||||
try { files = fs.readdirSync(STATION_DIR).filter(f => f.endsWith('.jsonl')).sort(); } catch { return 0; }
|
||||
const counts = {};
|
||||
let scanned = 0;
|
||||
for (const f of files) {
|
||||
try {
|
||||
const seen = new Set();
|
||||
for (const l of fs.readFileSync(path.join(STATION_DIR, f), 'utf8').split('\n')) {
|
||||
if (!l) continue;
|
||||
let r; try { r = JSON.parse(l); } catch { continue; }
|
||||
if (!Number.isFinite(r.dateutc) || seen.has(r.dateutc)) continue;
|
||||
seen.add(r.dateutc);
|
||||
applyReadingToRecords(r);
|
||||
const k = stationLocalDay(r);
|
||||
counts[k] = (counts[k] || 0) + 1;
|
||||
scanned++;
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
// Reading counts come from the files; keep the larger so pruned history isn't forgotten.
|
||||
for (const [k, n] of Object.entries(counts)) recordsStore.days[k].n = Math.max(recordsStore.days[k].n || 0, n);
|
||||
recordsDirty = true;
|
||||
saveRecordsStore();
|
||||
return scanned;
|
||||
}
|
||||
|
||||
function deriveRecords() {
|
||||
const days = Object.entries(recordsStore.days).sort(([a], [b]) => a.localeCompare(b));
|
||||
if (!days.length) return null;
|
||||
const top = (field, kind) => {
|
||||
let best = null, bestDay = null;
|
||||
for (const [d, v] of days) {
|
||||
const e = v[field]; if (!e) continue;
|
||||
if (best == null || (kind === 'max' ? e[0] > best[0] : e[0] < best[0])) { best = e; bestDay = d; }
|
||||
}
|
||||
return best ? { value: best[0], ts: best[1], day: bestDay } : null;
|
||||
};
|
||||
const rec = {
|
||||
highTemp: top('hi', 'max'), lowTemp: top('lo', 'min'), highGust: top('gust', 'max'),
|
||||
highRainRate: top('rate', 'max'), highUV: top('uv', 'max'), highSolar: top('solar', 'max'),
|
||||
highPressure: top('pmax', 'max'), lowPressure: top('pmin', 'min'),
|
||||
highHumidity: top('hmax', 'max'), lowHumidity: top('hmin', 'min'),
|
||||
};
|
||||
let wet = null;
|
||||
for (const [d, v] of days) if (v.rain != null && (!wet || v.rain > wet.value)) wet = { value: v.rain, day: d };
|
||||
rec.highDailyRain = wet;
|
||||
|
||||
const out = {};
|
||||
const flat = {
|
||||
highTemp: 'highTemp', lowTemp: 'lowTemp', highGust: 'highGust', highRainRate: 'highRainRate',
|
||||
highDailyRain: 'highDailyRain', highUV: 'highUV', highSolar: 'highSolar',
|
||||
highPressure: 'highPressure', lowPressure: 'lowPressure', highHumidity: 'highHumidity', lowHumidity: 'lowHumidity',
|
||||
};
|
||||
for (const [k, src] of Object.entries(flat)) {
|
||||
out[k] = rec[src] ? rec[src].value : null;
|
||||
out[k + 'Date'] = rec[src] ? (rec[src].ts || Date.parse(rec[src].day + 'T12:00:00Z')) : null;
|
||||
}
|
||||
out.totalReadings = days.reduce((a, [, v]) => a + (v.n || 0), 0);
|
||||
out.firstDay = days[0][0];
|
||||
out.lastDay = days[days.length - 1][0];
|
||||
out.firstReading = Date.parse(out.firstDay + 'T12:00:00Z');
|
||||
out.lastReading = Date.parse(out.lastDay + 'T12:00:00Z');
|
||||
out.daysRecorded = days.length;
|
||||
|
||||
// Notable days
|
||||
const dayRec = (field, kind) => { const t = top(field, kind); return t ? { value: t.value, day: t.day } : null; };
|
||||
out.notable = {
|
||||
hottestDay: dayRec('hi', 'max'),
|
||||
coldestDay: dayRec('lo', 'min'),
|
||||
coolestHigh: dayRec('hi', 'min'),
|
||||
warmestLow: dayRec('lo', 'max'),
|
||||
windiestDay: dayRec('gust', 'max'),
|
||||
wettestDay: rec.highDailyRain,
|
||||
};
|
||||
|
||||
// Per-period aggregation (month-of-year and calendar year)
|
||||
const agg = (keyFn) => {
|
||||
const g = {};
|
||||
for (const [d, v] of days) {
|
||||
const k = keyFn(d); const a = g[k] || (g[k] = { days: 0, rain: 0, hi: null, lo: null, gust: null });
|
||||
a.days++; a.rain += v.rain || 0;
|
||||
if (v.hi && (a.hi == null || v.hi[0] > a.hi)) a.hi = v.hi[0];
|
||||
if (v.lo && (a.lo == null || v.lo[0] < a.lo)) a.lo = v.lo[0];
|
||||
if (v.gust && (a.gust == null || v.gust[0] > a.gust)) a.gust = v.gust[0];
|
||||
}
|
||||
for (const a of Object.values(g)) a.rain = +a.rain.toFixed(2);
|
||||
return g;
|
||||
};
|
||||
const byYear = agg(d => d.slice(0, 4));
|
||||
out.years = Object.entries(byYear).sort(([a], [b]) => a.localeCompare(b)).map(([year, a]) => ({ year, ...a }));
|
||||
const byMonth = agg(d => d.slice(0, 7));
|
||||
const ym = Object.entries(byMonth).sort(([a], [b]) => a.localeCompare(b)).map(([month, a]) => ({ month, ...a }));
|
||||
out.months = ym;
|
||||
out.wettestMonth = ym.reduce((b, m) => (!b || m.rain > b.rain ? m : b), null);
|
||||
out.wettestYear = out.years.reduce((b, y) => (!b || y.rain > b.rain ? y : b), null);
|
||||
return out;
|
||||
}
|
||||
|
||||
try {
|
||||
if (!Object.keys(recordsStore.days).length) console.log(`[records] built from ${rebuildRecordsFromFiles()} readings`);
|
||||
else rebuildRecordsFromFiles();
|
||||
} catch (e) { console.error('[records] startup rebuild error:', e.message); }
|
||||
setInterval(saveRecordsStore, 30 * 1000).unref();
|
||||
for (const sig of ['SIGTERM', 'SIGINT']) process.on(sig, () => { saveRecordsStore(); process.exit(0); });
|
||||
|
||||
function loadStationReadings(rangeMs) {
|
||||
const now = Date.now();
|
||||
const cutoff = now - rangeMs;
|
||||
@@ -1244,65 +1414,44 @@ app.get('/api/station/backfill/status', requireDashboardAuth, (_req, res) => {
|
||||
|
||||
// ── Station analytics ─────────────────────────────────────────────────────────
|
||||
|
||||
// Records — scan all JSONL, cache 1h
|
||||
let recordsCache = null;
|
||||
let recordsCacheAt = 0;
|
||||
|
||||
function computeStationRecords() {
|
||||
let files;
|
||||
try { files = fs.readdirSync(STATION_DIR).filter(f => f.endsWith('.jsonl')).sort(); } catch { return null; }
|
||||
if (!files.length) return null;
|
||||
const rec = {
|
||||
highTemp: null, highTempDate: null, lowTemp: null, lowTempDate: null,
|
||||
highGust: null, highGustDate: null,
|
||||
highRainRate: null, highRainRateDate: null, highDailyRain: null, highDailyRainDate: null,
|
||||
highUV: null, highUVDate: null, highSolar: null, highSolarDate: null,
|
||||
highPressure: null, highPressureDate: null, lowPressure: null, lowPressureDate: null,
|
||||
highHumidity: null, highHumidityDate: null, lowHumidity: null, lowHumidityDate: null,
|
||||
totalReadings: 0, firstReading: null, lastReading: null,
|
||||
};
|
||||
for (const file of files) {
|
||||
try {
|
||||
const lines = fs.readFileSync(path.join(STATION_DIR, file), 'utf8').split('\n').filter(Boolean);
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const r = JSON.parse(line);
|
||||
if (!r.dateutc) continue;
|
||||
rec.totalReadings++;
|
||||
if (!rec.firstReading || r.dateutc < rec.firstReading) rec.firstReading = r.dateutc;
|
||||
if (!rec.lastReading || r.dateutc > rec.lastReading) rec.lastReading = r.dateutc;
|
||||
if (r.tempf != null) {
|
||||
if (rec.highTemp == null || r.tempf > rec.highTemp) { rec.highTemp = r.tempf; rec.highTempDate = r.dateutc; }
|
||||
if (rec.lowTemp == null || r.tempf < rec.lowTemp) { rec.lowTemp = r.tempf; rec.lowTempDate = r.dateutc; }
|
||||
}
|
||||
if (r.windgustmph != null && (rec.highGust == null || r.windgustmph > rec.highGust)) { rec.highGust = r.windgustmph; rec.highGustDate = r.dateutc; }
|
||||
if (r.rainratein != null && (rec.highRainRate == null || r.rainratein > rec.highRainRate)) { rec.highRainRate = r.rainratein; rec.highRainRateDate = r.dateutc; }
|
||||
if (r.dailyrainin != null && (rec.highDailyRain == null || r.dailyrainin > rec.highDailyRain)) { rec.highDailyRain = r.dailyrainin; rec.highDailyRainDate = r.dateutc; }
|
||||
if (r.uv != null && (rec.highUV == null || r.uv > rec.highUV)) { rec.highUV = r.uv; rec.highUVDate = r.dateutc; }
|
||||
if (r.solarradiation != null && (rec.highSolar == null || r.solarradiation > rec.highSolar)) { rec.highSolar = r.solarradiation; rec.highSolarDate = r.dateutc; }
|
||||
if (r.baromrelin != null) {
|
||||
if (rec.highPressure == null || r.baromrelin > rec.highPressure) { rec.highPressure = r.baromrelin; rec.highPressureDate = r.dateutc; }
|
||||
if (rec.lowPressure == null || r.baromrelin < rec.lowPressure) { rec.lowPressure = r.baromrelin; rec.lowPressureDate = r.dateutc; }
|
||||
}
|
||||
if (r.humidity != null) {
|
||||
if (rec.highHumidity == null || r.humidity > rec.highHumidity) { rec.highHumidity = r.humidity; rec.highHumidityDate = r.dateutc; }
|
||||
if (rec.lowHumidity == null || r.humidity < rec.lowHumidity) { rec.lowHumidity = r.humidity; rec.lowHumidityDate = r.dateutc; }
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
return rec;
|
||||
}
|
||||
|
||||
// Records — derived from the persistent per-day store (data/records.json)
|
||||
app.get('/api/station/records', requireDashboardAuth, (_req, res) => {
|
||||
if (recordsCache && Date.now() - recordsCacheAt < 3600000) return res.json(recordsCache);
|
||||
recordsCache = computeStationRecords();
|
||||
recordsCacheAt = Date.now();
|
||||
res.json(recordsCache);
|
||||
res.json(deriveRecords());
|
||||
});
|
||||
|
||||
// Re-scan raw readings into the records store (merge only; never lowers a record)
|
||||
app.post('/api/station/records/rebuild', requireDashboardAuth, (_req, res) => {
|
||||
const scanned = rebuildRecordsFromFiles();
|
||||
res.json({ ok: true, scanned });
|
||||
});
|
||||
|
||||
// Full backup: every saved reading (deduped, oldest first) as newline-delimited JSON
|
||||
app.get('/api/station/backup.jsonl', requireDashboardAuth, (_req, res) => {
|
||||
let files;
|
||||
try { files = fs.readdirSync(STATION_DIR).filter(f => f.endsWith('.jsonl')).sort(); } catch { files = []; }
|
||||
res.setHeader('Content-Type', 'application/x-ndjson');
|
||||
res.setHeader('Content-Disposition', `attachment; filename="station-backup-${new Date().toISOString().slice(0, 10)}.jsonl"`);
|
||||
for (const f of files) {
|
||||
try {
|
||||
const seen = new Set();
|
||||
const rows = [];
|
||||
for (const l of fs.readFileSync(path.join(STATION_DIR, f), 'utf8').split('\n')) {
|
||||
if (!l) continue;
|
||||
try { const t = JSON.parse(l).dateutc; if (seen.has(t)) continue; seen.add(t); } catch {}
|
||||
rows.push(l);
|
||||
}
|
||||
if (rows.length) res.write(rows.join('\n') + '\n');
|
||||
} catch {}
|
||||
}
|
||||
res.end();
|
||||
});
|
||||
|
||||
// Records store (per-day extremes) as JSON
|
||||
app.get('/api/station/records-backup.json', requireDashboardAuth, (_req, res) => {
|
||||
res.setHeader('Content-Disposition', `attachment; filename="station-records-${new Date().toISOString().slice(0, 10)}.json"`);
|
||||
res.json(recordsStore);
|
||||
});
|
||||
|
||||
// Rain totals — max dailyrainin per calendar day, summed
|
||||
app.get('/api/station/rain-totals', requireDashboardAuth, (_req, res) => {
|
||||
const now = Date.now();
|
||||
const ytdCutoff = new Date(new Date().getFullYear(), 0, 1).getTime();
|
||||
|
||||
Reference in new issue
Block a user