diff --git a/app/public/dashboard.html b/app/public/dashboard.html index f33aa6b..c1e50ea 100644 --- a/app/public/dashboard.html +++ b/app/public/dashboard.html @@ -90,6 +90,14 @@ a { color: var(--cool); text-decoration: none; } .keys-status.ok { color: var(--cool); } .keys-status.missing { color: var(--accent); } .keys-hint { font-size: 0.78rem; color: var(--ink-light); margin-top: 1rem; line-height: 1.5; } +.settings-divider { border: none; border-top: 1px solid var(--border); margin: 1.25rem 0; } +.backfill-header { display: flex; align-items: flex-start; justify-content: space-between; gap: 1rem; flex-wrap: wrap; } +.backfill-controls { display: flex; gap: 0.5rem; align-items: center; flex-shrink: 0; } +.backfill-controls select { padding: 0.45rem 0.6rem; background: var(--paper); border: 1px solid var(--border); border-radius: 8px; color: var(--ink); font-family: var(--mono); font-size: 0.85rem; } +.progress-bar-wrap { height: 6px; background: var(--paper-darker); border-radius: 3px; margin-top: 1rem; overflow: hidden; } +.progress-bar { height: 100%; background: var(--cool); border-radius: 3px; transition: width 0.5s; width: 0%; } +.progress-text { font-size: 0.82rem; color: var(--ink-light); margin-top: 0.4rem; } +.backfill-progress { margin-top: 0.75rem; } @@ -163,6 +171,31 @@ a { color: var(--cool); text-decoration: none; }

Find these at ambientweather.net → My Account → API Keys. Keys are stored on the server only — never exposed to the browser.

+ +
+ +
+
+
+
Import Historical Data
+

Pull existing data from Ambient's servers into your local records. Ambient keeps ~1 year of history.

+
+
+ + +
+
+ +
@@ -585,6 +618,64 @@ document.getElementById('saveKeysBtn').addEventListener('click', async () => { } }); +// ── Backfill ───────────────────────────────────────────────────────────────── +let backfillPoller = null; + +async function startBackfill() { + const days = document.getElementById('backfillDays').value; + const btn = document.getElementById('backfillBtn'); + const progress = document.getElementById('backfillProgress'); + btn.disabled = true; + btn.textContent = 'Running…'; + progress.style.display = 'block'; + document.getElementById('progressBar').style.width = '0%'; + document.getElementById('progressText').textContent = 'Starting…'; + + try { + const r = await fetch('/api/station/backfill', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ days: parseInt(days) }) + }); + if (r.status === 401) { location.href = '/dashboard/login'; return; } + const data = await r.json(); + if (!data.ok) { + document.getElementById('progressText').textContent = data.error || 'Failed to start.'; + btn.disabled = false; btn.textContent = 'Import'; + return; + } + backfillPoller = setInterval(pollBackfill, 2000); + } catch (e) { + document.getElementById('progressText').textContent = 'Error: ' + e.message; + btn.disabled = false; btn.textContent = 'Import'; + } +} + +async function pollBackfill() { + try { + const r = await fetch('/api/station/backfill/status'); + if (!r.ok) return; + const s = await r.json(); + const pct = s.total > 0 ? Math.min(100, Math.round((s.done / s.total) * 100)) : 0; + document.getElementById('progressBar').style.width = pct + '%'; + if (s.error) { + document.getElementById('progressText').textContent = 'Error: ' + s.error; + } else if (s.running) { + document.getElementById('progressText').textContent = `${s.done} of ${s.total} days processed · ${s.inserted.toLocaleString()} readings imported`; + } else { + document.getElementById('progressText').textContent = `Done — imported ${s.inserted.toLocaleString()} readings. Refresh charts to see the data.`; + document.getElementById('progressBar').style.width = '100%'; + clearInterval(backfillPoller); + backfillPoller = null; + document.getElementById('backfillBtn').disabled = false; + document.getElementById('backfillBtn').textContent = 'Import'; + loadHistory(currentRange); + } + } catch {} +} + +document.getElementById('backfillBtn').addEventListener('click', startBackfill); + updateLatest(); loadHistory(currentRange); checkKeyStatus(); diff --git a/app/server.js b/app/server.js index 1245eaa..e33681e 100644 --- a/app/server.js +++ b/app/server.js @@ -873,4 +873,107 @@ app.get('/api/station/latest', requireDashboardAuth, async (req, res) => { res.json(null); }); +// ── Ambient historical backfill ─────────────────────────────────────────────── +let backfillState = { running: false, done: 0, total: 0, inserted: 0, error: null }; + +async function runBackfill(daysBack) { + const apiKey = ambientServerConfig.apiKey; + const appKey = ambientServerConfig.appKey; + if (!apiKey || !appKey) { + backfillState = { running: false, done: 0, total: 0, inserted: 0, error: 'No API keys configured' }; + return; + } + + // Get device MAC address + let mac; + try { + const r = await fetchWithRetry( + `https://rt.ambientweather.net/v1/devices?apiKey=${encodeURIComponent(apiKey)}&applicationKey=${encodeURIComponent(appKey)}`, + { attempts: 3, timeoutMs: 15000 } + ); + const devices = await r.json(); + mac = devices?.[0]?.macAddress; + } catch (e) { + backfillState = { running: false, done: 0, total: 0, inserted: 0, error: `Device fetch failed: ${e.message}` }; + return; + } + + if (!mac) { + backfillState = { running: false, done: 0, total: 0, inserted: 0, error: 'No device found for these keys' }; + return; + } + + const cutoff = Date.now() - daysBack * 86400000; + backfillState = { running: true, done: 0, total: daysBack, inserted: 0, error: null }; + + // Per-day cache of existing dateutc values so we don't write duplicates + const existingByDay = new Map(); + function getExisting(day) { + if (existingByDay.has(day)) return existingByDay.get(day); + const set = new Set(); + try { + const lines = fs.readFileSync(path.join(STATION_DIR, `${day}.jsonl`), 'utf8').split('\n').filter(Boolean); + for (const l of lines) { try { set.add(JSON.parse(l).dateutc); } catch {} } + } catch {} + existingByDay.set(day, set); + return set; + } + + let endDate = Date.now(); + let inserted = 0; + + while (endDate > cutoff) { + await new Promise(r => setTimeout(r, 1100)); // respect 1 req/sec rate limit + try { + const url = `https://rt.ambientweather.net/v1/devices/${encodeURIComponent(mac)}?apiKey=${encodeURIComponent(apiKey)}&applicationKey=${encodeURIComponent(appKey)}&limit=288&endDate=${endDate}`; + const r = await fetchWithRetry(url, { attempts: 2, timeoutMs: 15000 }); + const records = await r.json(); + + if (!Array.isArray(records) || records.length === 0) break; + + let oldest = Infinity; + for (const rec of records) { + if (!rec.dateutc) continue; + if (rec.dateutc < oldest) oldest = rec.dateutc; + if (rec.dateutc < cutoff) continue; + const day = stationDateKey(rec.dateutc); + const existing = getExisting(day); + if (!existing.has(rec.dateutc)) { + fs.appendFileSync(path.join(STATION_DIR, `${day}.jsonl`), JSON.stringify(rec) + '\n'); + existing.add(rec.dateutc); + inserted++; + } + } + + backfillState.done = Math.min(daysBack, Math.ceil((Date.now() - oldest) / 86400000)); + backfillState.inserted = inserted; + + if (records.length < 288 || oldest <= cutoff) break; + endDate = oldest - 1; + } catch (e) { + backfillState.error = e.message; + break; + } + } + + backfillState.running = false; + backfillState.done = daysBack; + backfillState.inserted = inserted; + console.log(`[backfill] complete — inserted ${inserted} records`); +} + +app.post('/api/station/backfill', requireDashboardAuth, (req, res) => { + if (backfillState.running) return res.json({ ok: false, error: 'Backfill already running' }); + const days = Math.min(Math.max(parseInt(req.body?.days || '30'), 1), 365); + res.json({ ok: true, days }); + runBackfill(days).catch(e => { + backfillState.error = e.message; + backfillState.running = false; + }); +}); + +app.get('/api/station/backfill/status', requireDashboardAuth, (_req, res) => { + res.json(backfillState); +}); + app.listen(PORT, () => console.log(`Weather app running on port ${PORT}`));