Add Ambient historical data backfill to dashboard

- POST /api/station/backfill: pages backward through Ambient's historical API
  at 1 req/sec, deduplicates against existing JSONL records, writes to daily files
- GET /api/station/backfill/status: polls live progress (days done, readings inserted)
- Dashboard settings panel: day-range selector (7d/30d/90d/180d/1y), Import button,
  progress bar with live count; auto-reloads charts when complete

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

app/public/dashboard.html

@@ -90,6 +90,14 @@ a { color: var(--cool); text-decoration: none; }
 .keys-status.ok { color: var(--cool); }
 .keys-status.missing { color: var(--accent); }
 .keys-hint { font-size: 0.78rem; color: var(--ink-light); margin-top: 1rem; line-height: 1.5; }
+.settings-divider { border: none; border-top: 1px solid var(--border); margin: 1.25rem 0; }
+.backfill-header { display: flex; align-items: flex-start; justify-content: space-between; gap: 1rem; flex-wrap: wrap; }
+.backfill-controls { display: flex; gap: 0.5rem; align-items: center; flex-shrink: 0; }
+.backfill-controls select { padding: 0.45rem 0.6rem; background: var(--paper); border: 1px solid var(--border); border-radius: 8px; color: var(--ink); font-family: var(--mono); font-size: 0.85rem; }
+.progress-bar-wrap { height: 6px; background: var(--paper-darker); border-radius: 3px; margin-top: 1rem; overflow: hidden; }
+.progress-bar { height: 100%; background: var(--cool); border-radius: 3px; transition: width 0.5s; width: 0%; }
+.progress-text { font-size: 0.82rem; color: var(--ink-light); margin-top: 0.4rem; }
+.backfill-progress { margin-top: 0.75rem; }
 </style>
 </head>
 <body>
@@ -163,6 +171,31 @@ a { color: var(--cool); text-decoration: none; }
         <span id="saveMsg" class="save-msg"></span>
       </div>
       <p class="keys-hint">Find these at <strong>ambientweather.net → My Account → API Keys</strong>. Keys are stored on the server only — never exposed to the browser.</p>
+
+      <hr class="settings-divider">
+
+      <div class="backfill-section">
+        <div class="backfill-header">
+          <div>
+            <div class="chart-title" style="margin-bottom:0.25rem">Import Historical Data</div>
+            <p class="keys-hint" style="margin:0">Pull existing data from Ambient's servers into your local records. Ambient keeps ~1 year of history.</p>
+          </div>
+          <div class="backfill-controls">
+            <select id="backfillDays">
+              <option value="7">7 days</option>
+              <option value="30" selected>30 days</option>
+              <option value="90">90 days</option>
+              <option value="180">180 days</option>
+              <option value="365">1 year</option>
+            </select>
+            <button id="backfillBtn">Import</button>
+          </div>
+        </div>
+        <div id="backfillProgress" class="backfill-progress" style="display:none">
+          <div class="progress-bar-wrap"><div class="progress-bar" id="progressBar"></div></div>
+          <div class="progress-text" id="progressText"></div>
+        </div>
+      </div>
     </div>
   </details>
 </div>
@@ -585,6 +618,64 @@ document.getElementById('saveKeysBtn').addEventListener('click', async () => {
   }
 });
 
+// ── Backfill ─────────────────────────────────────────────────────────────────
+let backfillPoller = null;
+
+async function startBackfill() {
+  const days = document.getElementById('backfillDays').value;
+  const btn = document.getElementById('backfillBtn');
+  const progress = document.getElementById('backfillProgress');
+  btn.disabled = true;
+  btn.textContent = 'Running…';
+  progress.style.display = 'block';
+  document.getElementById('progressBar').style.width = '0%';
+  document.getElementById('progressText').textContent = 'Starting…';
+
+  try {
+    const r = await fetch('/api/station/backfill', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ days: parseInt(days) })
+    });
+    if (r.status === 401) { location.href = '/dashboard/login'; return; }
+    const data = await r.json();
+    if (!data.ok) {
+      document.getElementById('progressText').textContent = data.error || 'Failed to start.';
+      btn.disabled = false; btn.textContent = 'Import';
+      return;
+    }
+    backfillPoller = setInterval(pollBackfill, 2000);
+  } catch (e) {
+    document.getElementById('progressText').textContent = 'Error: ' + e.message;
+    btn.disabled = false; btn.textContent = 'Import';
+  }
+}
+
+async function pollBackfill() {
+  try {
+    const r = await fetch('/api/station/backfill/status');
+    if (!r.ok) return;
+    const s = await r.json();
+    const pct = s.total > 0 ? Math.min(100, Math.round((s.done / s.total) * 100)) : 0;
+    document.getElementById('progressBar').style.width = pct + '%';
+    if (s.error) {
+      document.getElementById('progressText').textContent = 'Error: ' + s.error;
+    } else if (s.running) {
+      document.getElementById('progressText').textContent = `${s.done} of ${s.total} days processed · ${s.inserted.toLocaleString()} readings imported`;
+    } else {
+      document.getElementById('progressText').textContent = `Done — imported ${s.inserted.toLocaleString()} readings. Refresh charts to see the data.`;
+      document.getElementById('progressBar').style.width = '100%';
+      clearInterval(backfillPoller);
+      backfillPoller = null;
+      document.getElementById('backfillBtn').disabled = false;
+      document.getElementById('backfillBtn').textContent = 'Import';
+      loadHistory(currentRange);
+    }
+  } catch {}
+}
+
+document.getElementById('backfillBtn').addEventListener('click', startBackfill);
+
 updateLatest();
 loadHistory(currentRange);
 checkKeyStatus();

app/server.js

@@ -873,4 +873,107 @@ app.get('/api/station/latest', requireDashboardAuth, async (req, res) => {
   res.json(null);
 });
 
+// ── Ambient historical backfill ───────────────────────────────────────────────
+let backfillState = { running: false, done: 0, total: 0, inserted: 0, error: null };
+
+async function runBackfill(daysBack) {
+  const apiKey = ambientServerConfig.apiKey;
+  const appKey = ambientServerConfig.appKey;
+  if (!apiKey || !appKey) {
+    backfillState = { running: false, done: 0, total: 0, inserted: 0, error: 'No API keys configured' };
+    return;
+  }
+
+  // Get device MAC address
+  let mac;
+  try {
+    const r = await fetchWithRetry(
+      `https://rt.ambientweather.net/v1/devices?apiKey=${encodeURIComponent(apiKey)}&applicationKey=${encodeURIComponent(appKey)}`,
+      { attempts: 3, timeoutMs: 15000 }
+    );
+    const devices = await r.json();
+    mac = devices?.[0]?.macAddress;
+  } catch (e) {
+    backfillState = { running: false, done: 0, total: 0, inserted: 0, error: `Device fetch failed: ${e.message}` };
+    return;
+  }
+
+  if (!mac) {
+    backfillState = { running: false, done: 0, total: 0, inserted: 0, error: 'No device found for these keys' };
+    return;
+  }
+
+  const cutoff = Date.now() - daysBack * 86400000;
+  backfillState = { running: true, done: 0, total: daysBack, inserted: 0, error: null };
+
+  // Per-day cache of existing dateutc values so we don't write duplicates
+  const existingByDay = new Map();
+  function getExisting(day) {
+    if (existingByDay.has(day)) return existingByDay.get(day);
+    const set = new Set();
+    try {
+      const lines = fs.readFileSync(path.join(STATION_DIR, `${day}.jsonl`), 'utf8').split('\n').filter(Boolean);
+      for (const l of lines) { try { set.add(JSON.parse(l).dateutc); } catch {} }
+    } catch {}
+    existingByDay.set(day, set);
+    return set;
+  }
+
+  let endDate = Date.now();
+  let inserted = 0;
+
+  while (endDate > cutoff) {
+    await new Promise(r => setTimeout(r, 1100)); // respect 1 req/sec rate limit
+    try {
+      const url = `https://rt.ambientweather.net/v1/devices/${encodeURIComponent(mac)}?apiKey=${encodeURIComponent(apiKey)}&applicationKey=${encodeURIComponent(appKey)}&limit=288&endDate=${endDate}`;
+      const r = await fetchWithRetry(url, { attempts: 2, timeoutMs: 15000 });
+      const records = await r.json();
+
+      if (!Array.isArray(records) || records.length === 0) break;
+
+      let oldest = Infinity;
+      for (const rec of records) {
+        if (!rec.dateutc) continue;
+        if (rec.dateutc < oldest) oldest = rec.dateutc;
+        if (rec.dateutc < cutoff) continue;
+        const day = stationDateKey(rec.dateutc);
+        const existing = getExisting(day);
+        if (!existing.has(rec.dateutc)) {
+          fs.appendFileSync(path.join(STATION_DIR, `${day}.jsonl`), JSON.stringify(rec) + '\n');
+          existing.add(rec.dateutc);
+          inserted++;
+        }
+      }
+
+      backfillState.done = Math.min(daysBack, Math.ceil((Date.now() - oldest) / 86400000));
+      backfillState.inserted = inserted;
+
+      if (records.length < 288 || oldest <= cutoff) break;
+      endDate = oldest - 1;
+    } catch (e) {
+      backfillState.error = e.message;
+      break;
+    }
+  }
+
+  backfillState.running = false;
+  backfillState.done = daysBack;
+  backfillState.inserted = inserted;
+  console.log(`[backfill] complete — inserted ${inserted} records`);
+}
+
+app.post('/api/station/backfill', requireDashboardAuth, (req, res) => {
+  if (backfillState.running) return res.json({ ok: false, error: 'Backfill already running' });
+  const days = Math.min(Math.max(parseInt(req.body?.days || '30'), 1), 365);
+  res.json({ ok: true, days });
+  runBackfill(days).catch(e => {
+    backfillState.error = e.message;
+    backfillState.running = false;
+  });
+});
+
+app.get('/api/station/backfill/status', requireDashboardAuth, (_req, res) => {
+  res.json(backfillState);
+});
+
 app.listen(PORT, () => console.log(`Weather app running on port ${PORT}`));