Log official NWS observations for a reference ZIP and overlay them on the dashboard

- Resolves ZIP (default 24550) to the nearest NWS observation station; logs every observation
  (normalised imperial values + raw METAR), back-fills ~7 days, snapshots the hourly forecast
  every 3h, and logs alerts for that point
- Chart overlay (temp, humidity/dew point, wind, pressure) with an NWS toggle in the History bar
- 'Official NWS observation' card on the Now tab comparing against your station
- Settings: change the reference ZIP and see logging status
- Exact-unit conversions for NWS values (verified against known conversions)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

app/server.js

@@ -1253,15 +1253,19 @@ async function alertWatcherCron() {
   // ── Log NWS alerts for the dashboard location regardless of subscribers ───────
   const dashLat = dashAuth?.stationLat;
   const dashLon = dashAuth?.stationLon;
-  if (dashLat != null) {
-    const dashAlertData = await fetch(`${base}/api/alerts?lat=${dashLat}&lon=${dashLon}`)
+  const nwsRes = nwsSaved().resolved;
+  const alertSpots = [];
+  if (dashLat != null) alertSpots.push({ lat: dashLat, lon: dashLon });
+  if (nwsRes && !(nwsRes.lat === dashLat && nwsRes.lon === dashLon)) alertSpots.push({ lat: nwsRes.lat, lon: nwsRes.lon, zip: nwsRes.zip });
+  for (const spot of alertSpots) {
+    const dashAlertData = await fetch(`${base}/api/alerts?lat=${spot.lat}&lon=${spot.lon}`)
       .then(r => r.json()).catch(() => ({ features: [] }));
     for (const a of dashAlertData.features || []) {
       const alertId = a.id || a.properties?.id;
       if (alertId && !loggedAlertIds.has(alertId)) {
         loggedAlertIds.add(alertId);
         const p = a.properties || {};
-        recordAlertLog({ type: 'nws', alertId, ...nwsLogFields(p), lat: dashLat, lon: dashLon });
+        recordAlertLog({ type: 'nws', alertId, ...nwsLogFields(p), lat: spot.lat, lon: spot.lon, ...(spot.zip ? { zip: spot.zip } : {}) });
       }
     }
   }
@@ -1532,6 +1536,190 @@ function bucketReadings(readings, points) {
   });
 }
 
+// ── NWS reference station (official observations for a ZIP code) ──────────────
+// Resolves the ZIP to its nearest NWS observation station, logs every observation it publishes
+// (normalised to imperial units + the raw METAR), keeps hourly-forecast snapshots, and serves the
+// data for the dashboard's overlay. Base URLs are overridable so it can be tested offline.
+const NWS_BASE = process.env.NWS_BASE || 'https://api.weather.gov';
+const ZIP_API = process.env.ZIP_API || 'https://api.zippopotam.us/us';
+const NWS_DIR = path.join(DATA_DIR, 'nws');
+const NWS_FC_DIR = path.join(NWS_DIR, 'forecast');
+fs.mkdirSync(NWS_FC_DIR, { recursive: true });
+const NWS_HEADERS = { headers: { 'User-Agent': 'weather-app-v2 (nate@necloud.us)', Accept: 'application/geo+json' } };
+const NWS_ZIP_FALLBACK = { '24550': { lat: 37.22, lon: -79.34, place: 'Evington, VA' } }; // used only if the ZIP lookup is unreachable
+const nwsState = { running: false, lastError: null, lastPollAt: 0, lastForecastAt: 0, lastObsTs: 0, obsCount: 0, firstObsTs: null };
+
+const nwsSaved = () => dashAuth?.nws || {};
+function nwsSave(patch) {
+  dashAuth = { ...(dashAuth || {}), nws: { ...nwsSaved(), ...patch } };
+  try { fs.writeFileSync(DASH_AUTH_FILE, JSON.stringify(dashAuth)); } catch {}
+}
+const haversineKm = (a, b, c, d) => { const R = 6371, r = Math.PI / 180, dLat = (c - a) * r, dLon = (d - b) * r; const x = Math.sin(dLat / 2) ** 2 + Math.cos(a * r) * Math.cos(c * r) * Math.sin(dLon / 2) ** 2; return 2 * R * Math.asin(Math.sqrt(x)); };
+
+async function nwsResolve(force = false) {
+  const cfg = nwsSaved();
+  const zip = cfg.zip || '24550';
+  const res = cfg.resolved;
+  if (!force && res && res.zip === zip && Date.now() - res.resolvedAt < 30 * 86400000) return res;
+  let lat = cfg.lat, lon = cfg.lon, place = null, approx = false;
+  if (lat == null || lon == null) {
+    try {
+      const r = await fetchWithRetry(`${ZIP_API}/${zip}`, { attempts: 2, timeoutMs: 10000 });
+      const z = await r.json();
+      const pl = z.places?.[0];
+      lat = parseFloat(pl.latitude); lon = parseFloat(pl.longitude);
+      place = `${pl['place name']}, ${pl['state abbreviation']}`;
+    } catch (e) {
+      const fb = NWS_ZIP_FALLBACK[zip];
+      if (!fb) throw new Error(`Could not look up ZIP ${zip}`);
+      lat = fb.lat; lon = fb.lon; place = fb.place; approx = true;
+    }
+  }
+  const pr = await fetchWithRetry(`${NWS_BASE}/points/${lat.toFixed(4)},${lon.toFixed(4)}`, { attempts: 2, timeoutMs: 12000, fetchOptions: NWS_HEADERS });
+  const pt = (await pr.json()).properties;
+  const sr = await fetchWithRetry(pt.observationStations, { attempts: 2, timeoutMs: 12000, fetchOptions: NWS_HEADERS });
+  const st = (await sr.json()).features?.[0];
+  if (!st) throw new Error('No observation station near this ZIP');
+  const [sLon, sLat] = st.geometry?.coordinates || [null, null];
+  const rl = pt.relativeLocation?.properties;
+  const resolved = {
+    zip, lat, lon, approx, place: place || (rl ? `${rl.city}, ${rl.state}` : zip),
+    stationId: st.properties.stationIdentifier, stationName: st.properties.name, stationLat: sLat, stationLon: sLon,
+    distanceMi: sLat != null ? +(haversineKm(lat, lon, sLat, sLon) * 0.621371).toFixed(1) : null,
+    forecastHourly: pt.forecastHourly, gridId: pt.gridId, timeZone: pt.timeZone, resolvedAt: Date.now(),
+  };
+  nwsSave({ zip, resolved });
+  console.log(`[nws] ZIP ${zip} → ${resolved.stationId} (${resolved.stationName}), ${resolved.distanceMi} mi`);
+  return resolved;
+}
+
+const nwsQ = (q, kind) => {
+  if (!q || q.value == null || !Number.isFinite(q.value)) return null;
+  const unit = String(q.unitCode || '').split(':').pop(), v = q.value; // e.g. "wmoUnit:km_h-1" -> "km_h-1"
+  switch (kind) {
+    case 'temp': return unit === 'degC' ? v * 9 / 5 + 32 : unit === 'K' ? (v - 273.15) * 9 / 5 + 32 : v;
+    case 'speed': return unit === 'km_h-1' ? v * 0.621371 : unit === 'm_s-1' ? v * 2.23694 : unit === 'kt' ? v * 1.15078 : v;
+    case 'press': return unit === 'Pa' ? v * 0.00029530 : unit === 'hPa' ? v * 0.02953 : v;
+    case 'dist': return unit === 'm' ? v * 0.000621371 : unit === 'km' ? v * 0.621371 : v;
+    case 'len': return unit === 'mm' ? v / 25.4 : unit === 'm' ? v * 39.3701 : v;
+    default: return v;
+  }
+};
+const r1 = (v, d = 1) => (v == null ? null : +v.toFixed(d));
+
+function normalizeNwsObs(f) {
+  const p = f.properties || {};
+  const ts = Date.parse(p.timestamp);
+  if (!Number.isFinite(ts)) return null;
+  return {
+    dateutc: ts, ts, station: (p.station || '').split('/').pop() || null,
+    tempf: r1(nwsQ(p.temperature, 'temp')), dewPoint: r1(nwsQ(p.dewpoint, 'temp')), humidity: r1(nwsQ(p.relativeHumidity)),
+    windspeedmph: r1(nwsQ(p.windSpeed, 'speed')), windgustmph: r1(nwsQ(p.windGust, 'speed')), winddir: r1(nwsQ(p.windDirection), 0),
+    baromrelin: r1(nwsQ(p.seaLevelPressure, 'press'), 3), baromabsin: r1(nwsQ(p.barometricPressure, 'press'), 3),
+    visibilityMi: r1(nwsQ(p.visibility, 'dist'), 1),
+    rain1hIn: r1(nwsQ(p.precipitationLastHour, 'len'), 2), rain3hIn: r1(nwsQ(p.precipitationLast3Hours, 'len'), 2), rain6hIn: r1(nwsQ(p.precipitationLast6Hours, 'len'), 2),
+    heatIndexF: r1(nwsQ(p.heatIndex, 'temp')), windChillF: r1(nwsQ(p.windChill, 'temp')),
+    max24F: r1(nwsQ(p.maxTemperatureLast24Hours, 'temp')), min24F: r1(nwsQ(p.minTemperatureLast24Hours, 'temp')),
+    text: p.textDescription || null,
+    clouds: (p.cloudLayers || []).map(c => ({ amount: c.amount, baseFt: c.base?.value != null ? Math.round(c.base.value * 3.28084) : null })),
+    presentWeather: (p.presentWeather || []).map(w => w.weather || w.rawString).filter(Boolean),
+    raw: p.rawMessage || null,
+  };
+}
+
+const nwsSeen = new Map(); // day -> Set(ts)
+function nwsDaySet(day) {
+  let set = nwsSeen.get(day);
+  if (set) return set;
+  set = new Set();
+  try { for (const l of fs.readFileSync(path.join(NWS_DIR, `${day}.jsonl`), 'utf8').split('\n')) if (l) { try { set.add(JSON.parse(l).ts); } catch {} } } catch {}
+  nwsSeen.set(day, set);
+  return set;
+}
+function nwsSaveObs(rec) {
+  const day = new Date(rec.ts).toISOString().slice(0, 10);
+  const set = nwsDaySet(day);
+  if (set.has(rec.ts)) return false;
+  fs.appendFileSync(path.join(NWS_DIR, `${day}.jsonl`), JSON.stringify(rec) + '\n');
+  set.add(rec.ts);
+  nwsState.obsCount++;
+  if (rec.ts > nwsState.lastObsTs) nwsState.lastObsTs = rec.ts;
+  if (!nwsState.firstObsTs || rec.ts < nwsState.firstObsTs) nwsState.firstObsTs = rec.ts;
+  return true;
+}
+
+async function nwsFetchObservations(res, limit) {
+  const r = await fetchWithRetry(`${NWS_BASE}/stations/${res.stationId}/observations?limit=${limit}`, { attempts: 2, timeoutMs: 15000, fetchOptions: NWS_HEADERS });
+  const feats = (await r.json()).features || [];
+  let added = 0;
+  for (const f of feats.reverse()) { const rec = normalizeNwsObs(f); if (rec && nwsSaveObs(rec)) added++; }
+  return { seen: feats.length, added };
+}
+
+async function nwsSnapshotForecast(res) {
+  if (!res.forecastHourly) return;
+  const r = await fetchWithRetry(res.forecastHourly, { attempts: 2, timeoutMs: 15000, fetchOptions: NWS_HEADERS });
+  const p = (await r.json()).properties || {};
+  const issued = Date.parse(p.updateTime || p.generatedAt) || Date.now();
+  const day = new Date(issued).toISOString().slice(0, 10);
+  const file = path.join(NWS_FC_DIR, `${day}.jsonl`);
+  try { if (fs.readFileSync(file, 'utf8').split('\n').some(l => l && JSON.parse(l).issued === issued)) return; } catch {}
+  const periods = (p.periods || []).slice(0, 48).map(x => ({
+    start: Date.parse(x.startTime), tempf: x.temperatureUnit === 'C' ? x.temperature * 9 / 5 + 32 : x.temperature,
+    pop: x.probabilityOfPrecipitation?.value ?? null, wind: x.windSpeed, windDir: x.windDirection,
+    humidity: x.relativeHumidity?.value ?? null, dewC: x.dewpoint?.value ?? null, short: x.shortForecast,
+  }));
+  fs.appendFileSync(file, JSON.stringify({ ts: Date.now(), issued, station: res.stationId, zip: res.zip, periods }) + '\n');
+}
+
+async function nwsPoll() {
+  if (nwsState.running) return;
+  nwsState.running = true;
+  try {
+    const first = !nwsSaved().backfilled;
+    const res = await nwsResolve();
+    const out = await nwsFetchObservations(res, first ? 500 : 15); // first run: everything NWS still keeps (~7 days)
+    if (first && out.seen) nwsSave({ backfilled: Date.now() });
+    if (Date.now() - nwsState.lastForecastAt > 3 * 3600000) { await nwsSnapshotForecast(res); nwsState.lastForecastAt = Date.now(); }
+    nwsState.lastError = null;
+    nwsState.lastPollAt = Date.now();
+    if (out.added) console.log(`[nws] logged ${out.added} observation(s) from ${res.stationId}`);
+  } catch (e) {
+    nwsState.lastError = e.message;
+    console.error('[nws] poll error:', e.message);
+  }
+  nwsState.running = false;
+}
+
+function loadNwsReadings(rangeMs) {
+  const now = Date.now(), cutoff = now - rangeMs;
+  const days = Math.min(Math.ceil(rangeMs / 86400000) + 1, 4000);
+  const out = [];
+  for (let i = 0; i < days; i++) {
+    const day = stationDateKey(now - i * 86400000);
+    try {
+      for (const l of fs.readFileSync(path.join(NWS_DIR, `${day}.jsonl`), 'utf8').split('\n')) {
+        if (!l) continue;
+        try { const o = JSON.parse(l); if (o.ts >= cutoff) { delete o.raw; out.push(o); } } catch {}
+      }
+    } catch {}
+  }
+  return out.sort((a, b) => a.ts - b.ts);
+}
+// Index what's already on disk so the status shows the right totals after a restart
+(function nwsIndex() {
+  try {
+    for (const f of fs.readdirSync(NWS_DIR).filter(f => /^\d{4}-\d{2}-\d{2}\.jsonl$/.test(f)).sort()) {
+      for (const l of fs.readFileSync(path.join(NWS_DIR, f), 'utf8').split('\n')) {
+        if (!l) continue;
+        try { const t = JSON.parse(l).ts; nwsState.obsCount++; if (t > nwsState.lastObsTs) nwsState.lastObsTs = t; if (!nwsState.firstObsTs || t < nwsState.firstObsTs) nwsState.firstObsTs = t; } catch {}
+      }
+    }
+  } catch {}
+})();
+setTimeout(nwsPoll, 20 * 1000).unref();
+setInterval(nwsPoll, 10 * 60 * 1000).unref();
+
 // Station history API
 app.get('/api/station/history', requireDashboardAuth, (req, res) => {
   const rangeMap = { '1h': 3600000, '6h': 21600000, '24h': 86400000, '2d': 172800000, '7d': 604800000, '30d': 2592000000, '90d': 7776000000, '1y': 31536000000, '2y': 63072000000, '3y': 94608000000 };
@@ -1567,6 +1755,36 @@ function firstStationDay() {
   } catch { return null; }
 }
 
+// ── NWS reference data API ───────────────────────────────────────────────────
+function nwsStatus() {
+  const c = nwsSaved();
+  return { zip: c.zip || '24550', resolved: c.resolved || null, observations: nwsState.obsCount, firstObservation: nwsState.firstObsTs, lastObservation: nwsState.lastObsTs || null, lastPoll: nwsState.lastPollAt || null, lastError: nwsState.lastError };
+}
+app.get('/api/nws/config', requireDashboardAuth, (_req, res) => res.json(nwsStatus()));
+app.post('/api/nws/config', requireDashboardAuth, requireAdmin, async (req, res) => {
+  const zip = String(req.body?.zip || '').trim();
+  if (!/^\d{5}$/.test(zip)) return res.status(400).json({ error: 'Enter a 5-digit US ZIP code' });
+  nwsSave({ zip, resolved: null, lat: undefined, lon: undefined, backfilled: undefined });
+  nwsState.lastObsTs = 0; nwsState.lastForecastAt = 0;
+  try { await nwsResolve(true); } catch (e) { return res.status(502).json({ error: `Could not set up that ZIP: ${e.message}` }); }
+  nwsPoll();
+  res.json(nwsStatus());
+});
+app.get('/api/nws/latest', requireDashboardAuth, (_req, res) => {
+  const recent = loadNwsReadings(6 * 3600000);
+  const latest = recent[recent.length - 1] || null;
+  let raw = null;
+  if (latest) { try { const day = new Date(latest.ts).toISOString().slice(0, 10); for (const l of fs.readFileSync(path.join(NWS_DIR, `${day}.jsonl`), 'utf8').split('\n')) { if (l && JSON.parse(l).ts === latest.ts) raw = JSON.parse(l).raw; } } catch {} }
+  res.json({ ...nwsStatus(), latest: latest ? { ...latest, raw } : null });
+});
+app.get('/api/nws/history', requireDashboardAuth, (req, res) => {
+  const range = String(req.query.range || '24h');
+  const map = { '1h': 3600000, '6h': 21600000, '24h': 86400000, '2d': 172800000, '7d': 604800000, '30d': 2592000000, '90d': 7776000000, '1y': 31536000000, '2y': 63072000000, '3y': 94608000000 };
+  let ms = map[range];
+  if (range === 'all') ms = 4000 * 86400000;
+  res.json(loadNwsReadings(ms || map['24h']));
+});
+
 // Latest station reading
 app.get('/api/station/latest', requireDashboardAuth, async (req, res) => {
   if (!ambientServerConfig.apiKey || !ambientServerConfig.appKey) {