Fix analytics data-loss bugs and clean up the admin dashboard; v2.2.4

- Self-heal consent desync: if the server's consent cookie and the
  client's localStorage flag fall out of sync, the client now notices
  and stops silently sending events into a void.
- Stop double-counting hard page loads in hit/visitor stats.
- Flush pending disk writes on SIGTERM/SIGINT so redeploys don't drop
  the last few events.
- Log rate-limiter drops once per window (diagnoses proxy misconfig).
- Widen the bot-detection allowlist to avoid flagging real browsers
  (Chrome-iOS, Samsung Browser, etc.) as bots.
- Stop merging distinct visitors who share an IP when a real session
  cookie is already present — only fall back to IP correlation when no
  cookie was sent at all.
- Include analyticsEvents/episodePlays in backup snapshots and restore
  them, so a restore no longer wipes engagement analytics.
- Regroup admin dashboard KPI cards by time scope (range vs all-time)
  and remove a duplicated Top Countries/States block.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

server/data.js

@@ -1064,6 +1064,8 @@ export async function createBackupSnapshot(reason = 'scheduled') {
       publishState: state.publishState,
       hitStats: state.hitStats,
       visitorStats: state.visitorStats,
+      analyticsEvents: state.analyticsEvents,
+      episodePlays: state.episodePlays,
       contactSubmissions: state.contactSubmissions,
       studyCommunityPosts: state.studyCommunityPosts,
       replyTemplates: state.replyTemplates,
@@ -1179,6 +1181,8 @@ export async function restoreFromBackup(filename) {
 
   state.hitStats = sanitizeLoadedHitStats(parsed?.hitStats)
   state.visitorStats = sanitizeLoadedVisitorStats(parsed?.visitorStats)
+  state.analyticsEvents = sanitizeLoadedAnalyticsEvents(parsed?.analyticsEvents)
+  state.episodePlays = sanitizeLoadedEpisodePlays(parsed?.episodePlays)
   state.contactSubmissions = sanitizeLoadedContactSubmissions(parsed?.contactSubmissions)
   state.replyTemplates = sanitizeReplyTemplates(parsed?.replyTemplates)
   state.replyHistory = sanitizeReplyHistory(parsed?.replyHistory)
@@ -1186,6 +1190,8 @@ export async function restoreFromBackup(filename) {
 
   queueHitStatsWrite()
   queueVisitorStatsWrite()
+  queueAnalyticsEventsWrite()
+  queueEpisodePlaysWrite()
   queueContactSubmissionsWrite()
   queueReplyTemplatesWrite()
   queueReplyHistoryWrite()
@@ -1194,6 +1200,8 @@ export async function restoreFromBackup(filename) {
   await Promise.all([
     state.hitStatsWritePromise,
     state.visitorStatsWritePromise,
+    state.analyticsEventsWritePromise,
+    state.episodePlaysWritePromise,
     state.contactSubmissionsWritePromise,
     state.replyTemplatesWritePromise,
     state.replyHistoryWritePromise,
@@ -1298,6 +1306,23 @@ function sanitizeLoadedHitStats(value) {
   }
 }
 
+function sanitizeLoadedAnalyticsEvents(value) {
+  return {
+    scrollDepth: value?.scrollDepth && typeof value.scrollDepth === 'object' ? value.scrollDepth : {},
+    timeOnPage: value?.timeOnPage && typeof value.timeOnPage === 'object' ? value.timeOnPage : {},
+    outboundClicks: value?.outboundClicks && typeof value.outboundClicks === 'object' ? value.outboundClicks : {},
+    linkClicks: value?.linkClicks && typeof value.linkClicks === 'object' ? value.linkClicks : {},
+    utmSources: value?.utmSources && typeof value.utmSources === 'object' ? value.utmSources : {},
+    searchQueries: value?.searchQueries && typeof value.searchQueries === 'object' ? value.searchQueries : {},
+    notFound: value?.notFound && typeof value.notFound === 'object' ? value.notFound : {},
+    audioEvents: value?.audioEvents && typeof value.audioEvents === 'object' ? value.audioEvents : {},
+  }
+}
+
+function sanitizeLoadedEpisodePlays(value) {
+  return (value && typeof value === 'object' && !Array.isArray(value)) ? value : {}
+}
+
 function sanitizeLoadedVisitorStats(value) {
   const loadedVisitors = value?.visitors && typeof value.visitors === 'object' ? value.visitors : {}
 

server/routes/analytics.js

@@ -75,28 +75,39 @@ async function resolveGeo(ip, ipHash) {
   return fallback
 }
 
-// Simple in-memory rate limiter for pageview endpoint
+// Simple in-memory rate limiter for pageview endpoint.
+// This keys on req.ip, which collapses to one value for every visitor behind
+// a shared IP (a proxy in front of the app, a school/church network, CGNAT).
+// If TRUST_PROXY_HOPS is misconfigured for the real deployment topology,
+// req.ip silently becomes the proxy's IP for everyone and this limiter starts
+// dropping real traffic across all of them together. Log the first drop per
+// IP per window so that scenario shows up instead of vanishing silently.
 const pageviewHits = new Map()
 function pageviewAllowed(ip) {
   const now = Date.now()
   const windowMs = 60 * 1000
-  const entry = pageviewHits.get(ip) ?? { count: 0, start: now }
-  if (now - entry.start > windowMs) { entry.count = 0; entry.start = now }
+  const entry = pageviewHits.get(ip) ?? { count: 0, start: now, warned: false }
+  if (now - entry.start > windowMs) { entry.count = 0; entry.start = now; entry.warned = false }
   entry.count += 1
   pageviewHits.set(ip, entry)
   if (pageviewHits.size > 5000) {
     const cutoff = now - windowMs
     for (const [k, v] of pageviewHits) { if (v.start < cutoff) pageviewHits.delete(k) }
   }
-  return entry.count <= 60
+  const allowed = entry.count <= 60
+  if (!allowed && !entry.warned) {
+    entry.warned = true
+    console.warn(`[analytics] pageview rate limit hit for an IP (${entry.count} requests/min) — if this fires often, check TRUST_PROXY_HOPS against the real deployment topology, since a misconfigured hop count collapses many visitors onto one IP here`)
+  }
+  return allowed
 }
 
 export function recordVisitor(req, res, overridePath = null, overrideReferrer = null) {
   const cookies = parseCookies(req.headers.cookie)
   let visitorId = cookies[VISITOR_COOKIE]
+  const hadCookie = Boolean(visitorId)
   if (!visitorId) {
     visitorId = randomUUID()
-    res.append('Set-Cookie', `${VISITOR_COOKIE}=${encodeURIComponent(visitorId)}; Max-Age=31536000; Path=/; SameSite=Lax; HttpOnly`)
   }
 
   const nowIso = new Date().toISOString()
@@ -112,9 +123,18 @@ export function recordVisitor(req, res, overridePath = null, overrideReferrer =
 
   const ipHash = createHash('sha256').update(ip).digest('hex')
 
-  const existingIdByIp = state.visitorStats.ipHashIndex[ipHash]
-  if (existingIdByIp && existingIdByIp !== visitorId) {
-    visitorId = existingIdByIp
+  // Only fall back to IP-based correlation when the browser sent no cookie
+  // at all — e.g. it never persisted (Safari ITP, a cookie-blocking
+  // extension) — so that one real person doesn't get double-counted every
+  // visit. If a cookie WAS sent, trust it: two different people sharing a
+  // network (family wifi, a school or church LAN) must not be silently
+  // merged into one visitor just because they share an IP, which would
+  // corrupt page history and undercount unique visitors.
+  if (!hadCookie) {
+    const existingIdByIp = state.visitorStats.ipHashIndex[ipHash]
+    if (existingIdByIp) {
+      visitorId = existingIdByIp
+    }
     res.append('Set-Cookie', `${VISITOR_COOKIE}=${encodeURIComponent(visitorId)}; Max-Age=31536000; Path=/; SameSite=Lax; HttpOnly`)
   }
 

server/study-helpers.js

@@ -413,7 +413,13 @@ export function detectBot(userAgent) {
     return { isBot: true, reason: 'http-client' }
   }
   if (/bot|crawler|spider|scraper|indexer|reader|fetcher|loader|agent|spyware|tracking|monitor/.test(ua)) {
-    if (!/chrome|firefox|safari|opera|edge|msie|trident|like gecko/.test(ua)) {
+    // Generic keywords like "reader" or "agent" can appear as substrings of
+    // otherwise-normal browser UAs (e.g. some webviews, regional browsers).
+    // Any UA carrying a real rendering-engine or vendor token is treated as
+    // a genuine browser rather than a bot, even if it also matches a keyword
+    // above — this list should stay a superset of the previous one so it
+    // only ever reduces false positives, never re-admits an actual bot.
+    if (!/chrome|firefox|safari|opera|edge|msie|trident|gecko|webkit|khtml|presto|crios|fxios|ucbrowser|samsungbrowser/.test(ua)) {
       return { isBot: true, reason: 'bot-keyword' }
     }
   }