Fix analytics data-loss bugs and clean up the admin dashboard; v2.2.4

- Self-heal consent desync: if the server's consent cookie and the
  client's localStorage flag fall out of sync, the client now notices
  and stops silently sending events into a void.
- Stop double-counting hard page loads in hit/visitor stats.
- Flush pending disk writes on SIGTERM/SIGINT so redeploys don't drop
  the last few events.
- Log rate-limiter drops once per window (diagnoses proxy misconfig).
- Widen the bot-detection allowlist to avoid flagging real browsers
  (Chrome-iOS, Samsung Browser, etc.) as bots.
- Stop merging distinct visitors who share an IP when a real session
  cookie is already present — only fall back to IP correlation when no
  cookie was sent at all.
- Include analyticsEvents/episodePlays in backup snapshots and restore
  them, so a restore no longer wipes engagement analytics.
- Regroup admin dashboard KPI cards by time scope (range vs all-time)
  and remove a duplicated Top Countries/States block.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

package.json

@@ -1,7 +1,7 @@
 {
   "name": "siteforge",
   "private": true,
-  "version": "2.2.1",
+  "version": "2.2.4",
   "type": "module",
   "scripts": {
     "dev": "vite",

server.js

@@ -232,8 +232,67 @@ Promise.all([
       console.error('[series-scheduler] error:', err)
     })
 
-    app.listen(PORT, () => {
+    const server = app.listen(PORT, () => {
       logResendEmailAlignmentWarnings()
       console.log(`Siteforge listening on http://localhost:${PORT}`)
     })
+
+    // On every deploy the running container gets SIGTERM'd and replaced.
+    // All the queueXWrite() helpers in data.js debounce disk writes onto
+    // per-resource promise chains — without draining those here, a redeploy
+    // can kill the process mid-write and silently lose the last few
+    // analytics events (or any other pending write) of that container's life.
+    function collectPendingWrites() {
+      return [
+        state.hitStatsWritePromise,
+        state.visitorStatsWritePromise,
+        state.contactSubmissionsWritePromise,
+        state.questionsWritePromise,
+        state.draftQuestionsWritePromise,
+        state.replyTemplatesWritePromise,
+        state.replyHistoryWritePromise,
+        state.podcastChecklistWritePromise,
+        state.studyUsersWritePromise,
+        state.studyCommunityWritePromise,
+        state.studyRemindersWritePromise,
+        state.studyCommentsWritePromise,
+        state.studyCertificatesWritePromise,
+        state.episodeScriptsWritePromise,
+        state.downloadCountsWritePromise,
+        state.episodePlaysWritePromise,
+        state.analyticsEventsWritePromise,
+        state.qrCodesWritePromise,
+        state.emailSettingsWritePromise,
+        state.calendarEventsWritePromise,
+        state.auditLogWritePromise,
+        state.webhooksWritePromise,
+        state.studySessionsWritePromise,
+        ...state.studyNotesWriteQueues.values(),
+        ...state.studyProgressWriteQueues.values(),
+      ]
+    }
+
+    let shuttingDown = false
+    async function gracefulShutdown(signal) {
+      if (shuttingDown) return
+      shuttingDown = true
+      console.log(`[shutdown] ${signal} received — closing server and flushing pending writes...`)
+      server.close()
+      const forceExit = setTimeout(() => {
+        console.error('[shutdown] timed out waiting for pending writes; exiting anyway')
+        process.exit(1)
+      }, 10_000)
+      try {
+        await Promise.all(collectPendingWrites())
+        console.log('[shutdown] all pending writes flushed')
+      } catch (err) {
+        console.error('[shutdown] error while flushing writes:', err)
+      } finally {
+        clearTimeout(forceExit)
+        process.exit(0)
+      }
+    }
+
+    process.on('SIGTERM', () => gracefulShutdown('SIGTERM'))
+    process.on('SIGINT', () => gracefulShutdown('SIGINT'))
   })

server/data.js

@@ -1064,6 +1064,8 @@ export async function createBackupSnapshot(reason = 'scheduled') {
       publishState: state.publishState,
       hitStats: state.hitStats,
       visitorStats: state.visitorStats,
+      analyticsEvents: state.analyticsEvents,
+      episodePlays: state.episodePlays,
       contactSubmissions: state.contactSubmissions,
       studyCommunityPosts: state.studyCommunityPosts,
       replyTemplates: state.replyTemplates,
@@ -1179,6 +1181,8 @@ export async function restoreFromBackup(filename) {
 
   state.hitStats = sanitizeLoadedHitStats(parsed?.hitStats)
   state.visitorStats = sanitizeLoadedVisitorStats(parsed?.visitorStats)
+  state.analyticsEvents = sanitizeLoadedAnalyticsEvents(parsed?.analyticsEvents)
+  state.episodePlays = sanitizeLoadedEpisodePlays(parsed?.episodePlays)
   state.contactSubmissions = sanitizeLoadedContactSubmissions(parsed?.contactSubmissions)
   state.replyTemplates = sanitizeReplyTemplates(parsed?.replyTemplates)
   state.replyHistory = sanitizeReplyHistory(parsed?.replyHistory)
@@ -1186,6 +1190,8 @@ export async function restoreFromBackup(filename) {
 
   queueHitStatsWrite()
   queueVisitorStatsWrite()
+  queueAnalyticsEventsWrite()
+  queueEpisodePlaysWrite()
   queueContactSubmissionsWrite()
   queueReplyTemplatesWrite()
   queueReplyHistoryWrite()
@@ -1194,6 +1200,8 @@ export async function restoreFromBackup(filename) {
   await Promise.all([
     state.hitStatsWritePromise,
     state.visitorStatsWritePromise,
+    state.analyticsEventsWritePromise,
+    state.episodePlaysWritePromise,
     state.contactSubmissionsWritePromise,
     state.replyTemplatesWritePromise,
     state.replyHistoryWritePromise,
@@ -1298,6 +1306,23 @@ function sanitizeLoadedHitStats(value) {
   }
 }
 
+function sanitizeLoadedAnalyticsEvents(value) {
+  return {
+    scrollDepth: value?.scrollDepth && typeof value.scrollDepth === 'object' ? value.scrollDepth : {},
+    timeOnPage: value?.timeOnPage && typeof value.timeOnPage === 'object' ? value.timeOnPage : {},
+    outboundClicks: value?.outboundClicks && typeof value.outboundClicks === 'object' ? value.outboundClicks : {},
+    linkClicks: value?.linkClicks && typeof value.linkClicks === 'object' ? value.linkClicks : {},
+    utmSources: value?.utmSources && typeof value.utmSources === 'object' ? value.utmSources : {},
+    searchQueries: value?.searchQueries && typeof value.searchQueries === 'object' ? value.searchQueries : {},
+    notFound: value?.notFound && typeof value.notFound === 'object' ? value.notFound : {},
+    audioEvents: value?.audioEvents && typeof value.audioEvents === 'object' ? value.audioEvents : {},
+  }
+}
+
+function sanitizeLoadedEpisodePlays(value) {
+  return (value && typeof value === 'object' && !Array.isArray(value)) ? value : {}
+}
+
 function sanitizeLoadedVisitorStats(value) {
   const loadedVisitors = value?.visitors && typeof value.visitors === 'object' ? value.visitors : {}
 

server/routes/analytics.js

@@ -75,28 +75,39 @@ async function resolveGeo(ip, ipHash) {
   return fallback
 }
 
-// Simple in-memory rate limiter for pageview endpoint
+// Simple in-memory rate limiter for pageview endpoint.
+// This keys on req.ip, which collapses to one value for every visitor behind
+// a shared IP (a proxy in front of the app, a school/church network, CGNAT).
+// If TRUST_PROXY_HOPS is misconfigured for the real deployment topology,
+// req.ip silently becomes the proxy's IP for everyone and this limiter starts
+// dropping real traffic across all of them together. Log the first drop per
+// IP per window so that scenario shows up instead of vanishing silently.
 const pageviewHits = new Map()
 function pageviewAllowed(ip) {
   const now = Date.now()
   const windowMs = 60 * 1000
-  const entry = pageviewHits.get(ip) ?? { count: 0, start: now }
+  const entry = pageviewHits.get(ip) ?? { count: 0, start: now, warned: false }
-  if (now - entry.start > windowMs) { entry.count = 0; entry.start = now }
+  if (now - entry.start > windowMs) { entry.count = 0; entry.start = now; entry.warned = false }
   entry.count += 1
   pageviewHits.set(ip, entry)
   if (pageviewHits.size > 5000) {
     const cutoff = now - windowMs
     for (const [k, v] of pageviewHits) { if (v.start < cutoff) pageviewHits.delete(k) }
   }
-  return entry.count <= 60
+  const allowed = entry.count <= 60
+  if (!allowed && !entry.warned) {
+    entry.warned = true
+    console.warn(`[analytics] pageview rate limit hit for an IP (${entry.count} requests/min) — if this fires often, check TRUST_PROXY_HOPS against the real deployment topology, since a misconfigured hop count collapses many visitors onto one IP here`)
+  }
+  return allowed
 }
 
 export function recordVisitor(req, res, overridePath = null, overrideReferrer = null) {
   const cookies = parseCookies(req.headers.cookie)
   let visitorId = cookies[VISITOR_COOKIE]
+  const hadCookie = Boolean(visitorId)
   if (!visitorId) {
     visitorId = randomUUID()
-    res.append('Set-Cookie', `${VISITOR_COOKIE}=${encodeURIComponent(visitorId)}; Max-Age=31536000; Path=/; SameSite=Lax; HttpOnly`)
   }
 
   const nowIso = new Date().toISOString()
@@ -112,9 +123,18 @@ export function recordVisitor(req, res, overridePath = null, overrideReferrer =
 
   const ipHash = createHash('sha256').update(ip).digest('hex')
 
+  // Only fall back to IP-based correlation when the browser sent no cookie
+  // at all — e.g. it never persisted (Safari ITP, a cookie-blocking
+  // extension) — so that one real person doesn't get double-counted every
+  // visit. If a cookie WAS sent, trust it: two different people sharing a
+  // network (family wifi, a school or church LAN) must not be silently
+  // merged into one visitor just because they share an IP, which would
+  // corrupt page history and undercount unique visitors.
+  if (!hadCookie) {
     const existingIdByIp = state.visitorStats.ipHashIndex[ipHash]
-  if (existingIdByIp && existingIdByIp !== visitorId) {
+    if (existingIdByIp) {
       visitorId = existingIdByIp
+    }
     res.append('Set-Cookie', `${VISITOR_COOKIE}=${encodeURIComponent(visitorId)}; Max-Age=31536000; Path=/; SameSite=Lax; HttpOnly`)
   }
 

server/study-helpers.js

@@ -413,7 +413,13 @@ export function detectBot(userAgent) {
     return { isBot: true, reason: 'http-client' }
   }
   if (/bot|crawler|spider|scraper|indexer|reader|fetcher|loader|agent|spyware|tracking|monitor/.test(ua)) {
-    if (!/chrome|firefox|safari|opera|edge|msie|trident|like gecko/.test(ua)) {
+    // Generic keywords like "reader" or "agent" can appear as substrings of
+    // otherwise-normal browser UAs (e.g. some webviews, regional browsers).
+    // Any UA carrying a real rendering-engine or vendor token is treated as
+    // a genuine browser rather than a bot, even if it also matches a keyword
+    // above — this list should stay a superset of the previous one so it
+    // only ever reduces false positives, never re-admits an actual bot.
+    if (!/chrome|firefox|safari|opera|edge|msie|trident|gecko|webkit|khtml|presto|crios|fxios|ucbrowser|samsungbrowser/.test(ua)) {
       return { isBot: true, reason: 'bot-keyword' }
     }
   }

src/App.tsx

@@ -15,7 +15,7 @@ import { EpisodeAudioPlayer } from './components/EpisodeAudioPlayer'
 import { NuggetsHostPortal } from './components/NuggetsHostPortal'
 import { BIBLE_BOOKS } from '../shared/bible-books.js'
 import './App.css'
-import { sendEvent, useScrollDepthTracking, useTimeOnPage, useUTMCapture, useLinkClickTracking } from './analytics'
+import { sendEvent, useScrollDepthTracking, useTimeOnPage, useUTMCapture, useLinkClickTracking, checkAnalyticsConsentResponse } from './analytics'
 
 const CONSENT_KEY = 'vbn_analytics_consent_choice'
 const HEADLINER_WIDGET_ID = 'WI_cmou3b4q7000701p0o9qmmcfj'
@@ -28,7 +28,16 @@ function ScrollToTop() {
 
 function usePageTracking() {
   const location = useLocation()
+  const isFirstRun = useRef(true)
   useEffect(() => {
+    // The initial hard page load is already counted server-side by the
+    // hit-counting middleware in server.js (it sees the raw HTML request).
+    // Only client-side route changes (no new HTTP request) need reporting
+    // here — otherwise every fresh visit/refresh gets double-counted.
+    if (isFirstRun.current) {
+      isFirstRun.current = false
+      return
+    }
     if (localStorage.getItem(CONSENT_KEY) !== 'accepted') return
     const referrer = document.referrer || ''
     const controller = new AbortController()
@@ -39,7 +48,7 @@ function usePageTracking() {
       body: JSON.stringify({ path: location.pathname, referrer }),
       keepalive: true,
       signal: controller.signal,
-    }).catch(() => {}).finally(() => clearTimeout(timeout))
+    }).then(checkAnalyticsConsentResponse).catch(() => {}).finally(() => clearTimeout(timeout))
   }, [location.pathname])
 }
 
@@ -1794,7 +1803,7 @@ function LatestEpisodePlayer({ content }: { content: SiteContent }) {
         listenStartRef.current = Date.now()
         if (!playTrackedRef.current && title) {
           playTrackedRef.current = true
-          fetch('/api/analytics/play', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ title }) }).catch(() => {})
+          fetch('/api/analytics/play', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ title }) }).then(checkAnalyticsConsentResponse).catch(() => {})
         }
       }).catch(() => {})
     }

src/analytics.ts

@@ -24,6 +24,19 @@ function hasConsent(): boolean {
   } catch { /* sessionStorage unavailable */ }
 })()
 
+// The client gates on a localStorage flag; the server gates on a cookie.
+// If the cookie is lost (cleared separately, dropped by the browser on a
+// plain-HTTP deployment, etc.) while localStorage still says "accepted",
+// the client would otherwise keep believing tracking is on and silently
+// lose every event forever. Self-heal by clearing the stale flag so the
+// consent banner reappears instead of leaking data indefinitely.
+export async function checkAnalyticsConsentResponse(res: Response): Promise<void> {
+  try {
+    const data = await res.json()
+    if (data?.reason === 'no-consent') localStorage.removeItem(CONSENT_KEY)
+  } catch { /* ignore */ }
+}
+
 export function sendEvent(type: string, data: Record<string, unknown>): void {
   if (!hasConsent()) return
   fetch('/api/analytics/event', {
@@ -31,7 +44,7 @@ export function sendEvent(type: string, data: Record<string, unknown>): void {
     headers: { 'Content-Type': 'application/json' },
     body: JSON.stringify({ type, ...data }),
     keepalive: true,
-  }).catch(() => {})
+  }).then(checkAnalyticsConsentResponse).catch(() => {})
 }
 
 // --- Scroll depth ---

src/components/AnalyticsPanel.tsx

@@ -273,28 +273,38 @@ export function AnalyticsPanel({
         </div>
       )}
 
-      {/* KPI Grid */}
+      {/* KPI Grid — split into two clearly-labeled groups so a number from
-      <div className="admin-stats-grid">
+          the selected range is never mistaken for an all-time total, or
+          vice versa. Previously all six cards sat in one grid with three
+          different implicit time windows (selected range, a hardcoded last
+          30 days, and all-time), which read as contradictory when e.g. "7
+          days" was selected but a "Last 30 Days" card sat right next to it. */}
+      <h3 style={{ marginBottom: '0.5rem', fontSize: '0.9rem', color: '#b0a48c' }}>{rangeLabel}</h3>
+      <div className="admin-stats-grid" style={{ marginBottom: '1.5rem' }}>
         <article>
           <h3>Real Hits</h3>
           <p>{realHits.toLocaleString()}</p>
-          <small>{rangeLabel}</small>
+          <small>Human traffic</small>
         </article>
         <article>
           <h3>Bot Traffic</h3>
           <p>{botHits.toLocaleString()}</p>
           <small>{rangeTotalHits > 0 ? `${((botHits / rangeTotalHits) * 100).toFixed(1)}% of total` : '0% of total'}</small>
         </article>
+        <article>
+          <h3>Total Hits</h3>
+          <p>{rangeTotalHits.toLocaleString()}</p>
+          <small>Real + bot</small>
+        </article>
+      </div>
+
+      <h3 style={{ marginBottom: '0.5rem', fontSize: '0.9rem', color: '#b0a48c' }}>All-Time</h3>
+      <div className="admin-stats-grid">
         <article>
           <h3>All-Time Real Hits</h3>
           <p>{((stats as AdminStats & { allTimeRealHits?: number }).allTimeRealHits ?? stats.totalHits).toLocaleString()}</p>
           <small>Since {formatDate(stats.firstHitAt)}</small>
         </article>
-        <article>
-          <h3>Last 30 Days</h3>
-          <p>{last30DaysRealTotal.toLocaleString()}</p>
-          <small>Real hits · {last30DaysBotTotal.toLocaleString()} bots</small>
-        </article>
         <article>
           <h3>Unique Visitors</h3>
           <p>{stats.visitors.uniqueVisitors.toLocaleString()}</p>
@@ -305,6 +315,11 @@ export function AnalyticsPanel({
           <p>{stats.visitors.totalVisits > 0 ? `${Math.round((stats.visitors.returningVisits / stats.visitors.totalVisits) * 100)}%` : '0%'}</p>
           <small>{stats.visitors.returningVisits.toLocaleString()} return visits</small>
         </article>
+        <article>
+          <h3>Last 30 Days</h3>
+          <p>{last30DaysRealTotal.toLocaleString()}</p>
+          <small>Real hits · {last30DaysBotTotal.toLocaleString()} bots</small>
+        </article>
       </div>
 
       {/* Chart Tabs */}
@@ -409,6 +424,14 @@ export function AnalyticsPanel({
                 ))}
               </ul>
             </div>
+            <div>
+              <h3>Top States</h3>
+              <ul>
+                {stats.visitors.topStates.slice(0, 10).map((item: { name: string; hits: number }) => (
+                  <li key={item.name}><span>{item.name}</span><strong>{item.hits.toLocaleString()}</strong></li>
+                ))}
+              </ul>
+            </div>
             <div>
               <h3>Top Countries</h3>
               <ul>
@@ -451,29 +474,12 @@ export function AnalyticsPanel({
 
       <div className="admin-stats-grid admin-stats-grid--visitor">
         <article><h3>Total Visits</h3><p>{stats.visitors.totalVisits.toLocaleString()}</p><small>All time</small></article>
-        <article><h3>Unique Visitors</h3><p>{stats.visitors.uniqueVisitors.toLocaleString()}</p><small>All time</small></article>
         <article><h3>Returning Visits</h3><p>{stats.visitors.returningVisits.toLocaleString()}</p></article>
         <article><h3>First Visit</h3><p>{formatDate(stats.visitors.firstVisitAt)}</p></article>
       </div>
-
+      {/* Top Countries/States/Cities live in the Geographic tab above, alongside
-      <div className="admin-stats-lists">
+          the map — not repeated here to avoid showing the same numbers twice
-        <div>
+          on one page. */}
-          <h3>Top Countries</h3>
-          <ul>
-            {stats.visitors.topCountries.map((item: { name: string; hits: number }) => (
-              <li key={item.name}><span>{item.name}</span><strong>{item.hits.toLocaleString()}</strong></li>
-            ))}
-          </ul>
-        </div>
-        <div>
-          <h3>Top States</h3>
-          <ul>
-            {stats.visitors.topStates.map((item: { name: string; hits: number }) => (
-              <li key={item.name}><span>{item.name}</span><strong>{item.hits.toLocaleString()}</strong></li>
-            ))}
-          </ul>
-        </div>
-      </div>
 
       <div className="admin-visits-table-wrap">
         <h3>Recent Visitor Log</h3>

src/components/EpisodeAudioPlayer.tsx

@@ -1,5 +1,5 @@
 import { useEffect, useRef, useState } from 'react'
-import { sendEvent } from '../analytics'
+import { sendEvent, checkAnalyticsConsentResponse } from '../analytics'
 
 interface EpisodeAudioPlayerProps {
   src: string
@@ -101,7 +101,7 @@ export function EpisodeAudioPlayer({ src, title, size = 'full', spotifyUrl }: Ep
             method: 'POST',
             headers: { 'Content-Type': 'application/json' },
             body: JSON.stringify({ title }),
-          }).catch(() => {})
+          }).then(checkAnalyticsConsentResponse).catch(() => {})
         }
       }).catch(() => {})
     }