analytics fixs

server.js

@@ -612,6 +612,7 @@ const EMPTY_VISITOR_STATS = {
   firstVisitAt: null,
   lastVisitAt: null,
   visitors: {},
+  ipHashIndex: {},   // ipHash → visitorId — prevents same IP counting as multiple unique visitors
   recentVisits: [],
   geoCacheByIp: {},
 }
@@ -1343,6 +1344,24 @@ function sanitizeUserAgent(userAgent) {
   return userAgent.trim().slice(0, 300) || 'unknown'
 }
 
+function detectDevice(userAgent) {
+  if (!userAgent || typeof userAgent !== 'string') return 'unknown'
+  const ua = userAgent.toLowerCase()
+  if (/tablet|ipad|playbook|silk|(android(?!.*mobile))/.test(ua)) return 'tablet'
+  if (/mobile|iphone|ipod|android|blackberry|opera mini|opera mobi|iemobile|windows phone|palm|smartphone/.test(ua)) return 'mobile'
+  return 'desktop'
+}
+
+function sanitizeReferrer(referrer) {
+  if (!referrer || typeof referrer !== 'string') return ''
+  try {
+    const parsed = new URL(referrer.trim())
+    return `${parsed.hostname}${parsed.pathname}`.slice(0, 200)
+  } catch {
+    return ''
+  }
+}
+
 function detectBot(userAgent, pathInfo = {}) {
   if (!userAgent || typeof userAgent !== 'string') {
     return { isBot: true, reason: 'missing-user-agent' }
@@ -1472,7 +1491,7 @@ async function resolveGeo(ip) {
   return fallback
 }
 
-async function recordVisitor(req, res) {
+async function recordVisitor(req, res, overridePath = null, overrideReferrer = null) {
   const cookies = parseCookies(req.headers.cookie)
   let visitorId = cookies[VISITOR_COOKIE]
   if (!visitorId) {
@@ -1481,24 +1500,42 @@ async function recordVisitor(req, res) {
   }
 
   const nowIso = new Date().toISOString()
-  const pathKey = normalizeHitPath(req.path)
+  const pathKey = overridePath ? normalizeHitPath(overridePath) : normalizeHitPath(req.path)
+  const referrer = overrideReferrer !== null ? sanitizeReferrer(overrideReferrer) : sanitizeReferrer(req.get('referer') || req.get('referrer') || '')
   const ip = getClientIp(req)
   const ua = sanitizeUserAgent(req.get('user-agent'))
+  const device = detectDevice(ua)
+
+  const ipHash = createHash('sha256').update(ip).digest('hex')
+  const geo = await resolveGeo(ip)
+
+  // Resolve canonical visitorId by IP hash — if this IP was seen before under a
+  // different cookie (e.g. cleared cookies), reuse the existing record so the
+  // same person is never counted as a second unique visitor.
+  const existingIdByIp = visitorStats.ipHashIndex[ipHash]
+  if (existingIdByIp && existingIdByIp !== visitorId) {
+    // Reuse the existing record for this IP; overwrite cookie with canonical ID
+    visitorId = existingIdByIp
+    res.append('Set-Cookie', `${VISITOR_COOKIE}=${encodeURIComponent(visitorId)}; Max-Age=31536000; Path=/; SameSite=Lax`)
+  }
 
   const existingVisitor = visitorStats.visitors[visitorId]
   const isReturning = Boolean(existingVisitor)
-  const geo = await resolveGeo(ip)
 
   if (!existingVisitor) {
     visitorStats.uniqueVisitors += 1
+    visitorStats.ipHashIndex[ipHash] = visitorId
   } else {
     visitorStats.returningVisits += 1
   }
 
-  const ipHash = createHash('sha256').update(ip).digest('hex')
   const nextVisitCount = (existingVisitor?.visitCount ?? 0) + 1
   const userAgents = Array.from(new Set([...(existingVisitor?.userAgents ?? []), ua])).slice(0, 5)
 
+  // Append to page history, keeping last 100 entries per visitor
+  const prevHistory = existingVisitor?.pageHistory ?? []
+  const pageHistory = [...prevHistory, { at: nowIso, path: pathKey, referrer }].slice(-100)
+
   visitorStats.visitors[visitorId] = {
     visitorId,
     ip,
@@ -1510,6 +1547,8 @@ async function recordVisitor(req, res) {
     returningVisitor: isReturning,
     location: geo,
     userAgents,
+    device,
+    pageHistory,
   }
 
   visitorStats.totalVisits += 1
@@ -1520,6 +1559,8 @@ async function recordVisitor(req, res) {
     visitorId,
     ip,
     path: pathKey,
+    referrer,
+    device,
     country: geo.country,
     state: geo.state,
     county: geo.county,
@@ -1536,13 +1577,26 @@ function loadVisitorStatsFromDisk() {
   return readFile(VISITOR_STATS_FILE, 'utf8')
     .then(raw => {
       const parsed = JSON.parse(raw)
+      const loadedVisitors = parsed?.visitors && typeof parsed.visitors === 'object' ? parsed.visitors : {}
+
+      // Rebuild ipHashIndex from saved visitors if not persisted (handles upgrades from old data)
+      let ipHashIndex = parsed?.ipHashIndex && typeof parsed.ipHashIndex === 'object' ? parsed.ipHashIndex : {}
+      if (Object.keys(ipHashIndex).length === 0 && Object.keys(loadedVisitors).length > 0) {
+        for (const [vid, visitor] of Object.entries(loadedVisitors)) {
+          if (visitor?.ipHash && typeof visitor.ipHash === 'string') {
+            ipHashIndex[visitor.ipHash] = vid
+          }
+        }
+      }
+
       visitorStats = {
         totalVisits: Number(parsed?.totalVisits) || 0,
         uniqueVisitors: Number(parsed?.uniqueVisitors) || 0,
         returningVisits: Number(parsed?.returningVisits) || 0,
         firstVisitAt: typeof parsed?.firstVisitAt === 'string' ? parsed.firstVisitAt : null,
         lastVisitAt: typeof parsed?.lastVisitAt === 'string' ? parsed.lastVisitAt : null,
-        visitors: parsed?.visitors && typeof parsed.visitors === 'object' ? parsed.visitors : {},
+        visitors: loadedVisitors,
+        ipHashIndex,
         recentVisits: Array.isArray(parsed?.recentVisits) ? parsed.recentVisits.slice(0, MAX_RECENT_VISITS) : [],
         geoCacheByIp: parsed?.geoCacheByIp && typeof parsed.geoCacheByIp === 'object' ? parsed.geoCacheByIp : {},
       }
@@ -4023,6 +4077,25 @@ app.post('/api/analytics-consent', (req, res) => {
   res.json({ ok: true, consent })
 })
 
+// Client-side SPA pageview tracking (fires on every React Router navigation)
+app.post('/api/analytics/pageview', async (req, res) => {
+  if (!hasVisitorConsent(req)) {
+    res.json({ ok: false, reason: 'no-consent' })
+    return
+  }
+  const ua = req.get('user-agent') ?? ''
+  const { isBot } = detectBot(ua)
+  if (isBot) {
+    res.json({ ok: false, reason: 'bot' })
+    return
+  }
+  const rawPath = typeof req.body?.path === 'string' ? req.body.path : '/'
+  const rawReferrer = typeof req.body?.referrer === 'string' ? req.body.referrer : ''
+  recordHit(rawPath, false)
+  await recordVisitor(req, res, rawPath, rawReferrer)
+  res.json({ ok: true })
+})
+
 app.get('/api/admin-stats', requireAdminAuth, (_req, res) => {
   const topPaths = Object.entries(hitStats.byPath)
     .sort((a, b) => b[1] - a[1])
@@ -4059,7 +4132,10 @@ app.get('/api/admin-stats', requireAdminAuth, (_req, res) => {
     .slice(0, 10)
     .map(([reason, count]) => ({ reason, count }))
 
-  const recentVisitorRows = visitorStats.recentVisits.slice(0, 100)
+  const recentVisitorRows = visitorStats.recentVisits.slice(0, 100).map(row => {
+    const fullVisitor = visitorStats.visitors[row.visitorId]
+    return { ...row, pageHistory: fullVisitor?.pageHistory ?? [] }
+  })
   const enrollmentCountsBySlug = {}
   for (const user of studyUsers) {
     const userEnrollments = Array.isArray(user?.enrolledStudySlugs) ? user.enrolledStudySlugs : []
@@ -4126,6 +4202,26 @@ app.get('/api/admin-stats', requireAdminAuth, (_req, res) => {
       topStates: buildTopLocations(recentVisitorRows, 'state'),
       topCounties: buildTopLocations(recentVisitorRows, 'county'),
       topCities: buildTopLocations(recentVisitorRows, 'city'),
+      deviceBreakdown: (() => {
+        const counts = { mobile: 0, desktop: 0, tablet: 0, unknown: 0 }
+        for (const row of recentVisitorRows) {
+          const d = row.device ?? 'unknown'
+          counts[d] = (counts[d] ?? 0) + 1
+        }
+        return counts
+      })(),
+      topReferrers: (() => {
+        const counts = {}
+        for (const row of recentVisitorRows) {
+          if (!row.referrer) continue
+          counts[row.referrer] = (counts[row.referrer] ?? 0) + 1
+        }
+        return Object.entries(counts)
+          .sort((a, b) => b[1] - a[1])
+          .slice(0, 10)
+          .map(([referrer, count]) => ({ referrer, count }))
+      })(),
+      last30DaysReal: buildLastNDaysStats(30).map(item => ({ day: item.day, hits: hitStats.byDayReal?.[item.day] ?? 0 })),
       recentVisits: recentVisitorRows,
     },
     writeStatus: {