refractor server.js

2026-06-03 15:08:14 -04:00
parent e17a818deb
commit 346ecb217c
20 changed files with 5980 additions and 5947 deletions
@@ -0,0 +1,279 @@
+import { createHash, randomUUID } from 'node:crypto'
+import { requireAdminAuth } from '../auth.js'
+import { getClientIp, hasVisitorConsent, setConsentCookie, parseCookies } from '../helpers.js'
+import {
+  VISITOR_COOKIE,
+  MAX_RECENT_VISITS,
+} from '../config.js'
+import { state } from '../state.js'
+import { queueVisitorStatsWrite, queueHitStatsWrite, normalizeMessageType } from '../data.js'
+import {
+  detectBot,
+  sanitizeUserAgent,
+  detectDevice,
+  sanitizeReferrer,
+  normalizeHitPath,
+  isPrivateOrLocalIp,
+  buildTopLocations,
+  buildLastNDaysStats,
+  shouldCountHit,
+  recordHit,
+} from '../study-helpers.js'
+import { getStudyCatalog } from '../study-helpers.js'
+
+async function resolveGeo(ip) {
+  if (!ip || isPrivateOrLocalIp(ip)) {
+    return { country: 'Local/Unknown', state: 'Local/Unknown', county: 'Local/Unknown', city: 'Local/Unknown' }
+  }
+
+  const cached = state.visitorStats.geoCacheByIp[ip]
+  if (cached) return cached
+
+  const providers = [
+    async () => {
+      const controller = new AbortController()
+      const timeout = setTimeout(() => controller.abort(), 2500)
+      try {
+        const response = await fetch(
+          `http://ip-api.com/json/${encodeURIComponent(ip)}?fields=status,country,regionName,city,district`,
+          { signal: controller.signal },
+        )
+        if (!response.ok) return null
+        const data = await response.json()
+        if (data?.status !== 'success') return null
+        return { country: data?.country || 'Unknown', state: data?.regionName || 'Unknown', county: data?.district || 'Unknown', city: data?.city || 'Unknown' }
+      } finally { clearTimeout(timeout) }
+    },
+    async () => {
+      const controller = new AbortController()
+      const timeout = setTimeout(() => controller.abort(), 2500)
+      try {
+        const response = await fetch(`https://ipwho.is/${encodeURIComponent(ip)}`, { signal: controller.signal })
+        if (!response.ok) return null
+        const data = await response.json()
+        if (!data?.success) return null
+        return { country: data?.country || 'Unknown', state: data?.region || 'Unknown', county: data?.region || 'Unknown', city: data?.city || 'Unknown' }
+      } finally { clearTimeout(timeout) }
+    },
+  ]
+
+  for (const provider of providers) {
+    try {
+      const geo = await provider()
+      if (geo) {
+        state.visitorStats.geoCacheByIp[ip] = geo
+        queueVisitorStatsWrite()
+        return geo
+      }
+    } catch { /* Try next provider */ }
+  }
+
+  const fallback = { country: 'Unknown', state: 'Unknown', county: 'Unknown', city: 'Unknown' }
+  state.visitorStats.geoCacheByIp[ip] = fallback
+  queueVisitorStatsWrite()
+  return fallback
+}
+
+export async function recordVisitor(req, res, overridePath = null, overrideReferrer = null) {
+  const cookies = parseCookies(req.headers.cookie)
+  let visitorId = cookies[VISITOR_COOKIE]
+  if (!visitorId) {
+    visitorId = randomUUID()
+    res.append('Set-Cookie', `${VISITOR_COOKIE}=${encodeURIComponent(visitorId)}; Max-Age=31536000; Path=/; SameSite=Lax`)
+  }
+
+  const nowIso = new Date().toISOString()
+  const pathKey = overridePath ? normalizeHitPath(overridePath) : normalizeHitPath(req.path)
+  const referrer = overrideReferrer !== null ? sanitizeReferrer(overrideReferrer) : sanitizeReferrer(req.get('referer') || req.get('referrer') || '')
+  const ip = getClientIp(req)
+  const ua = sanitizeUserAgent(req.get('user-agent'))
+  const device = detectDevice(ua)
+
+  const ipHash = createHash('sha256').update(ip).digest('hex')
+  const geo = await resolveGeo(ip)
+
+  const existingIdByIp = state.visitorStats.ipHashIndex[ipHash]
+  if (existingIdByIp && existingIdByIp !== visitorId) {
+    visitorId = existingIdByIp
+    res.append('Set-Cookie', `${VISITOR_COOKIE}=${encodeURIComponent(visitorId)}; Max-Age=31536000; Path=/; SameSite=Lax`)
+  }
+
+  const existingVisitor = state.visitorStats.visitors[visitorId]
+  const isReturning = Boolean(existingVisitor)
+
+  if (!existingVisitor) {
+    state.visitorStats.uniqueVisitors += 1
+    state.visitorStats.ipHashIndex[ipHash] = visitorId
+  } else {
+    state.visitorStats.returningVisits += 1
+  }
+
+  const nextVisitCount = (existingVisitor?.visitCount ?? 0) + 1
+  const userAgents = Array.from(new Set([...(existingVisitor?.userAgents ?? []), ua])).slice(0, 5)
+  const prevHistory = existingVisitor?.pageHistory ?? []
+  const pageHistory = [...prevHistory, { at: nowIso, path: pathKey, referrer }].slice(-100)
+
+  state.visitorStats.visitors[visitorId] = {
+    visitorId, ip, ipHash,
+    firstSeenAt: existingVisitor?.firstSeenAt ?? nowIso,
+    lastSeenAt: nowIso,
+    visitCount: nextVisitCount,
+    lastPath: pathKey,
+    returningVisitor: isReturning,
+    location: geo,
+    userAgents,
+    device,
+    pageHistory,
+  }
+
+  state.visitorStats.totalVisits += 1
+  state.visitorStats.firstVisitAt = state.visitorStats.firstVisitAt ?? nowIso
+  state.visitorStats.lastVisitAt = nowIso
+  state.visitorStats.recentVisits.unshift({
+    at: nowIso, visitorId, ip, path: pathKey, referrer, device,
+    country: geo.country, state: geo.state, county: geo.county, city: geo.city,
+    returningVisitor: isReturning, visitCount: nextVisitCount,
+  })
+  state.visitorStats.recentVisits = state.visitorStats.recentVisits.slice(0, MAX_RECENT_VISITS)
+
+  queueVisitorStatsWrite()
+}
+
+export function register(app) {
+  app.post('/api/analytics-consent', (req, res) => {
+    const consent = req.body?.consent === true
+    setConsentCookie(res, consent)
+    res.json({ ok: true, consent })
+  })
+
+  app.post('/api/analytics/pageview', async (req, res) => {
+    if (!hasVisitorConsent(req)) {
+      res.json({ ok: false, reason: 'no-consent' }); return
+    }
+    const ua = req.get('user-agent') ?? ''
+    const { isBot } = detectBot(ua)
+    if (isBot) {
+      res.json({ ok: false, reason: 'bot' }); return
+    }
+    const rawPath = typeof req.body?.path === 'string' ? req.body.path : '/'
+    const rawReferrer = typeof req.body?.referrer === 'string' ? req.body.referrer : ''
+    recordHit(rawPath, false)
+    queueHitStatsWrite()
+    await recordVisitor(req, res, rawPath, rawReferrer)
+    res.json({ ok: true })
+  })
+
+  app.get('/api/admin-stats', requireAdminAuth, (_req, res) => {
+    const topPaths = Object.entries(state.hitStats.byPath)
+      .sort((a, b) => b[1] - a[1]).slice(0, 10).map(([pathKey, hits]) => ({ path: pathKey, hits }))
+    const topPathsReal = Object.entries(state.hitStats.byPathReal)
+      .sort((a, b) => b[1] - a[1]).slice(0, 10).map(([pathKey, hits]) => ({ path: pathKey, hits }))
+    const topPathsBot = Object.entries(state.hitStats.byPathBot)
+      .sort((a, b) => b[1] - a[1]).slice(0, 10).map(([pathKey, hits]) => ({ path: pathKey, hits }))
+
+    const last7Days = buildLastNDaysStats(7)
+    const last7DaysReal = last7Days.map(item => ({ day: item.day, hits: state.hitStats.byDayReal?.[item.day] ?? 0 }))
+    const last7DaysBot = last7Days.map(item => ({ day: item.day, hits: state.hitStats.byDayBot?.[item.day] ?? 0 }))
+
+    const last30Days = buildLastNDaysStats(30)
+    const last30DaysTotal = last30Days.reduce((sum, item) => sum + item.hits, 0)
+    const last30DaysRealTotal = last30Days.reduce((sum, item) => sum + (state.hitStats.byDayReal?.[item.day] ?? 0), 0)
+    const last30DaysBotTotal = last30Days.reduce((sum, item) => sum + (state.hitStats.byDayBot?.[item.day] ?? 0), 0)
+
+    const botReasons = Object.entries(state.hitStats.botReasons ?? {})
+      .sort((a, b) => b[1] - a[1]).slice(0, 10).map(([reason, count]) => ({ reason, count }))
+
+    const recentVisitorRows = state.visitorStats.recentVisits.slice(0, 100).map(row => {
+      const fullVisitor = state.visitorStats.visitors[row.visitorId]
+      return { ...row, pageHistory: fullVisitor?.pageHistory ?? [] }
+    })
+
+    const enrollmentCountsBySlug = {}
+    for (const user of state.studyUsers) {
+      const userEnrollments = Array.isArray(user?.enrolledStudySlugs) ? user.enrolledStudySlugs : []
+      for (const studySlug of userEnrollments) {
+        enrollmentCountsBySlug[studySlug] = (enrollmentCountsBySlug[studySlug] ?? 0) + 1
+      }
+    }
+    const enrollmentsByStudy = getStudyCatalog()
+      .map(study => ({ slug: study.slug, title: study.title, count: enrollmentCountsBySlug[study.slug] ?? 0 }))
+      .sort((a, b) => b.count - a.count)
+
+    const studyCatalogBySlug = new Map(getStudyCatalog().map(study => [study.slug, study]))
+    const users = state.studyUsers
+      .map(user => {
+        const enrolledStudySlugs = Array.isArray(user?.enrolledStudySlugs) ? user.enrolledStudySlugs : []
+        const enrolledStudies = enrolledStudySlugs.map(slug => {
+          const study = studyCatalogBySlug.get(slug)
+          return study ? { slug: study.slug, title: study.title } : null
+        }).filter(Boolean)
+        return { id: user.id, username: user.username, displayName: user.displayName ?? '', enrolledStudies }
+      })
+      .sort((a, b) => {
+        if (b.enrolledStudies.length !== a.enrolledStudies.length) return b.enrolledStudies.length - a.enrolledStudies.length
+        return a.username.localeCompare(b.username)
+      })
+
+    const enrolledUsers = state.studyUsers.filter(user => (user.enrolledStudySlugs?.length ?? 0) > 0).length
+    const totalEnrollments = Object.values(enrollmentCountsBySlug).reduce((sum, count) => sum + count, 0)
+
+    res.json({
+      totalHits: state.hitStats.totalHits,
+      realHits: state.hitStats.realHits ?? 0,
+      botHits: state.hitStats.botHits ?? 0,
+      firstHitAt: state.hitStats.firstHitAt,
+      lastHitAt: state.hitStats.lastHitAt,
+      topPaths, topPathsReal, topPathsBot,
+      last7Days, last7DaysReal, last7DaysBot,
+      last30DaysTotal, last30DaysRealTotal, last30DaysBotTotal,
+      botReasons,
+      visitors: {
+        totalVisits: state.visitorStats.totalVisits,
+        uniqueVisitors: state.visitorStats.uniqueVisitors,
+        returningVisits: state.visitorStats.returningVisits,
+        firstVisitAt: state.visitorStats.firstVisitAt,
+        lastVisitAt: state.visitorStats.lastVisitAt,
+        topCountries: buildTopLocations(recentVisitorRows, 'country'),
+        topStates: buildTopLocations(recentVisitorRows, 'state'),
+        topCounties: buildTopLocations(recentVisitorRows, 'county'),
+        topCities: buildTopLocations(recentVisitorRows, 'city'),
+        deviceBreakdown: (() => {
+          const counts = { mobile: 0, desktop: 0, tablet: 0, unknown: 0 }
+          for (const row of recentVisitorRows) {
+            const d = row.device ?? 'unknown'
+            counts[d] = (counts[d] ?? 0) + 1
+          }
+          return counts
+        })(),
+        topReferrers: (() => {
+          const counts = {}
+          for (const row of recentVisitorRows) {
+            if (!row.referrer) continue
+            counts[row.referrer] = (counts[row.referrer] ?? 0) + 1
+          }
+          return Object.entries(counts).sort((a, b) => b[1] - a[1]).slice(0, 10).map(([referrer, count]) => ({ referrer, count }))
+        })(),
+        last30DaysReal: buildLastNDaysStats(30).map(item => ({ day: item.day, hits: state.hitStats.byDayReal?.[item.day] ?? 0 })),
+        recentVisits: recentVisitorRows,
+      },
+      writeStatus: {
+        hitStats: state.lastHitStatsWrite,
+        visitorStats: state.lastVisitorStatsWrite,
+        backups: state.lastBackupStatus,
+        cachePurge: state.lastCachePurgeStatus,
+        deployHook: state.lastDeployHookStatus,
+      },
+      contactTotals: {
+        totalSubmissions: state.contactSubmissions.length,
+        totalQuestions: state.contactSubmissions.filter(entry => normalizeMessageType(entry?.messageType) === 'question').length,
+      },
+      studyEnrollment: {
+        totalUsers: state.studyUsers.length,
+        enrolledUsers,
+        totalEnrollments,
+        enrollmentsByStudy,
+        users,
+      },
+    })
+  })
+}