import { createHash, randomUUID } from 'node:crypto' import { requireAdminAuth } from '../auth.js' import { getClientIp, hasVisitorConsent, setConsentCookie, parseCookies } from '../helpers.js' import { VISITOR_COOKIE, MAX_RECENT_VISITS, } from '../config.js' import { state } from '../state.js' import { queueVisitorStatsWrite, queueHitStatsWrite, normalizeMessageType } from '../data.js' import { detectBot, sanitizeUserAgent, detectDevice, sanitizeReferrer, normalizeHitPath, isPrivateOrLocalIp, buildTopLocations, buildLastNDaysStats, shouldCountHit, recordHit, } from '../study-helpers.js' import { getStudyCatalog } from '../study-helpers.js' async function resolveGeo(ip) { if (!ip || isPrivateOrLocalIp(ip)) { return { country: 'Local/Unknown', state: 'Local/Unknown', county: 'Local/Unknown', city: 'Local/Unknown' } } const cached = state.visitorStats.geoCacheByIp[ip] if (cached) return cached const providers = [ async () => { const controller = new AbortController() const timeout = setTimeout(() => controller.abort(), 2500) try { const response = await fetch( `http://ip-api.com/json/${encodeURIComponent(ip)}?fields=status,country,regionName,city,district`, { signal: controller.signal }, ) if (!response.ok) return null const data = await response.json() if (data?.status !== 'success') return null return { country: data?.country || 'Unknown', state: data?.regionName || 'Unknown', county: data?.district || 'Unknown', city: data?.city || 'Unknown' } } finally { clearTimeout(timeout) } }, async () => { const controller = new AbortController() const timeout = setTimeout(() => controller.abort(), 2500) try { const response = await fetch(`https://ipwho.is/${encodeURIComponent(ip)}`, { signal: controller.signal }) if (!response.ok) return null const data = await response.json() if (!data?.success) return null return { country: data?.country || 'Unknown', state: data?.region || 'Unknown', county: data?.region || 'Unknown', city: data?.city || 'Unknown' } } finally { clearTimeout(timeout) } }, ] for (const provider of providers) { try { const geo = await provider() if (geo) { state.visitorStats.geoCacheByIp[ip] = geo queueVisitorStatsWrite() return geo } } catch { /* Try next provider */ } } const fallback = { country: 'Unknown', state: 'Unknown', county: 'Unknown', city: 'Unknown' } state.visitorStats.geoCacheByIp[ip] = fallback queueVisitorStatsWrite() return fallback } export async function recordVisitor(req, res, overridePath = null, overrideReferrer = null) { const cookies = parseCookies(req.headers.cookie) let visitorId = cookies[VISITOR_COOKIE] if (!visitorId) { visitorId = randomUUID() res.append('Set-Cookie', `${VISITOR_COOKIE}=${encodeURIComponent(visitorId)}; Max-Age=31536000; Path=/; SameSite=Lax`) } const nowIso = new Date().toISOString() const pathKey = overridePath ? normalizeHitPath(overridePath) : normalizeHitPath(req.path) const referrer = overrideReferrer !== null ? sanitizeReferrer(overrideReferrer) : sanitizeReferrer(req.get('referer') || req.get('referrer') || '') const ip = getClientIp(req) const ua = sanitizeUserAgent(req.get('user-agent')) const device = detectDevice(ua) const ipHash = createHash('sha256').update(ip).digest('hex') const geo = await resolveGeo(ip) const existingIdByIp = state.visitorStats.ipHashIndex[ipHash] if (existingIdByIp && existingIdByIp !== visitorId) { visitorId = existingIdByIp res.append('Set-Cookie', `${VISITOR_COOKIE}=${encodeURIComponent(visitorId)}; Max-Age=31536000; Path=/; SameSite=Lax`) } const existingVisitor = state.visitorStats.visitors[visitorId] const isReturning = Boolean(existingVisitor) if (!existingVisitor) { state.visitorStats.uniqueVisitors += 1 state.visitorStats.ipHashIndex[ipHash] = visitorId } else { state.visitorStats.returningVisits += 1 } const nextVisitCount = (existingVisitor?.visitCount ?? 0) + 1 const userAgents = Array.from(new Set([...(existingVisitor?.userAgents ?? []), ua])).slice(0, 5) const prevHistory = existingVisitor?.pageHistory ?? [] const pageHistory = [...prevHistory, { at: nowIso, path: pathKey, referrer }].slice(-100) state.visitorStats.visitors[visitorId] = { visitorId, ip, ipHash, firstSeenAt: existingVisitor?.firstSeenAt ?? nowIso, lastSeenAt: nowIso, visitCount: nextVisitCount, lastPath: pathKey, returningVisitor: isReturning, location: geo, userAgents, device, pageHistory, } state.visitorStats.totalVisits += 1 state.visitorStats.firstVisitAt = state.visitorStats.firstVisitAt ?? nowIso state.visitorStats.lastVisitAt = nowIso state.visitorStats.recentVisits.unshift({ at: nowIso, visitorId, ip, path: pathKey, referrer, device, country: geo.country, state: geo.state, county: geo.county, city: geo.city, returningVisitor: isReturning, visitCount: nextVisitCount, }) state.visitorStats.recentVisits = state.visitorStats.recentVisits.slice(0, MAX_RECENT_VISITS) queueVisitorStatsWrite() } export function register(app) { app.post('/api/analytics-consent', (req, res) => { const consent = req.body?.consent === true setConsentCookie(res, consent) res.json({ ok: true, consent }) }) app.post('/api/analytics/pageview', async (req, res) => { if (!hasVisitorConsent(req)) { res.json({ ok: false, reason: 'no-consent' }); return } const ua = req.get('user-agent') ?? '' const { isBot } = detectBot(ua) if (isBot) { res.json({ ok: false, reason: 'bot' }); return } const rawPath = typeof req.body?.path === 'string' ? req.body.path : '/' const rawReferrer = typeof req.body?.referrer === 'string' ? req.body.referrer : '' recordHit(rawPath, false) queueHitStatsWrite() await recordVisitor(req, res, rawPath, rawReferrer) res.json({ ok: true }) }) app.get('/api/admin-stats', requireAdminAuth, (_req, res) => { const topPaths = Object.entries(state.hitStats.byPath) .sort((a, b) => b[1] - a[1]).slice(0, 10).map(([pathKey, hits]) => ({ path: pathKey, hits })) const topPathsReal = Object.entries(state.hitStats.byPathReal) .sort((a, b) => b[1] - a[1]).slice(0, 10).map(([pathKey, hits]) => ({ path: pathKey, hits })) const topPathsBot = Object.entries(state.hitStats.byPathBot) .sort((a, b) => b[1] - a[1]).slice(0, 10).map(([pathKey, hits]) => ({ path: pathKey, hits })) const last7Days = buildLastNDaysStats(7) const last7DaysReal = last7Days.map(item => ({ day: item.day, hits: state.hitStats.byDayReal?.[item.day] ?? 0 })) const last7DaysBot = last7Days.map(item => ({ day: item.day, hits: state.hitStats.byDayBot?.[item.day] ?? 0 })) const last30Days = buildLastNDaysStats(30) const last30DaysTotal = last30Days.reduce((sum, item) => sum + item.hits, 0) const last30DaysRealTotal = last30Days.reduce((sum, item) => sum + (state.hitStats.byDayReal?.[item.day] ?? 0), 0) const last30DaysBotTotal = last30Days.reduce((sum, item) => sum + (state.hitStats.byDayBot?.[item.day] ?? 0), 0) const botReasons = Object.entries(state.hitStats.botReasons ?? {}) .sort((a, b) => b[1] - a[1]).slice(0, 10).map(([reason, count]) => ({ reason, count })) const recentVisitorRows = state.visitorStats.recentVisits.slice(0, 100).map(row => { const fullVisitor = state.visitorStats.visitors[row.visitorId] return { ...row, pageHistory: fullVisitor?.pageHistory ?? [] } }) const enrollmentCountsBySlug = {} for (const user of state.studyUsers) { const userEnrollments = Array.isArray(user?.enrolledStudySlugs) ? user.enrolledStudySlugs : [] for (const studySlug of userEnrollments) { enrollmentCountsBySlug[studySlug] = (enrollmentCountsBySlug[studySlug] ?? 0) + 1 } } const enrollmentsByStudy = getStudyCatalog() .map(study => ({ slug: study.slug, title: study.title, count: enrollmentCountsBySlug[study.slug] ?? 0 })) .sort((a, b) => b.count - a.count) const studyCatalogBySlug = new Map(getStudyCatalog().map(study => [study.slug, study])) const users = state.studyUsers .map(user => { const enrolledStudySlugs = Array.isArray(user?.enrolledStudySlugs) ? user.enrolledStudySlugs : [] const enrolledStudies = enrolledStudySlugs.map(slug => { const study = studyCatalogBySlug.get(slug) return study ? { slug: study.slug, title: study.title } : null }).filter(Boolean) return { id: user.id, username: user.username, displayName: user.displayName ?? '', enrolledStudies } }) .sort((a, b) => { if (b.enrolledStudies.length !== a.enrolledStudies.length) return b.enrolledStudies.length - a.enrolledStudies.length return a.username.localeCompare(b.username) }) const enrolledUsers = state.studyUsers.filter(user => (user.enrolledStudySlugs?.length ?? 0) > 0).length const totalEnrollments = Object.values(enrollmentCountsBySlug).reduce((sum, count) => sum + count, 0) res.json({ totalHits: state.hitStats.totalHits, realHits: state.hitStats.realHits ?? 0, botHits: state.hitStats.botHits ?? 0, firstHitAt: state.hitStats.firstHitAt, lastHitAt: state.hitStats.lastHitAt, topPaths, topPathsReal, topPathsBot, last7Days, last7DaysReal, last7DaysBot, last30DaysTotal, last30DaysRealTotal, last30DaysBotTotal, botReasons, visitors: { totalVisits: state.visitorStats.totalVisits, uniqueVisitors: state.visitorStats.uniqueVisitors, returningVisits: state.visitorStats.returningVisits, firstVisitAt: state.visitorStats.firstVisitAt, lastVisitAt: state.visitorStats.lastVisitAt, topCountries: buildTopLocations(recentVisitorRows, 'country'), topStates: buildTopLocations(recentVisitorRows, 'state'), topCounties: buildTopLocations(recentVisitorRows, 'county'), topCities: buildTopLocations(recentVisitorRows, 'city'), deviceBreakdown: (() => { const counts = { mobile: 0, desktop: 0, tablet: 0, unknown: 0 } for (const row of recentVisitorRows) { const d = row.device ?? 'unknown' counts[d] = (counts[d] ?? 0) + 1 } return counts })(), topReferrers: (() => { const counts = {} for (const row of recentVisitorRows) { if (!row.referrer) continue counts[row.referrer] = (counts[row.referrer] ?? 0) + 1 } return Object.entries(counts).sort((a, b) => b[1] - a[1]).slice(0, 10).map(([referrer, count]) => ({ referrer, count })) })(), last30DaysReal: buildLastNDaysStats(30).map(item => ({ day: item.day, hits: state.hitStats.byDayReal?.[item.day] ?? 0 })), recentVisits: recentVisitorRows, }, writeStatus: { hitStats: state.lastHitStatsWrite, visitorStats: state.lastVisitorStatsWrite, backups: state.lastBackupStatus, cachePurge: state.lastCachePurgeStatus, deployHook: state.lastDeployHookStatus, }, contactTotals: { totalSubmissions: state.contactSubmissions.length, totalQuestions: state.contactSubmissions.filter(entry => normalizeMessageType(entry?.messageType) === 'question').length, }, studyEnrollment: { totalUsers: state.studyUsers.length, enrolledUsers, totalEnrollments, enrollmentsByStudy, users, }, }) }) }