nmemmert
6acc3f1d04
Splits the analytics panel into Traffic / Engagement / System tabs so episode plays, engagement metrics, and contact/study data are no longer buried under the visitor log. Fixes a bug where topCountries, topStates, topCities, device breakdown, and referrers were computed from only the 100 most-recent visitor rows instead of all rows in the selected range — causing older geographic data (e.g. visits from India, Texas, Georgia) to silently disappear from the aggregate totals. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
407 lines
18 KiB
JavaScript
407 lines
18 KiB
JavaScript
import { createHash, randomUUID } from 'node:crypto'
|
|
import { requireAdminAuth, isValidAdminSession } from '../auth.js'
|
|
import { getClientIp, hasVisitorConsent, setConsentCookie, parseCookies } from '../helpers.js'
|
|
import { getStudyUserFromRequest } from '../study-helpers.js'
|
|
import {
|
|
VISITOR_COOKIE,
|
|
MAX_RECENT_VISITS,
|
|
} from '../config.js'
|
|
import { state } from '../state.js'
|
|
import { queueVisitorStatsWrite, queueHitStatsWrite, normalizeMessageType, recordEpisodePlay, recordAnalyticsEvent } from '../data.js'
|
|
import {
|
|
detectBot,
|
|
sanitizeUserAgent,
|
|
detectDevice,
|
|
sanitizeReferrer,
|
|
normalizeHitPath,
|
|
isPrivateOrLocalIp,
|
|
buildTopLocations,
|
|
buildLastNDaysStats,
|
|
shouldCountHit,
|
|
recordHit,
|
|
} from '../study-helpers.js'
|
|
import { getStudyCatalog } from '../study-helpers.js'
|
|
|
|
async function resolveGeo(ip) {
|
|
if (!ip || isPrivateOrLocalIp(ip)) {
|
|
return { country: 'Local/Unknown', state: 'Local/Unknown', county: 'Local/Unknown', city: 'Local/Unknown' }
|
|
}
|
|
|
|
const cached = state.visitorStats.geoCacheByIp[ip]
|
|
if (cached) return cached
|
|
|
|
const providers = [
|
|
async () => {
|
|
const controller = new AbortController()
|
|
const timeout = setTimeout(() => controller.abort(), 2500)
|
|
try {
|
|
const response = await fetch(
|
|
`http://ip-api.com/json/${encodeURIComponent(ip)}?fields=status,country,regionName,city,district`,
|
|
{ signal: controller.signal },
|
|
)
|
|
if (!response.ok) return null
|
|
const data = await response.json()
|
|
if (data?.status !== 'success') return null
|
|
return { country: data?.country || 'Unknown', state: data?.regionName || 'Unknown', county: data?.district || 'Unknown', city: data?.city || 'Unknown' }
|
|
} finally { clearTimeout(timeout) }
|
|
},
|
|
async () => {
|
|
const controller = new AbortController()
|
|
const timeout = setTimeout(() => controller.abort(), 2500)
|
|
try {
|
|
const response = await fetch(`https://ipwho.is/${encodeURIComponent(ip)}`, { signal: controller.signal })
|
|
if (!response.ok) return null
|
|
const data = await response.json()
|
|
if (!data?.success) return null
|
|
return { country: data?.country || 'Unknown', state: data?.region || 'Unknown', county: data?.region || 'Unknown', city: data?.city || 'Unknown' }
|
|
} finally { clearTimeout(timeout) }
|
|
},
|
|
]
|
|
|
|
for (const provider of providers) {
|
|
try {
|
|
const geo = await provider()
|
|
if (geo) {
|
|
state.visitorStats.geoCacheByIp[ip] = geo
|
|
queueVisitorStatsWrite()
|
|
return geo
|
|
}
|
|
} catch { /* Try next provider */ }
|
|
}
|
|
|
|
const fallback = { country: 'Unknown', state: 'Unknown', county: 'Unknown', city: 'Unknown' }
|
|
state.visitorStats.geoCacheByIp[ip] = fallback
|
|
queueVisitorStatsWrite()
|
|
return fallback
|
|
}
|
|
|
|
export async function recordVisitor(req, res, overridePath = null, overrideReferrer = null) {
|
|
const cookies = parseCookies(req.headers.cookie)
|
|
let visitorId = cookies[VISITOR_COOKIE]
|
|
if (!visitorId) {
|
|
visitorId = randomUUID()
|
|
res.append('Set-Cookie', `${VISITOR_COOKIE}=${encodeURIComponent(visitorId)}; Max-Age=31536000; Path=/; SameSite=Lax`)
|
|
}
|
|
|
|
const nowIso = new Date().toISOString()
|
|
const pathKey = overridePath ? normalizeHitPath(overridePath) : normalizeHitPath(req.path)
|
|
const referrer = overrideReferrer !== null ? sanitizeReferrer(overrideReferrer) : sanitizeReferrer(req.get('referer') || req.get('referrer') || '')
|
|
const ip = getClientIp(req)
|
|
const ua = sanitizeUserAgent(req.get('user-agent'))
|
|
const device = detectDevice(ua)
|
|
const studyUser = getStudyUserFromRequest(req)
|
|
const studyIdentity = studyUser
|
|
? { userId: studyUser.id, username: studyUser.username, displayName: studyUser.displayName ?? studyUser.username }
|
|
: null
|
|
|
|
const ipHash = createHash('sha256').update(ip).digest('hex')
|
|
const geo = await resolveGeo(ip)
|
|
|
|
const existingIdByIp = state.visitorStats.ipHashIndex[ipHash]
|
|
if (existingIdByIp && existingIdByIp !== visitorId) {
|
|
visitorId = existingIdByIp
|
|
res.append('Set-Cookie', `${VISITOR_COOKIE}=${encodeURIComponent(visitorId)}; Max-Age=31536000; Path=/; SameSite=Lax`)
|
|
}
|
|
|
|
const existingVisitor = state.visitorStats.visitors[visitorId]
|
|
const isReturning = Boolean(existingVisitor)
|
|
|
|
if (!existingVisitor) {
|
|
state.visitorStats.uniqueVisitors += 1
|
|
state.visitorStats.ipHashIndex[ipHash] = visitorId
|
|
} else {
|
|
state.visitorStats.returningVisits += 1
|
|
}
|
|
|
|
const nextVisitCount = (existingVisitor?.visitCount ?? 0) + 1
|
|
const userAgents = Array.from(new Set([...(existingVisitor?.userAgents ?? []), ua])).slice(0, 5)
|
|
const prevHistory = existingVisitor?.pageHistory ?? []
|
|
const pageHistory = [...prevHistory, { at: nowIso, path: pathKey, referrer }].slice(-100)
|
|
|
|
const knownIdentities = existingVisitor?.knownIdentities ?? []
|
|
if (studyIdentity && !knownIdentities.some(i => i.userId === studyIdentity.userId)) {
|
|
knownIdentities.push(studyIdentity)
|
|
}
|
|
|
|
state.visitorStats.visitors[visitorId] = {
|
|
visitorId, ip, ipHash,
|
|
firstSeenAt: existingVisitor?.firstSeenAt ?? nowIso,
|
|
lastSeenAt: nowIso,
|
|
visitCount: nextVisitCount,
|
|
lastPath: pathKey,
|
|
returningVisitor: isReturning,
|
|
location: geo,
|
|
userAgents,
|
|
device,
|
|
pageHistory,
|
|
...(knownIdentities.length > 0 ? { knownIdentities } : {}),
|
|
}
|
|
|
|
state.visitorStats.totalVisits += 1
|
|
state.visitorStats.firstVisitAt = state.visitorStats.firstVisitAt ?? nowIso
|
|
state.visitorStats.lastVisitAt = nowIso
|
|
state.visitorStats.recentVisits.unshift({
|
|
at: nowIso, visitorId, ip, path: pathKey, referrer, device,
|
|
country: geo.country, state: geo.state, county: geo.county, city: geo.city,
|
|
returningVisitor: isReturning, visitCount: nextVisitCount,
|
|
...(studyIdentity ? { studyUser: studyIdentity } : {}),
|
|
})
|
|
state.visitorStats.recentVisits = state.visitorStats.recentVisits.slice(0, MAX_RECENT_VISITS)
|
|
|
|
queueVisitorStatsWrite()
|
|
}
|
|
|
|
export function register(app) {
|
|
app.post('/api/analytics-consent', (req, res) => {
|
|
const consent = req.body?.consent === true
|
|
setConsentCookie(res, consent)
|
|
res.json({ ok: true, consent })
|
|
})
|
|
|
|
app.post('/api/analytics/play', (req, res) => {
|
|
if (isValidAdminSession(req)) { res.json({ ok: false, reason: 'admin' }); return }
|
|
const title = typeof req.body?.title === 'string' ? req.body.title.trim().slice(0, 200) : ''
|
|
if (!title) { res.status(400).json({ ok: false, reason: 'missing-title' }); return }
|
|
recordEpisodePlay(title)
|
|
res.json({ ok: true })
|
|
})
|
|
|
|
app.post('/api/analytics/event', (req, res) => {
|
|
if (isValidAdminSession(req)) { res.json({ ok: false, reason: 'admin' }); return }
|
|
if (!hasVisitorConsent(req)) { res.json({ ok: false, reason: 'no-consent' }); return }
|
|
const ua = req.get('user-agent') ?? ''
|
|
const { isBot } = detectBot(ua)
|
|
if (isBot) { res.json({ ok: false, reason: 'bot' }); return }
|
|
const type = typeof req.body?.type === 'string' ? req.body.type : ''
|
|
const ALLOWED_TYPES = ['scroll_depth', 'time_on_page', 'outbound_click', 'link_click', 'utm', 'search_query', 'not_found', 'audio_pause', 'audio_completion', 'audio_listen_time']
|
|
if (!ALLOWED_TYPES.includes(type)) { res.status(400).json({ ok: false, reason: 'invalid-type' }); return }
|
|
recordAnalyticsEvent(type, req.body)
|
|
res.json({ ok: true })
|
|
})
|
|
|
|
app.post('/api/analytics/pageview', async (req, res) => {
|
|
if (isValidAdminSession(req)) {
|
|
res.json({ ok: false, reason: 'admin' }); return
|
|
}
|
|
if (!hasVisitorConsent(req)) {
|
|
res.json({ ok: false, reason: 'no-consent' }); return
|
|
}
|
|
const ua = req.get('user-agent') ?? ''
|
|
const { isBot } = detectBot(ua)
|
|
if (isBot) {
|
|
res.json({ ok: false, reason: 'bot' }); return
|
|
}
|
|
const rawPath = typeof req.body?.path === 'string' ? req.body.path : '/'
|
|
const rawReferrer = typeof req.body?.referrer === 'string' ? req.body.referrer : ''
|
|
recordHit(rawPath, false)
|
|
queueHitStatsWrite()
|
|
await recordVisitor(req, res, rawPath, rawReferrer)
|
|
res.json({ ok: true })
|
|
})
|
|
|
|
app.get('/api/admin-stats', requireAdminAuth, (req, res) => {
|
|
// Time-range filter: 7d | 30d | 90d (default: all-time for aggregate, 30d for charts)
|
|
const rangeParam = req.query.range
|
|
const rangeDays = rangeParam === '7d' ? 7 : rangeParam === '90d' ? 90 : 30
|
|
const rangeLabel = rangeParam === '7d' ? '7d' : rangeParam === '90d' ? '90d' : '30d'
|
|
|
|
// Compute a cutoff date string (YYYY-MM-DD) for filtering daily buckets
|
|
const cutoffDate = (() => {
|
|
const d = new Date()
|
|
d.setDate(d.getDate() - (rangeDays - 1))
|
|
return d.toISOString().slice(0, 10)
|
|
})()
|
|
|
|
// Filter byDayReal/byDayBot keys to only those within the range
|
|
const filteredDayKeys = Object.keys(state.hitStats.byDayReal ?? {}).filter(day => day >= cutoffDate)
|
|
|
|
// Aggregate hits for the range
|
|
const rangeRealHits = filteredDayKeys.reduce((sum, day) => sum + (state.hitStats.byDayReal?.[day] ?? 0), 0)
|
|
const rangeBotHits = filteredDayKeys.reduce((sum, day) => sum + (state.hitStats.byDayBot?.[day] ?? 0), 0)
|
|
const rangeTotalHits = rangeRealHits + rangeBotHits
|
|
|
|
// Filter per-path stats by range — approximate using recent visitor rows scoped to range
|
|
const rangeVisitorRows = state.visitorStats.recentVisits.filter(row => {
|
|
if (!row.visitedAt) return true // include if no timestamp
|
|
return row.visitedAt >= cutoffDate
|
|
})
|
|
|
|
const rangePathCountsReal = {}
|
|
const rangePathCountsBot = {}
|
|
for (const row of rangeVisitorRows) {
|
|
const p = row.path ?? '/'
|
|
if (row.isBot) rangePathCountsBot[p] = (rangePathCountsBot[p] ?? 0) + 1
|
|
else rangePathCountsReal[p] = (rangePathCountsReal[p] ?? 0) + 1
|
|
}
|
|
const topPathsReal = Object.entries(rangePathCountsReal).sort((a, b) => b[1] - a[1]).slice(0, 10).map(([path, hits]) => ({ path, hits }))
|
|
const topPathsBot = Object.entries(rangePathCountsBot).sort((a, b) => b[1] - a[1]).slice(0, 10).map(([path, hits]) => ({ path, hits }))
|
|
const topPaths = [...topPathsReal, ...topPathsBot].reduce((acc, { path, hits }) => {
|
|
const existing = acc.find(x => x.path === path)
|
|
if (existing) existing.hits += hits; else acc.push({ path, hits })
|
|
return acc
|
|
}, []).sort((a, b) => b.hits - a.hits).slice(0, 10)
|
|
|
|
const last7Days = buildLastNDaysStats(7)
|
|
const last7DaysReal = last7Days.map(item => ({ day: item.day, hits: state.hitStats.byDayReal?.[item.day] ?? 0 }))
|
|
const last7DaysBot = last7Days.map(item => ({ day: item.day, hits: state.hitStats.byDayBot?.[item.day] ?? 0 }))
|
|
|
|
const last30Days = buildLastNDaysStats(30)
|
|
const last30DaysTotal = last30Days.reduce((sum, item) => sum + item.hits, 0)
|
|
const last30DaysRealTotal = last30Days.reduce((sum, item) => sum + (state.hitStats.byDayReal?.[item.day] ?? 0), 0)
|
|
const last30DaysBotTotal = last30Days.reduce((sum, item) => sum + (state.hitStats.byDayBot?.[item.day] ?? 0), 0)
|
|
|
|
const botReasons = Object.entries(state.hitStats.botReasons ?? {})
|
|
.sort((a, b) => b[1] - a[1]).slice(0, 10).map(([reason, count]) => ({ reason, count }))
|
|
|
|
// All range rows used for geographic/device/referrer aggregates — no cap so no visits are invisible
|
|
const allRangeRows = rangeVisitorRows
|
|
|
|
// Capped to 100 for the visitor log display (with page history hydrated)
|
|
const recentVisitorRows = rangeVisitorRows.slice(0, 100).map(row => {
|
|
const fullVisitor = state.visitorStats.visitors[row.visitorId]
|
|
return { ...row, pageHistory: fullVisitor?.pageHistory ?? [] }
|
|
})
|
|
|
|
const enrollmentCountsBySlug = {}
|
|
for (const user of state.studyUsers) {
|
|
const userEnrollments = Array.isArray(user?.enrolledStudySlugs) ? user.enrolledStudySlugs : []
|
|
for (const studySlug of userEnrollments) {
|
|
enrollmentCountsBySlug[studySlug] = (enrollmentCountsBySlug[studySlug] ?? 0) + 1
|
|
}
|
|
}
|
|
const enrollmentsByStudy = getStudyCatalog()
|
|
.map(study => ({ slug: study.slug, title: study.title, count: enrollmentCountsBySlug[study.slug] ?? 0 }))
|
|
.sort((a, b) => b.count - a.count)
|
|
|
|
const studyCatalogBySlug = new Map(getStudyCatalog().map(study => [study.slug, study]))
|
|
const users = state.studyUsers
|
|
.map(user => {
|
|
const enrolledStudySlugs = Array.isArray(user?.enrolledStudySlugs) ? user.enrolledStudySlugs : []
|
|
const enrolledStudies = enrolledStudySlugs.map(slug => {
|
|
const study = studyCatalogBySlug.get(slug)
|
|
return study ? { slug: study.slug, title: study.title } : null
|
|
}).filter(Boolean)
|
|
return { id: user.id, username: user.username, displayName: user.displayName ?? '', enrolledStudies }
|
|
})
|
|
.sort((a, b) => {
|
|
if (b.enrolledStudies.length !== a.enrolledStudies.length) return b.enrolledStudies.length - a.enrolledStudies.length
|
|
return a.username.localeCompare(b.username)
|
|
})
|
|
|
|
const enrolledUsers = state.studyUsers.filter(user => (user.enrolledStudySlugs?.length ?? 0) > 0).length
|
|
const totalEnrollments = Object.values(enrollmentCountsBySlug).reduce((sum, count) => sum + count, 0)
|
|
|
|
// Enrollment funnel: signups → first study visit → first section completed
|
|
const funnelSignups = state.studyUsers.length
|
|
const funnelFirstVisit = state.studyUsers.filter(u => u.firstVisitAt).length
|
|
const funnelFirstCompletion = state.studyUsers.filter(u => u.firstCompletionAt).length
|
|
|
|
res.json({
|
|
totalHits: state.hitStats.totalHits,
|
|
realHits: rangeRealHits,
|
|
botHits: rangeBotHits,
|
|
rangeTotalHits,
|
|
rangeLabel,
|
|
rangeDays,
|
|
// All-time totals for reference
|
|
allTimeRealHits: state.hitStats.realHits ?? 0,
|
|
allTimeBotHits: state.hitStats.botHits ?? 0,
|
|
allTimeTotalHits: state.hitStats.totalHits,
|
|
firstHitAt: state.hitStats.firstHitAt,
|
|
lastHitAt: state.hitStats.lastHitAt,
|
|
topPaths, topPathsReal, topPathsBot,
|
|
last7Days, last7DaysReal, last7DaysBot,
|
|
last30DaysTotal, last30DaysRealTotal, last30DaysBotTotal,
|
|
botReasons,
|
|
visitors: {
|
|
totalVisits: state.visitorStats.totalVisits,
|
|
uniqueVisitors: state.visitorStats.uniqueVisitors,
|
|
returningVisits: state.visitorStats.returningVisits,
|
|
firstVisitAt: state.visitorStats.firstVisitAt,
|
|
lastVisitAt: state.visitorStats.lastVisitAt,
|
|
topCountries: buildTopLocations(allRangeRows, 'country'),
|
|
topStates: buildTopLocations(allRangeRows, 'state'),
|
|
topCounties: buildTopLocations(allRangeRows, 'county'),
|
|
topCities: buildTopLocations(allRangeRows, 'city'),
|
|
deviceBreakdown: (() => {
|
|
const counts = { mobile: 0, desktop: 0, tablet: 0, unknown: 0 }
|
|
for (const row of allRangeRows) {
|
|
const d = row.device ?? 'unknown'
|
|
counts[d] = (counts[d] ?? 0) + 1
|
|
}
|
|
return counts
|
|
})(),
|
|
topReferrers: (() => {
|
|
const counts = {}
|
|
for (const row of allRangeRows) {
|
|
if (!row.referrer) continue
|
|
counts[row.referrer] = (counts[row.referrer] ?? 0) + 1
|
|
}
|
|
return Object.entries(counts).sort((a, b) => b[1] - a[1]).slice(0, 10).map(([referrer, count]) => ({ referrer, count }))
|
|
})(),
|
|
last30DaysReal: buildLastNDaysStats(30).map(item => ({ day: item.day, hits: state.hitStats.byDayReal?.[item.day] ?? 0 })),
|
|
recentVisits: recentVisitorRows,
|
|
},
|
|
writeStatus: {
|
|
hitStats: state.lastHitStatsWrite,
|
|
visitorStats: state.lastVisitorStatsWrite,
|
|
backups: state.lastBackupStatus,
|
|
cachePurge: state.lastCachePurgeStatus,
|
|
deployHook: state.lastDeployHookStatus,
|
|
},
|
|
contactTotals: {
|
|
totalSubmissions: state.contactSubmissions.length,
|
|
totalQuestions: state.contactSubmissions.filter(entry => normalizeMessageType(entry?.messageType) === 'question').length,
|
|
},
|
|
studyEnrollment: {
|
|
totalUsers: state.studyUsers.length,
|
|
enrolledUsers,
|
|
totalEnrollments,
|
|
enrollmentsByStudy,
|
|
users,
|
|
funnel: { signups: funnelSignups, firstVisit: funnelFirstVisit, firstCompletion: funnelFirstCompletion },
|
|
},
|
|
episodePlays: Object.entries(state.episodePlays)
|
|
.map(([title, data]) => ({
|
|
title,
|
|
total: data.total ?? 0,
|
|
byDay: data.byDay ?? {},
|
|
last30Days: buildLastNDaysStats(30).map(item => ({ day: item.day, plays: data.byDay?.[item.day] ?? 0 })),
|
|
}))
|
|
.sort((a, b) => b.total - a.total),
|
|
engagement: {
|
|
scrollDepth: Object.entries(state.analyticsEvents.scrollDepth ?? {})
|
|
.map(([path, marks]) => ({ path, ...marks }))
|
|
.sort((a, b) => (b[90] ?? 0) - (a[90] ?? 0))
|
|
.slice(0, 20),
|
|
timeOnPage: Object.entries(state.analyticsEvents.timeOnPage ?? {})
|
|
.map(([path, { totalSeconds, count }]) => ({ path, avgSeconds: count > 0 ? Math.round(totalSeconds / count) : 0, count }))
|
|
.sort((a, b) => b.avgSeconds - a.avgSeconds)
|
|
.slice(0, 20),
|
|
topOutboundClicks: Object.entries(state.analyticsEvents.outboundClicks ?? {})
|
|
.map(([url, count]) => ({ url, count }))
|
|
.sort((a, b) => b.count - a.count)
|
|
.slice(0, 20),
|
|
topLinkClicks: Object.entries(state.analyticsEvents.linkClicks ?? {})
|
|
.map(([url, { count, internal }]) => ({ url, count, internal }))
|
|
.sort((a, b) => b.count - a.count)
|
|
.slice(0, 50),
|
|
topUTMSources: Object.entries(state.analyticsEvents.utmSources ?? {})
|
|
.map(([source, count]) => ({ source, count }))
|
|
.sort((a, b) => b.count - a.count),
|
|
topSearchQueries: Object.entries(state.analyticsEvents.searchQueries ?? {})
|
|
.map(([query, count]) => ({ query, count }))
|
|
.sort((a, b) => b.count - a.count)
|
|
.slice(0, 30),
|
|
top404s: Object.entries(state.analyticsEvents.notFound ?? {})
|
|
.map(([path, count]) => ({ path, count }))
|
|
.sort((a, b) => b.count - a.count)
|
|
.slice(0, 20),
|
|
audioEvents: Object.entries(state.analyticsEvents.audioEvents ?? {})
|
|
.map(([title, data]) => ({ title, ...data }))
|
|
.sort((a, b) => b.completions - a.completions),
|
|
},
|
|
})
|
|
})
|
|
}
|