nate/Siteforge
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Analytics overhaul: privacy fixes, All Time range, readability improvements; v2.0.29
Publish Container / docker (push) Successful in 14m17s
Details

Browse Source 
- Remove raw IP storage; store only SHA-256 hash prefix in visitor records
- Non-blocking geo lookup: recordVisitor now synchronous, geo resolved in background
- UTM capture race condition fixed via sessionStorage pre-consent capture
- Add consent + bot checks to play endpoint (was missing)
- Add rate limiter to pageview endpoint (60 req/min per IP)
- Add 'All time' range option alongside 7d/30d/90d with trendData aggregation
- AnalyticsPanel: tabbed layout (Traffic/Engagement/System), dynamic chart titles,
  range buttons with active state, visitor log uses ipHash instead of raw IP,
  trendData drives main chart with weekly bucketing for all-time view
- Re-add prop-types dependency (required by react-simple-maps bundler)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
nmemmertandClaude Sonnet 4.6 committed 2026-09-17 16:27:53 -04:00
1 parent da5c1c5078
commit 4002f1ee93
6 files changed
+333 -231
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/routes/analytics.js
+90 -44
View File
@@ -22,12 +22,12 @@ import {
getStudyCatalog,
} from '../study-helpers.js'
async function resolveGeo(ip) {
async function resolveGeo(ip, ipHash) {
if (!ip || isPrivateOrLocalIp(ip)) {
return { country: 'Local/Unknown', state: 'Local/Unknown', county: 'Local/Unknown', city: 'Local/Unknown' }
}
const cached = state.visitorStats.geoCacheByIp[ip]
const cached = state.visitorStats.geoCacheByIp[ipHash]
if (cached) return cached
const providers = [
@@ -62,7 +62,7 @@ async function resolveGeo(ip) {
try {
const geo = await provider()
if (geo) {
state.visitorStats.geoCacheByIp[ip] = geo
state.visitorStats.geoCacheByIp[ipHash] = geo
queueVisitorStatsWrite()
return geo
}
@@ -70,17 +70,33 @@ async function resolveGeo(ip) {
}
const fallback = { country: 'Unknown', state: 'Unknown', county: 'Unknown', city: 'Unknown' }
state.visitorStats.geoCacheByIp[ip] = fallback
state.visitorStats.geoCacheByIp[ipHash] = fallback
queueVisitorStatsWrite()
return fallback
}
export async function recordVisitor(req, res, overridePath = null, overrideReferrer = null) {
// Simple in-memory rate limiter for pageview endpoint
const pageviewHits = new Map()
function pageviewAllowed(ip) {
const now = Date.now()
const windowMs = 60 * 1000
const entry = pageviewHits.get(ip) ?? { count: 0, start: now }
if (now - entry.start > windowMs) { entry.count = 0; entry.start = now }
entry.count += 1
pageviewHits.set(ip, entry)
if (pageviewHits.size > 5000) {
const cutoff = now - windowMs
for (const [k, v] of pageviewHits) { if (v.start < cutoff) pageviewHits.delete(k) }
}
return entry.count <= 60
}
export function recordVisitor(req, res, overridePath = null, overrideReferrer = null) {
const cookies = parseCookies(req.headers.cookie)
let visitorId = cookies[VISITOR_COOKIE]
if (!visitorId) {
visitorId = randomUUID()
res.append('Set-Cookie', `${VISITOR_COOKIE}=${encodeURIComponent(visitorId)}; Max-Age=31536000; Path=/; SameSite=Lax`)
res.append('Set-Cookie', `${VISITOR_COOKIE}=${encodeURIComponent(visitorId)}; Max-Age=31536000; Path=/; SameSite=Lax; HttpOnly`)
}
const nowIso = new Date().toISOString()
@@ -95,12 +111,11 @@ export async function recordVisitor(req, res, overridePath = null, overrideRefer
: null
const ipHash = createHash('sha256').update(ip).digest('hex')
const geo = await resolveGeo(ip)
const existingIdByIp = state.visitorStats.ipHashIndex[ipHash]
if (existingIdByIp && existingIdByIp !== visitorId) {
visitorId = existingIdByIp
res.append('Set-Cookie', `${VISITOR_COOKIE}=${encodeURIComponent(visitorId)}; Max-Age=31536000; Path=/; SameSite=Lax`)
res.append('Set-Cookie', `${VISITOR_COOKIE}=${encodeURIComponent(visitorId)}; Max-Age=31536000; Path=/; SameSite=Lax; HttpOnly`)
}
const existingVisitor = state.visitorStats.visitors[visitorId]
@@ -123,8 +138,12 @@ export async function recordVisitor(req, res, overridePath = null, overrideRefer
knownIdentities.push(studyIdentity)
}
// Use cached geo immediately; will be resolved async if not yet cached
const cachedGeo = state.visitorStats.geoCacheByIp[ipHash]
const geo = cachedGeo ?? { country: 'Unknown', state: 'Unknown', county: 'Unknown', city: 'Unknown' }
state.visitorStats.visitors[visitorId] = {
visitorId, ip, ipHash,
visitorId, ipHash,
firstSeenAt: existingVisitor?.firstSeenAt ?? nowIso,
lastSeenAt: nowIso,
visitCount: nextVisitCount,
@@ -141,7 +160,8 @@ export async function recordVisitor(req, res, overridePath = null, overrideRefer
state.visitorStats.firstVisitAt = state.visitorStats.firstVisitAt ?? nowIso
state.visitorStats.lastVisitAt = nowIso
state.visitorStats.recentVisits.unshift({
at: nowIso, visitorId, ip, path: pathKey, referrer, device,
at: nowIso, visitorId, ipHash,
path: pathKey, referrer, device,
country: geo.country, state: geo.state, county: geo.county, city: geo.city,
returningVisitor: isReturning, visitCount: nextVisitCount,
...(studyIdentity ? { studyUser: studyIdentity } : {}),
@@ -149,6 +169,23 @@ export async function recordVisitor(req, res, overridePath = null, overrideRefer
state.visitorStats.recentVisits = state.visitorStats.recentVisits.slice(0, MAX_RECENT_VISITS)
queueVisitorStatsWrite()
// Resolve geo in background if not already cached
if (!cachedGeo) {
resolveGeo(ip, ipHash).then(resolvedGeo => {
if (state.visitorStats.visitors[visitorId]) {
state.visitorStats.visitors[visitorId].location = resolvedGeo
}
const entry = state.visitorStats.recentVisits.find(v => v.visitorId === visitorId && v.at === nowIso)
if (entry) {
entry.country = resolvedGeo.country
entry.state = resolvedGeo.state
entry.county = resolvedGeo.county
entry.city = resolvedGeo.city
}
queueVisitorStatsWrite()
}).catch(() => {})
}
}
export function register(app) {
@@ -160,6 +197,10 @@ export function register(app) {
app.post('/api/analytics/play', (req, res) => {
if (isValidAdminSession(req)) { res.json({ ok: false, reason: 'admin' }); return }
if (!hasVisitorConsent(req)) { res.json({ ok: false, reason: 'no-consent' }); return }
const ua = req.get('user-agent') ?? ''
const { isBot } = detectBot(ua)
if (isBot) { res.json({ ok: false, reason: 'bot' }); return }
const title = typeof req.body?.title === 'string' ? req.body.title.trim().slice(0, 200) : ''
if (!title) { res.status(400).json({ ok: false, reason: 'missing-title' }); return }
recordEpisodePlay(title)
@@ -179,56 +220,63 @@ export function register(app) {
res.json({ ok: true })
})
app.post('/api/analytics/pageview', async (req, res) => {
if (isValidAdminSession(req)) {
res.json({ ok: false, reason: 'admin' }); return
}
if (!hasVisitorConsent(req)) {
res.json({ ok: false, reason: 'no-consent' }); return
}
app.post('/api/analytics/pageview', (req, res) => {
if (isValidAdminSession(req)) { res.json({ ok: false, reason: 'admin' }); return }
if (!hasVisitorConsent(req)) { res.json({ ok: false, reason: 'no-consent' }); return }
const ua = req.get('user-agent') ?? ''
const { isBot } = detectBot(ua)
if (isBot) {
res.json({ ok: false, reason: 'bot' }); return
}
if (isBot) { res.json({ ok: false, reason: 'bot' }); return }
const ip = getClientIp(req)
if (!pageviewAllowed(ip)) { res.json({ ok: false, reason: 'rate-limited' }); return }
const rawPath = typeof req.body?.path === 'string' ? req.body.path : '/'
const rawReferrer = typeof req.body?.referrer === 'string' ? req.body.referrer : ''
recordHit(rawPath, false)
queueHitStatsWrite()
await recordVisitor(req, res, rawPath, rawReferrer)
recordVisitor(req, res, rawPath, rawReferrer)
res.json({ ok: true })
})
app.get('/api/admin-stats', requireAdminAuth, (req, res) => {
// Time-range filter: 7d | 30d | 90d (default: all-time for aggregate, 30d for charts)
const rangeParam = req.query.range
const rangeDays = rangeParam === '7d' ? 7 : rangeParam === '90d' ? 90 : 30
const rangeLabel = rangeParam === '7d' ? '7d' : rangeParam === '90d' ? '90d' : '30d'
const isAllTime = rangeParam === 'all'
const rangeDays = rangeParam === '7d' ? 7 : rangeParam === '90d' ? 90 : isAllTime ? null : 30
const rangeLabel = rangeParam === '7d' ? 'Last 7 days' : rangeParam === '90d' ? 'Last 90 days' : isAllTime ? 'All time' : 'Last 30 days'
// Compute a cutoff date string (YYYY-MM-DD) for filtering daily buckets
const cutoffDate = (() => {
// Cutoff date (null = all-time)
const cutoffDate = isAllTime ? null : (() => {
const d = new Date()
d.setDate(d.getDate() - (rangeDays - 1))
d.setDate(d.getDate() - ((rangeDays ?? 30) - 1))
return d.toISOString().slice(0, 10)
})()
// Filter byDayReal/byDayBot keys to only those within the range
// All known day keys, sorted
const allDayKeys = new Set([
...Object.keys(state.hitStats.byDayReal ?? {}),
...Object.keys(state.hitStats.byDayBot ?? {}),
])
const filteredDayKeys = [...allDayKeys].filter(day => day >= cutoffDate)
const filteredDayKeys = isAllTime
? [...allDayKeys].sort()
: [...allDayKeys].filter(day => day >= cutoffDate).sort()
// Aggregate hits for the range
const rangeRealHits = filteredDayKeys.reduce((sum, day) => sum + (state.hitStats.byDayReal?.[day] ?? 0), 0)
const rangeBotHits = filteredDayKeys.reduce((sum, day) => sum + (state.hitStats.byDayBot?.[day] ?? 0), 0)
const rangeTotalHits = rangeRealHits + rangeBotHits
// Filter per-path stats by range — approximate using recent visitor rows scoped to range
const rangeVisitorRows = state.visitorStats.recentVisits.filter(row => {
if (!row.at) return false // exclude rows with no timestamp from range views
return row.at >= cutoffDate
})
// Trend chart data — one point per day for the selected range
const trendData = filteredDayKeys.map(day => ({
day,
real: state.hitStats.byDayReal?.[day] ?? 0,
bot: state.hitStats.byDayBot?.[day] ?? 0,
}))
// Visitor rows for range
const rangeVisitorRows = isAllTime
? state.visitorStats.recentVisits
: state.visitorStats.recentVisits.filter(row => {
if (!row.at) return false
return row.at >= cutoffDate
})
const rangePathCountsReal = {}
const rangePathCountsBot = {}
@@ -257,9 +305,6 @@ export function register(app) {
const botReasons = Object.entries(state.hitStats.botReasons ?? {})
.sort((a, b) => b[1] - a[1]).slice(0, 10).map(([reason, count]) => ({ reason, count }))
// All range rows used for geographic/device/referrer aggregates — no cap so no visits are invisible
const allRangeRows = rangeVisitorRows
// Capped to 100 for the visitor log display (with page history hydrated)
const recentVisitorRows = rangeVisitorRows.slice(0, 100).map(row => {
const fullVisitor = state.visitorStats.visitors[row.visitorId]
@@ -295,7 +340,6 @@ export function register(app) {
const enrolledUsers = state.studyUsers.filter(user => (user.enrolledStudySlugs?.length ?? 0) > 0).length
const totalEnrollments = Object.values(enrollmentCountsBySlug).reduce((sum, count) => sum + count, 0)
// Enrollment funnel: signups → first study visit → first section completed
const funnelSignups = state.studyUsers.length
const funnelFirstVisit = state.studyUsers.filter(u => u.firstVisitAt).length
const funnelFirstCompletion = state.studyUsers.filter(u => u.firstCompletionAt).length
@@ -307,6 +351,8 @@ export function register(app) {
rangeTotalHits,
rangeLabel,
rangeDays,
isAllTime,
trendData,
// All-time totals for reference
allTimeRealHits: state.hitStats.realHits ?? 0,
allTimeBotHits: state.hitStats.botHits ?? 0,
@@ -323,13 +369,13 @@ export function register(app) {
returningVisits: state.visitorStats.returningVisits,
firstVisitAt: state.visitorStats.firstVisitAt,
lastVisitAt: state.visitorStats.lastVisitAt,
topCountries: buildTopLocations(allRangeRows, 'country'),
topStates: buildTopLocations(allRangeRows, 'state'),
topCounties: buildTopLocations(allRangeRows, 'county'),
topCities: buildTopLocations(allRangeRows, 'city'),
topCountries: buildTopLocations(rangeVisitorRows, 'country'),
topStates: buildTopLocations(rangeVisitorRows, 'state'),
topCounties: buildTopLocations(rangeVisitorRows, 'county'),
topCities: buildTopLocations(rangeVisitorRows, 'city'),
deviceBreakdown: (() => {
const counts = { mobile: 0, desktop: 0, tablet: 0, unknown: 0 }
for (const row of allRangeRows) {
for (const row of rangeVisitorRows) {
const d = row.device ?? 'unknown'
counts[d] = (counts[d] ?? 0) + 1
}
@@ -337,7 +383,7 @@ export function register(app) {
})(),
topReferrers: (() => {
const counts = {}
for (const row of allRangeRows) {
for (const row of rangeVisitorRows) {
if (!row.referrer) continue
counts[row.referrer] = (counts[row.referrer] ?? 0) + 1
}