From e561bfa000dfa935537a0c43d10cbba17919bc06 Mon Sep 17 00:00:00 2001 From: nmemmert Date: Wed, 3 Jun 2026 11:35:45 -0400 Subject: [PATCH] analytics fixs --- server.js | 108 +++++++++++++++- src/AdminPage.tsx | 6 + src/App.css | 55 ++++++++ src/App.tsx | 15 +++ src/components/AnalyticsPanel.tsx | 208 +++++++++++++++++++++++------- 5 files changed, 339 insertions(+), 53 deletions(-) diff --git a/server.js b/server.js index d2f2251..b8fadc4 100644 --- a/server.js +++ b/server.js @@ -612,6 +612,7 @@ const EMPTY_VISITOR_STATS = { firstVisitAt: null, lastVisitAt: null, visitors: {}, + ipHashIndex: {}, // ipHash → visitorId — prevents same IP counting as multiple unique visitors recentVisits: [], geoCacheByIp: {}, } @@ -1343,6 +1344,24 @@ function sanitizeUserAgent(userAgent) { return userAgent.trim().slice(0, 300) || 'unknown' } +function detectDevice(userAgent) { + if (!userAgent || typeof userAgent !== 'string') return 'unknown' + const ua = userAgent.toLowerCase() + if (/tablet|ipad|playbook|silk|(android(?!.*mobile))/.test(ua)) return 'tablet' + if (/mobile|iphone|ipod|android|blackberry|opera mini|opera mobi|iemobile|windows phone|palm|smartphone/.test(ua)) return 'mobile' + return 'desktop' +} + +function sanitizeReferrer(referrer) { + if (!referrer || typeof referrer !== 'string') return '' + try { + const parsed = new URL(referrer.trim()) + return `${parsed.hostname}${parsed.pathname}`.slice(0, 200) + } catch { + return '' + } +} + function detectBot(userAgent, pathInfo = {}) { if (!userAgent || typeof userAgent !== 'string') { return { isBot: true, reason: 'missing-user-agent' } @@ -1472,7 +1491,7 @@ async function resolveGeo(ip) { return fallback } -async function recordVisitor(req, res) { +async function recordVisitor(req, res, overridePath = null, overrideReferrer = null) { const cookies = parseCookies(req.headers.cookie) let visitorId = cookies[VISITOR_COOKIE] if (!visitorId) { @@ -1481,24 +1500,42 @@ async function recordVisitor(req, res) { } const nowIso = new Date().toISOString() - const pathKey = normalizeHitPath(req.path) + const pathKey = overridePath ? normalizeHitPath(overridePath) : normalizeHitPath(req.path) + const referrer = overrideReferrer !== null ? sanitizeReferrer(overrideReferrer) : sanitizeReferrer(req.get('referer') || req.get('referrer') || '') const ip = getClientIp(req) const ua = sanitizeUserAgent(req.get('user-agent')) + const device = detectDevice(ua) + + const ipHash = createHash('sha256').update(ip).digest('hex') + const geo = await resolveGeo(ip) + + // Resolve canonical visitorId by IP hash — if this IP was seen before under a + // different cookie (e.g. cleared cookies), reuse the existing record so the + // same person is never counted as a second unique visitor. + const existingIdByIp = visitorStats.ipHashIndex[ipHash] + if (existingIdByIp && existingIdByIp !== visitorId) { + // Reuse the existing record for this IP; overwrite cookie with canonical ID + visitorId = existingIdByIp + res.append('Set-Cookie', `${VISITOR_COOKIE}=${encodeURIComponent(visitorId)}; Max-Age=31536000; Path=/; SameSite=Lax`) + } const existingVisitor = visitorStats.visitors[visitorId] const isReturning = Boolean(existingVisitor) - const geo = await resolveGeo(ip) if (!existingVisitor) { visitorStats.uniqueVisitors += 1 + visitorStats.ipHashIndex[ipHash] = visitorId } else { visitorStats.returningVisits += 1 } - const ipHash = createHash('sha256').update(ip).digest('hex') const nextVisitCount = (existingVisitor?.visitCount ?? 0) + 1 const userAgents = Array.from(new Set([...(existingVisitor?.userAgents ?? []), ua])).slice(0, 5) + // Append to page history, keeping last 100 entries per visitor + const prevHistory = existingVisitor?.pageHistory ?? [] + const pageHistory = [...prevHistory, { at: nowIso, path: pathKey, referrer }].slice(-100) + visitorStats.visitors[visitorId] = { visitorId, ip, @@ -1510,6 +1547,8 @@ async function recordVisitor(req, res) { returningVisitor: isReturning, location: geo, userAgents, + device, + pageHistory, } visitorStats.totalVisits += 1 @@ -1520,6 +1559,8 @@ async function recordVisitor(req, res) { visitorId, ip, path: pathKey, + referrer, + device, country: geo.country, state: geo.state, county: geo.county, @@ -1536,13 +1577,26 @@ function loadVisitorStatsFromDisk() { return readFile(VISITOR_STATS_FILE, 'utf8') .then(raw => { const parsed = JSON.parse(raw) + const loadedVisitors = parsed?.visitors && typeof parsed.visitors === 'object' ? parsed.visitors : {} + + // Rebuild ipHashIndex from saved visitors if not persisted (handles upgrades from old data) + let ipHashIndex = parsed?.ipHashIndex && typeof parsed.ipHashIndex === 'object' ? parsed.ipHashIndex : {} + if (Object.keys(ipHashIndex).length === 0 && Object.keys(loadedVisitors).length > 0) { + for (const [vid, visitor] of Object.entries(loadedVisitors)) { + if (visitor?.ipHash && typeof visitor.ipHash === 'string') { + ipHashIndex[visitor.ipHash] = vid + } + } + } + visitorStats = { totalVisits: Number(parsed?.totalVisits) || 0, uniqueVisitors: Number(parsed?.uniqueVisitors) || 0, returningVisits: Number(parsed?.returningVisits) || 0, firstVisitAt: typeof parsed?.firstVisitAt === 'string' ? parsed.firstVisitAt : null, lastVisitAt: typeof parsed?.lastVisitAt === 'string' ? parsed.lastVisitAt : null, - visitors: parsed?.visitors && typeof parsed.visitors === 'object' ? parsed.visitors : {}, + visitors: loadedVisitors, + ipHashIndex, recentVisits: Array.isArray(parsed?.recentVisits) ? parsed.recentVisits.slice(0, MAX_RECENT_VISITS) : [], geoCacheByIp: parsed?.geoCacheByIp && typeof parsed.geoCacheByIp === 'object' ? parsed.geoCacheByIp : {}, } @@ -4023,6 +4077,25 @@ app.post('/api/analytics-consent', (req, res) => { res.json({ ok: true, consent }) }) +// Client-side SPA pageview tracking (fires on every React Router navigation) +app.post('/api/analytics/pageview', async (req, res) => { + if (!hasVisitorConsent(req)) { + res.json({ ok: false, reason: 'no-consent' }) + return + } + const ua = req.get('user-agent') ?? '' + const { isBot } = detectBot(ua) + if (isBot) { + res.json({ ok: false, reason: 'bot' }) + return + } + const rawPath = typeof req.body?.path === 'string' ? req.body.path : '/' + const rawReferrer = typeof req.body?.referrer === 'string' ? req.body.referrer : '' + recordHit(rawPath, false) + await recordVisitor(req, res, rawPath, rawReferrer) + res.json({ ok: true }) +}) + app.get('/api/admin-stats', requireAdminAuth, (_req, res) => { const topPaths = Object.entries(hitStats.byPath) .sort((a, b) => b[1] - a[1]) @@ -4059,7 +4132,10 @@ app.get('/api/admin-stats', requireAdminAuth, (_req, res) => { .slice(0, 10) .map(([reason, count]) => ({ reason, count })) - const recentVisitorRows = visitorStats.recentVisits.slice(0, 100) + const recentVisitorRows = visitorStats.recentVisits.slice(0, 100).map(row => { + const fullVisitor = visitorStats.visitors[row.visitorId] + return { ...row, pageHistory: fullVisitor?.pageHistory ?? [] } + }) const enrollmentCountsBySlug = {} for (const user of studyUsers) { const userEnrollments = Array.isArray(user?.enrolledStudySlugs) ? user.enrolledStudySlugs : [] @@ -4126,6 +4202,26 @@ app.get('/api/admin-stats', requireAdminAuth, (_req, res) => { topStates: buildTopLocations(recentVisitorRows, 'state'), topCounties: buildTopLocations(recentVisitorRows, 'county'), topCities: buildTopLocations(recentVisitorRows, 'city'), + deviceBreakdown: (() => { + const counts = { mobile: 0, desktop: 0, tablet: 0, unknown: 0 } + for (const row of recentVisitorRows) { + const d = row.device ?? 'unknown' + counts[d] = (counts[d] ?? 0) + 1 + } + return counts + })(), + topReferrers: (() => { + const counts = {} + for (const row of recentVisitorRows) { + if (!row.referrer) continue + counts[row.referrer] = (counts[row.referrer] ?? 0) + 1 + } + return Object.entries(counts) + .sort((a, b) => b[1] - a[1]) + .slice(0, 10) + .map(([referrer, count]) => ({ referrer, count })) + })(), + last30DaysReal: buildLastNDaysStats(30).map(item => ({ day: item.day, hits: hitStats.byDayReal?.[item.day] ?? 0 })), recentVisits: recentVisitorRows, }, writeStatus: { diff --git a/src/AdminPage.tsx b/src/AdminPage.tsx index 940dad8..9a60460 100644 --- a/src/AdminPage.tsx +++ b/src/AdminPage.tsx @@ -162,17 +162,23 @@ export interface AdminStats { topStates: Array<{ name: string; hits: number }> topCounties: Array<{ name: string; hits: number }> topCities: Array<{ name: string; hits: number }> + deviceBreakdown: { mobile: number; desktop: number; tablet: number; unknown: number } + topReferrers: Array<{ referrer: string; count: number }> + last30DaysReal: Array<{ day: string; hits: number }> recentVisits: Array<{ at: string visitorId: string ip: string path: string + referrer?: string + device?: string country: string state: string county: string city: string returningVisitor: boolean visitCount: number + pageHistory?: Array<{ at: string; path: string; referrer?: string }> }> } writeStatus: { diff --git a/src/App.css b/src/App.css index 4548323..c1a2cfd 100644 --- a/src/App.css +++ b/src/App.css @@ -3300,6 +3300,61 @@ background: rgba(201, 168, 76, 0.04); } +.admin-visitor-row--expanded { + background: rgba(201, 168, 76, 0.07) !important; +} + +.admin-visitor-history-row td { + padding: 0 !important; + background: #0d0d0a; + border-bottom: 1px solid rgba(201, 168, 76, 0.12); +} + +.admin-visitor-history { + padding: 0.75rem 1.25rem 1rem; +} + +.admin-visitor-history-label { + font-size: 0.78rem; + color: #8a7f5a; + margin: 0 0 0.5rem; + text-transform: uppercase; + letter-spacing: 0.06em; +} + +.admin-visitor-history-list { + list-style: none; + margin: 0; + padding: 0; + display: flex; + flex-direction: column; + gap: 0.3rem; +} + +.admin-visitor-history-entry { + display: flex; + gap: 0.75rem; + align-items: baseline; + font-size: 0.82rem; + flex-wrap: wrap; +} + +.admin-visitor-history-time { + color: #5a5440; + white-space: nowrap; + flex-shrink: 0; +} + +.admin-visitor-history-path { + color: #c9a84c; + font-family: monospace; +} + +.admin-visitor-history-ref { + color: #5a5440; + font-size: 0.78rem; +} + .admin-visits-table th { color: var(--brand-gold); letter-spacing: 0.08em; diff --git a/src/App.tsx b/src/App.tsx index d3efccb..200c41e 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -15,6 +15,20 @@ const SPOTIFY_EMBED_URL = const CONSENT_KEY = 'vbn_analytics_consent_choice' const HEADLINER_WIDGET_ID = 'WI_cmou3b4q7000701p0o9qmmcfj' +function usePageTracking() { + const location = useLocation() + useEffect(() => { + if (localStorage.getItem(CONSENT_KEY) !== 'accepted') return + const referrer = document.referrer || '' + fetch('/api/analytics/pageview', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ path: location.pathname, referrer }), + keepalive: true, + }).catch(() => {}) + }, [location.pathname]) +} + function toSpotifyEpisodeEmbedUrl(url: string | undefined): string { if (!url) return '' @@ -2120,6 +2134,7 @@ export default function App() { const [content, setContent] = useState(DEFAULTS) const navigate = useNavigate() const location = useLocation() + usePageTracking() useEffect(() => { fetch('/api/admin-content') diff --git a/src/components/AnalyticsPanel.tsx b/src/components/AnalyticsPanel.tsx index ec4a5d4..e37779d 100644 --- a/src/components/AnalyticsPanel.tsx +++ b/src/components/AnalyticsPanel.tsx @@ -68,7 +68,8 @@ export function AnalyticsPanel({ onDeployHook, onRefreshStatus, }: Props) { - const [chartTab, setChartTab] = useState<'overview' | 'breakdown' | 'geographic'>('overview') + const [chartTab, setChartTab] = useState<'overview' | 'breakdown' | 'geographic' | 'referrers'>('overview') + const [expandedVisitor, setExpandedVisitor] = useState(null) if (statsStatus === 'loading') return

Loading analytics…

if (statsStatus === 'error') return

Failed to load analytics.

@@ -165,6 +166,33 @@ export function AnalyticsPanel({ ], } + // 30-day trend chart + const last30DaysReal = stats.visitors.last30DaysReal ?? [] + const thirtyDayChartData = { + labels: last30DaysReal.map((d: { day: string; hits: number }) => d.day.slice(5)), + datasets: [{ + label: 'Real Visitors', + data: last30DaysReal.map((d: { day: string; hits: number }) => d.hits), + borderColor: '#c9a84c', + backgroundColor: 'rgba(201, 168, 76, 0.08)', + tension: 0.3, + fill: true, + pointRadius: 2, + }], + } + + // Device breakdown (doughnut) + const deviceBreakdown = stats.visitors.deviceBreakdown ?? { mobile: 0, desktop: 0, tablet: 0, unknown: 0 } + const deviceChartData = { + labels: ['Desktop', 'Mobile', 'Tablet', 'Unknown'], + datasets: [{ + data: [deviceBreakdown.desktop, deviceBreakdown.mobile, deviceBreakdown.tablet, deviceBreakdown.unknown], + backgroundColor: ['#c9a84c', '#a0853d', '#6d5a2e', '#444'], + borderColor: '#1a1a15', + borderWidth: 2, + }], + } + const chartOptions = { responsive: true, maintainAspectRatio: true, @@ -240,51 +268,77 @@ export function AnalyticsPanel({ {/* Chart Tabs */}
- + +
{/* Overview Charts */} {chartTab === 'overview' && ( -
-
-

7-Day Trend (Real vs Bot)

- + <> +
+
+

7-Day Trend (Real vs Bot)

+ +
+
+

Traffic Composition

+ +
-
-

Traffic Composition

- +
+
+

30-Day Real Visitor Trend

+ +
-
+ )} - {/* Bot Breakdown Charts */} + {/* Bot & Device Charts */} {chartTab === 'breakdown' && ( -
-
-

Bot Sources

- {botReasons.length === 0 ? ( -

No bot traffic detected.

- ) : ( - - )} -
-
-

Bot Detection Details

- {botReasons.length === 0 ? ( -

No bots detected yet.

- ) : ( + <> +
+
+

Device Breakdown

+ +
+
+

Device Counts

    - {botReasons.map((reason: { reason: string; count: number }) => ( -
  • - {reason.reason.replace(/-/g, ' ')} - {reason.count.toLocaleString()} -
    • - ))} +
  • Desktop{deviceBreakdown.desktop.toLocaleString()}
    • +
  • Mobile{deviceBreakdown.mobile.toLocaleString()}
    • +
  • Tablet{deviceBreakdown.tablet.toLocaleString()}
    • + {deviceBreakdown.unknown > 0 &&
  • Unknown{deviceBreakdown.unknown.toLocaleString()}
    • }
- )} +
-
+
+
+

Bot Sources

+ {botReasons.length === 0 ? ( +

No bot traffic detected.

+ ) : ( + + )} +
+
+

Bot Detection Details

+ {botReasons.length === 0 ? ( +

No bots detected yet.

+ ) : ( +
    + {botReasons.map((reason: { reason: string; count: number }) => ( +
  • + {reason.reason.replace(/-/g, ' ')} + {reason.count.toLocaleString()} +
    • + ))} +
+ )} +
+
+ )} {/* Geographic Charts */} @@ -301,6 +355,27 @@ export function AnalyticsPanel({
)} + {/* Referrers Tab */} + {chartTab === 'referrers' && ( +
+
+

Top Traffic Sources

+ {(stats.visitors.topReferrers ?? []).length === 0 ? ( +

No referrer data yet. Referrers are captured on SPA navigations after consent.

+ ) : ( +
    + {(stats.visitors.topReferrers ?? []).map((item: { referrer: string; count: number }) => ( +
  • + {item.referrer} + {item.count.toLocaleString()} +
    • + ))} +
+ )} +
+
+ )} + {/* Recent Visitors Table */}

Visitor Details

@@ -347,25 +422,64 @@ export function AnalyticsPanel({ IP Country Path + Referrer + Device Returning Visit # - {stats.visitors.recentVisits.map(row => ( - - {formatDate(row.at)} - {maskIp(row.ip)} - {row.country || '—'} - {row.path} - - - {row.returningVisitor ? 'Returning' : 'New'} - - - {row.visitCount} - - ))} + {stats.visitors.recentVisits.map(row => { + const rowKey = `${row.visitorId}-${row.at}` + const isExpanded = expandedVisitor === rowKey + return ( + <> + setExpandedVisitor(isExpanded ? null : rowKey)} + title="Click to view page history" + > + {formatDate(row.at)} + {maskIp(row.ip)} + {row.country || '—'} + {row.path} + {row.referrer || '—'} + {row.device || '—'} + + + {row.returningVisitor ? 'Returning' : 'New'} + + + {row.visitCount} {isExpanded ? '▲' : '▼'} + + {isExpanded && (row.pageHistory ?? []).length > 0 && ( + + +
+

Full page history for this visitor ({(row.pageHistory ?? []).length} pages):

+
    + {[...(row.pageHistory ?? [])].reverse().map((entry, i) => ( +
  1. + {formatDate(entry.at)} + {entry.path} + {entry.referrer && from {entry.referrer}} +
    2. + ))} +
+
+ + + )} + {isExpanded && (row.pageHistory ?? []).length === 0 && ( + +

No page history yet — history is built from SPA navigations after consent.

+ + )} + + ) + })}