diff --git a/package.json b/package.json index 2f95496..fd387df 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "siteforge", "private": true, - "version": "2.2.1", + "version": "2.2.4", "type": "module", "scripts": { "dev": "vite", diff --git a/server.js b/server.js index 975b7c5..a282747 100644 --- a/server.js +++ b/server.js @@ -232,8 +232,67 @@ Promise.all([ console.error('[series-scheduler] error:', err) }) - app.listen(PORT, () => { + const server = app.listen(PORT, () => { logResendEmailAlignmentWarnings() console.log(`Siteforge listening on http://localhost:${PORT}`) }) + + // On every deploy the running container gets SIGTERM'd and replaced. + // All the queueXWrite() helpers in data.js debounce disk writes onto + // per-resource promise chains — without draining those here, a redeploy + // can kill the process mid-write and silently lose the last few + // analytics events (or any other pending write) of that container's life. + function collectPendingWrites() { + return [ + state.hitStatsWritePromise, + state.visitorStatsWritePromise, + state.contactSubmissionsWritePromise, + state.questionsWritePromise, + state.draftQuestionsWritePromise, + state.replyTemplatesWritePromise, + state.replyHistoryWritePromise, + state.podcastChecklistWritePromise, + state.studyUsersWritePromise, + state.studyCommunityWritePromise, + state.studyRemindersWritePromise, + state.studyCommentsWritePromise, + state.studyCertificatesWritePromise, + state.episodeScriptsWritePromise, + state.downloadCountsWritePromise, + state.episodePlaysWritePromise, + state.analyticsEventsWritePromise, + state.qrCodesWritePromise, + state.emailSettingsWritePromise, + state.calendarEventsWritePromise, + state.auditLogWritePromise, + state.webhooksWritePromise, + state.studySessionsWritePromise, + ...state.studyNotesWriteQueues.values(), + ...state.studyProgressWriteQueues.values(), + ] + } + + let shuttingDown = false + async function gracefulShutdown(signal) { + if (shuttingDown) return + shuttingDown = true + console.log(`[shutdown] ${signal} received — closing server and flushing pending writes...`) + server.close() + const forceExit = setTimeout(() => { + console.error('[shutdown] timed out waiting for pending writes; exiting anyway') + process.exit(1) + }, 10_000) + try { + await Promise.all(collectPendingWrites()) + console.log('[shutdown] all pending writes flushed') + } catch (err) { + console.error('[shutdown] error while flushing writes:', err) + } finally { + clearTimeout(forceExit) + process.exit(0) + } + } + + process.on('SIGTERM', () => gracefulShutdown('SIGTERM')) + process.on('SIGINT', () => gracefulShutdown('SIGINT')) }) diff --git a/server/data.js b/server/data.js index 0bec49f..8184460 100644 --- a/server/data.js +++ b/server/data.js @@ -1064,6 +1064,8 @@ export async function createBackupSnapshot(reason = 'scheduled') { publishState: state.publishState, hitStats: state.hitStats, visitorStats: state.visitorStats, + analyticsEvents: state.analyticsEvents, + episodePlays: state.episodePlays, contactSubmissions: state.contactSubmissions, studyCommunityPosts: state.studyCommunityPosts, replyTemplates: state.replyTemplates, @@ -1179,6 +1181,8 @@ export async function restoreFromBackup(filename) { state.hitStats = sanitizeLoadedHitStats(parsed?.hitStats) state.visitorStats = sanitizeLoadedVisitorStats(parsed?.visitorStats) + state.analyticsEvents = sanitizeLoadedAnalyticsEvents(parsed?.analyticsEvents) + state.episodePlays = sanitizeLoadedEpisodePlays(parsed?.episodePlays) state.contactSubmissions = sanitizeLoadedContactSubmissions(parsed?.contactSubmissions) state.replyTemplates = sanitizeReplyTemplates(parsed?.replyTemplates) state.replyHistory = sanitizeReplyHistory(parsed?.replyHistory) @@ -1186,6 +1190,8 @@ export async function restoreFromBackup(filename) { queueHitStatsWrite() queueVisitorStatsWrite() + queueAnalyticsEventsWrite() + queueEpisodePlaysWrite() queueContactSubmissionsWrite() queueReplyTemplatesWrite() queueReplyHistoryWrite() @@ -1194,6 +1200,8 @@ export async function restoreFromBackup(filename) { await Promise.all([ state.hitStatsWritePromise, state.visitorStatsWritePromise, + state.analyticsEventsWritePromise, + state.episodePlaysWritePromise, state.contactSubmissionsWritePromise, state.replyTemplatesWritePromise, state.replyHistoryWritePromise, @@ -1298,6 +1306,23 @@ function sanitizeLoadedHitStats(value) { } } +function sanitizeLoadedAnalyticsEvents(value) { + return { + scrollDepth: value?.scrollDepth && typeof value.scrollDepth === 'object' ? value.scrollDepth : {}, + timeOnPage: value?.timeOnPage && typeof value.timeOnPage === 'object' ? value.timeOnPage : {}, + outboundClicks: value?.outboundClicks && typeof value.outboundClicks === 'object' ? value.outboundClicks : {}, + linkClicks: value?.linkClicks && typeof value.linkClicks === 'object' ? value.linkClicks : {}, + utmSources: value?.utmSources && typeof value.utmSources === 'object' ? value.utmSources : {}, + searchQueries: value?.searchQueries && typeof value.searchQueries === 'object' ? value.searchQueries : {}, + notFound: value?.notFound && typeof value.notFound === 'object' ? value.notFound : {}, + audioEvents: value?.audioEvents && typeof value.audioEvents === 'object' ? value.audioEvents : {}, + } +} + +function sanitizeLoadedEpisodePlays(value) { + return (value && typeof value === 'object' && !Array.isArray(value)) ? value : {} +} + function sanitizeLoadedVisitorStats(value) { const loadedVisitors = value?.visitors && typeof value.visitors === 'object' ? value.visitors : {} diff --git a/server/routes/analytics.js b/server/routes/analytics.js index 49bf514..381094a 100644 --- a/server/routes/analytics.js +++ b/server/routes/analytics.js @@ -75,28 +75,39 @@ async function resolveGeo(ip, ipHash) { return fallback } -// Simple in-memory rate limiter for pageview endpoint +// Simple in-memory rate limiter for pageview endpoint. +// This keys on req.ip, which collapses to one value for every visitor behind +// a shared IP (a proxy in front of the app, a school/church network, CGNAT). +// If TRUST_PROXY_HOPS is misconfigured for the real deployment topology, +// req.ip silently becomes the proxy's IP for everyone and this limiter starts +// dropping real traffic across all of them together. Log the first drop per +// IP per window so that scenario shows up instead of vanishing silently. const pageviewHits = new Map() function pageviewAllowed(ip) { const now = Date.now() const windowMs = 60 * 1000 - const entry = pageviewHits.get(ip) ?? { count: 0, start: now } - if (now - entry.start > windowMs) { entry.count = 0; entry.start = now } + const entry = pageviewHits.get(ip) ?? { count: 0, start: now, warned: false } + if (now - entry.start > windowMs) { entry.count = 0; entry.start = now; entry.warned = false } entry.count += 1 pageviewHits.set(ip, entry) if (pageviewHits.size > 5000) { const cutoff = now - windowMs for (const [k, v] of pageviewHits) { if (v.start < cutoff) pageviewHits.delete(k) } } - return entry.count <= 60 + const allowed = entry.count <= 60 + if (!allowed && !entry.warned) { + entry.warned = true + console.warn(`[analytics] pageview rate limit hit for an IP (${entry.count} requests/min) — if this fires often, check TRUST_PROXY_HOPS against the real deployment topology, since a misconfigured hop count collapses many visitors onto one IP here`) + } + return allowed } export function recordVisitor(req, res, overridePath = null, overrideReferrer = null) { const cookies = parseCookies(req.headers.cookie) let visitorId = cookies[VISITOR_COOKIE] + const hadCookie = Boolean(visitorId) if (!visitorId) { visitorId = randomUUID() - res.append('Set-Cookie', `${VISITOR_COOKIE}=${encodeURIComponent(visitorId)}; Max-Age=31536000; Path=/; SameSite=Lax; HttpOnly`) } const nowIso = new Date().toISOString() @@ -112,9 +123,18 @@ export function recordVisitor(req, res, overridePath = null, overrideReferrer = const ipHash = createHash('sha256').update(ip).digest('hex') - const existingIdByIp = state.visitorStats.ipHashIndex[ipHash] - if (existingIdByIp && existingIdByIp !== visitorId) { - visitorId = existingIdByIp + // Only fall back to IP-based correlation when the browser sent no cookie + // at all — e.g. it never persisted (Safari ITP, a cookie-blocking + // extension) — so that one real person doesn't get double-counted every + // visit. If a cookie WAS sent, trust it: two different people sharing a + // network (family wifi, a school or church LAN) must not be silently + // merged into one visitor just because they share an IP, which would + // corrupt page history and undercount unique visitors. + if (!hadCookie) { + const existingIdByIp = state.visitorStats.ipHashIndex[ipHash] + if (existingIdByIp) { + visitorId = existingIdByIp + } res.append('Set-Cookie', `${VISITOR_COOKIE}=${encodeURIComponent(visitorId)}; Max-Age=31536000; Path=/; SameSite=Lax; HttpOnly`) } diff --git a/server/study-helpers.js b/server/study-helpers.js index 50f03e8..5465537 100644 --- a/server/study-helpers.js +++ b/server/study-helpers.js @@ -413,7 +413,13 @@ export function detectBot(userAgent) { return { isBot: true, reason: 'http-client' } } if (/bot|crawler|spider|scraper|indexer|reader|fetcher|loader|agent|spyware|tracking|monitor/.test(ua)) { - if (!/chrome|firefox|safari|opera|edge|msie|trident|like gecko/.test(ua)) { + // Generic keywords like "reader" or "agent" can appear as substrings of + // otherwise-normal browser UAs (e.g. some webviews, regional browsers). + // Any UA carrying a real rendering-engine or vendor token is treated as + // a genuine browser rather than a bot, even if it also matches a keyword + // above — this list should stay a superset of the previous one so it + // only ever reduces false positives, never re-admits an actual bot. + if (!/chrome|firefox|safari|opera|edge|msie|trident|gecko|webkit|khtml|presto|crios|fxios|ucbrowser|samsungbrowser/.test(ua)) { return { isBot: true, reason: 'bot-keyword' } } } diff --git a/src/App.tsx b/src/App.tsx index e41e504..b0dd49d 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -15,7 +15,7 @@ import { EpisodeAudioPlayer } from './components/EpisodeAudioPlayer' import { NuggetsHostPortal } from './components/NuggetsHostPortal' import { BIBLE_BOOKS } from '../shared/bible-books.js' import './App.css' -import { sendEvent, useScrollDepthTracking, useTimeOnPage, useUTMCapture, useLinkClickTracking } from './analytics' +import { sendEvent, useScrollDepthTracking, useTimeOnPage, useUTMCapture, useLinkClickTracking, checkAnalyticsConsentResponse } from './analytics' const CONSENT_KEY = 'vbn_analytics_consent_choice' const HEADLINER_WIDGET_ID = 'WI_cmou3b4q7000701p0o9qmmcfj' @@ -28,7 +28,16 @@ function ScrollToTop() { function usePageTracking() { const location = useLocation() + const isFirstRun = useRef(true) useEffect(() => { + // The initial hard page load is already counted server-side by the + // hit-counting middleware in server.js (it sees the raw HTML request). + // Only client-side route changes (no new HTTP request) need reporting + // here — otherwise every fresh visit/refresh gets double-counted. + if (isFirstRun.current) { + isFirstRun.current = false + return + } if (localStorage.getItem(CONSENT_KEY) !== 'accepted') return const referrer = document.referrer || '' const controller = new AbortController() @@ -39,7 +48,7 @@ function usePageTracking() { body: JSON.stringify({ path: location.pathname, referrer }), keepalive: true, signal: controller.signal, - }).catch(() => {}).finally(() => clearTimeout(timeout)) + }).then(checkAnalyticsConsentResponse).catch(() => {}).finally(() => clearTimeout(timeout)) }, [location.pathname]) } @@ -1794,7 +1803,7 @@ function LatestEpisodePlayer({ content }: { content: SiteContent }) { listenStartRef.current = Date.now() if (!playTrackedRef.current && title) { playTrackedRef.current = true - fetch('/api/analytics/play', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ title }) }).catch(() => {}) + fetch('/api/analytics/play', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ title }) }).then(checkAnalyticsConsentResponse).catch(() => {}) } }).catch(() => {}) } diff --git a/src/analytics.ts b/src/analytics.ts index f64e8d9..a456e9b 100644 --- a/src/analytics.ts +++ b/src/analytics.ts @@ -24,6 +24,19 @@ function hasConsent(): boolean { } catch { /* sessionStorage unavailable */ } })() +// The client gates on a localStorage flag; the server gates on a cookie. +// If the cookie is lost (cleared separately, dropped by the browser on a +// plain-HTTP deployment, etc.) while localStorage still says "accepted", +// the client would otherwise keep believing tracking is on and silently +// lose every event forever. Self-heal by clearing the stale flag so the +// consent banner reappears instead of leaking data indefinitely. +export async function checkAnalyticsConsentResponse(res: Response): Promise { + try { + const data = await res.json() + if (data?.reason === 'no-consent') localStorage.removeItem(CONSENT_KEY) + } catch { /* ignore */ } +} + export function sendEvent(type: string, data: Record): void { if (!hasConsent()) return fetch('/api/analytics/event', { @@ -31,7 +44,7 @@ export function sendEvent(type: string, data: Record): void { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ type, ...data }), keepalive: true, - }).catch(() => {}) + }).then(checkAnalyticsConsentResponse).catch(() => {}) } // --- Scroll depth --- diff --git a/src/components/AnalyticsPanel.tsx b/src/components/AnalyticsPanel.tsx index c446c9f..afcf323 100644 --- a/src/components/AnalyticsPanel.tsx +++ b/src/components/AnalyticsPanel.tsx @@ -273,28 +273,38 @@ export function AnalyticsPanel({ )} - {/* KPI Grid */} -
+ {/* KPI Grid — split into two clearly-labeled groups so a number from + the selected range is never mistaken for an all-time total, or + vice versa. Previously all six cards sat in one grid with three + different implicit time windows (selected range, a hardcoded last + 30 days, and all-time), which read as contradictory when e.g. "7 + days" was selected but a "Last 30 Days" card sat right next to it. */} +

{rangeLabel}

+

Real Hits

{realHits.toLocaleString()}

- {rangeLabel} + Human traffic

Bot Traffic

{botHits.toLocaleString()}

{rangeTotalHits > 0 ? `${((botHits / rangeTotalHits) * 100).toFixed(1)}% of total` : '0% of total'}
+
+

Total Hits

+

{rangeTotalHits.toLocaleString()}

+ Real + bot +
+
+ +

All-Time

+

All-Time Real Hits

{((stats as AdminStats & { allTimeRealHits?: number }).allTimeRealHits ?? stats.totalHits).toLocaleString()}

Since {formatDate(stats.firstHitAt)}
-
-

Last 30 Days

-

{last30DaysRealTotal.toLocaleString()}

- Real hits · {last30DaysBotTotal.toLocaleString()} bots -

Unique Visitors

{stats.visitors.uniqueVisitors.toLocaleString()}

@@ -305,6 +315,11 @@ export function AnalyticsPanel({

{stats.visitors.totalVisits > 0 ? `${Math.round((stats.visitors.returningVisits / stats.visitors.totalVisits) * 100)}%` : '0%'}

{stats.visitors.returningVisits.toLocaleString()} return visits
+
+

Last 30 Days

+

{last30DaysRealTotal.toLocaleString()}

+ Real hits · {last30DaysBotTotal.toLocaleString()} bots +
{/* Chart Tabs */} @@ -409,6 +424,14 @@ export function AnalyticsPanel({ ))}
+
+

Top States

+
    + {stats.visitors.topStates.slice(0, 10).map((item: { name: string; hits: number }) => ( +
  • {item.name}{item.hits.toLocaleString()}
    • + ))} +
+

Top Countries

    @@ -451,29 +474,12 @@ export function AnalyticsPanel({

    Total Visits

    {stats.visitors.totalVisits.toLocaleString()}

    All time
    -

    Unique Visitors

    {stats.visitors.uniqueVisitors.toLocaleString()}

    All time

    Returning Visits

    {stats.visitors.returningVisits.toLocaleString()}

    First Visit

    {formatDate(stats.visitors.firstVisitAt)}

    - -
    -
    -

    Top Countries

    -
      - {stats.visitors.topCountries.map((item: { name: string; hits: number }) => ( -
    • {item.name}{item.hits.toLocaleString()}
      • - ))} -
    -
    -
    -

    Top States

    -
      - {stats.visitors.topStates.map((item: { name: string; hits: number }) => ( -
    • {item.name}{item.hits.toLocaleString()}
      • - ))} -
    -
    -
    + {/* Top Countries/States/Cities live in the Geographic tab above, alongside + the map — not repeated here to avoid showing the same numbers twice + on one page. */}

    Recent Visitor Log

    diff --git a/src/components/EpisodeAudioPlayer.tsx b/src/components/EpisodeAudioPlayer.tsx index 51cce68..1a4def9 100644 --- a/src/components/EpisodeAudioPlayer.tsx +++ b/src/components/EpisodeAudioPlayer.tsx @@ -1,5 +1,5 @@ import { useEffect, useRef, useState } from 'react' -import { sendEvent } from '../analytics' +import { sendEvent, checkAnalyticsConsentResponse } from '../analytics' interface EpisodeAudioPlayerProps { src: string @@ -101,7 +101,7 @@ export function EpisodeAudioPlayer({ src, title, size = 'full', spotifyUrl }: Ep method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ title }), - }).catch(() => {}) + }).then(checkAnalyticsConsentResponse).catch(() => {}) } }).catch(() => {}) }