Implement comprehensive analytics redesign with bot detection and charts
- Add bot detection logic to differentiate between real traffic and bot traffic - Detect common bots: search crawlers, social media crawlers, headless browsers, monitoring tools, security scanners - Expand hit-stats.json to track realHits, botHits, byPathReal, byPathBot, byDayReal, byDayBot, botReasons - Update analytics API endpoint to return bot/visitor breakdown with 7-day and 30-day comparisons - Add Chart.js integration with react-chartjs-2 for interactive visualizations - Create new AnalyticsPanel component with tabbed interface: Overview, Bot Breakdown, Geographic - Implement graphs showing: 7-day trend (real vs bot), traffic composition (doughnut), bot sources (pie), top pages (bar) - Display clear KPI cards showing real traffic %, bot traffic %, return visitor rate - Add deployment & cache status section to analytics dashboard - Improve data presentation with responsive grid layouts and color-coded metrics
This commit is contained in:
nmemmert
@@ -313,10 +313,17 @@ async function invokeWebhook(url, action) {
|
||||
|
||||
const EMPTY_HIT_STATS = {
|
||||
totalHits: 0,
|
||||
realHits: 0,
|
||||
botHits: 0,
|
||||
firstHitAt: null,
|
||||
lastHitAt: null,
|
||||
byPath: {},
|
||||
byPathReal: {},
|
||||
byPathBot: {},
|
||||
byDay: {},
|
||||
byDayReal: {},
|
||||
byDayBot: {},
|
||||
botReasons: {},
|
||||
}
|
||||
|
||||
let hitStats = { ...EMPTY_HIT_STATS }
|
||||
@@ -649,6 +656,54 @@ function sanitizeUserAgent(userAgent) {
|
||||
return userAgent.trim().slice(0, 300) || 'unknown'
|
||||
}
|
||||
|
||||
function detectBot(userAgent, pathInfo = {}) {
|
||||
if (!userAgent || typeof userAgent !== 'string') {
|
||||
return { isBot: true, reason: 'missing-user-agent' }
|
||||
}
|
||||
|
||||
const ua = userAgent.toLowerCase()
|
||||
|
||||
// Search engine crawlers
|
||||
if (/googlebot|bingbot|yandexbot|baiduspider|slurp|duckduckbot|sluplicate|googlebot-mobile/.test(ua)) {
|
||||
return { isBot: true, reason: 'search-crawler' }
|
||||
}
|
||||
|
||||
// Social media crawlers
|
||||
if (/facebookexternalhit|twitterbot|linkedinbot|pinterest|whatsapp|slack|discord|telegram|reddit|mastodon/.test(ua)) {
|
||||
return { isBot: true, reason: 'social-crawler' }
|
||||
}
|
||||
|
||||
// Headless browsers and automation
|
||||
if (/headless|phantomjs|puppeteer|playwright|selenium|nightmarebot|watir|webdriver|wdio|nightmare/.test(ua)) {
|
||||
return { isBot: true, reason: 'headless-browser' }
|
||||
}
|
||||
|
||||
// Monitoring and uptime checkers
|
||||
if (/uptimerobot|pingdom|statuspage|pagerduty|sentry|datadog|grafana|prometheus|newrelic|appdynamics/.test(ua)) {
|
||||
return { isBot: true, reason: 'monitoring-tool' }
|
||||
}
|
||||
|
||||
// Security scanners and tools
|
||||
if (/nmap|nikto|masscan|metasploit|nessus|openvas|qualys|burpsuite|zap|acunetix|sqlmap/.test(ua)) {
|
||||
return { isBot: true, reason: 'security-scanner' }
|
||||
}
|
||||
|
||||
// HTTP clients and frameworks
|
||||
if (/^(curl|wget|python|java|go|node|ruby|php|perl|lua|rust)[\/-]/.test(ua)) {
|
||||
return { isBot: true, reason: 'http-client' }
|
||||
}
|
||||
|
||||
// Common crawler keywords
|
||||
if (/bot|crawler|spider|scraper|indexer|reader|fetcher|loader|agent|spyware|tracking|monitor/.test(ua)) {
|
||||
// But allow some common real user agents that might contain these words
|
||||
if (!/chrome|firefox|safari|opera|edge|msie|trident|like gecko/.test(ua)) {
|
||||
return { isBot: true, reason: 'bot-keyword' }
|
||||
}
|
||||
}
|
||||
|
||||
return { isBot: false, reason: null }
|
||||
}
|
||||
|
||||
async function resolveGeo(ip) {
|
||||
if (!ip || isPrivateOrLocalIp(ip)) {
|
||||
return {
|
||||
@@ -842,10 +897,14 @@ function pruneStatsByDays(daysRaw) {
|
||||
}
|
||||
|
||||
const nextByDay = {}
|
||||
const nextByDayReal = {}
|
||||
const nextByDayBot = {}
|
||||
for (const [day, count] of Object.entries(hitStats.byDay)) {
|
||||
const ts = new Date(`${day}T00:00:00.000Z`).getTime()
|
||||
if (Number.isFinite(ts) && ts >= cutoff) {
|
||||
nextByDay[day] = count
|
||||
nextByDayReal[day] = hitStats.byDayReal?.[day] ?? 0
|
||||
nextByDayBot[day] = hitStats.byDayBot?.[day] ?? 0
|
||||
}
|
||||
}
|
||||
|
||||
@@ -858,6 +917,8 @@ function pruneStatsByDays(daysRaw) {
|
||||
visitorStats.lastVisitAt = keepRecent.length > 0 ? keepRecent[0].at : null
|
||||
|
||||
hitStats.byDay = nextByDay
|
||||
hitStats.byDayReal = nextByDayReal
|
||||
hitStats.byDayBot = nextByDayBot
|
||||
|
||||
queueHitStatsWrite()
|
||||
queueVisitorStatsWrite()
|
||||
@@ -1094,7 +1155,7 @@ function queueHitStatsWrite() {
|
||||
})
|
||||
}
|
||||
|
||||
function recordHit(pathname) {
|
||||
function recordHit(pathname, isBot = false, botReason = null) {
|
||||
const nowIso = new Date().toISOString()
|
||||
const dayKey = nowIso.slice(0, 10)
|
||||
const safePath = normalizeHitPath(pathname)
|
||||
@@ -1102,6 +1163,21 @@ function recordHit(pathname) {
|
||||
hitStats.totalHits += 1
|
||||
hitStats.lastHitAt = nowIso
|
||||
hitStats.firstHitAt = hitStats.firstHitAt ?? nowIso
|
||||
|
||||
if (isBot) {
|
||||
hitStats.botHits += 1
|
||||
hitStats.byPathBot[safePath] = (hitStats.byPathBot[safePath] ?? 0) + 1
|
||||
hitStats.byDayBot[dayKey] = (hitStats.byDayBot[dayKey] ?? 0) + 1
|
||||
if (botReason) {
|
||||
hitStats.botReasons[botReason] = (hitStats.botReasons[botReason] ?? 0) + 1
|
||||
}
|
||||
} else {
|
||||
hitStats.realHits += 1
|
||||
hitStats.byPathReal[safePath] = (hitStats.byPathReal[safePath] ?? 0) + 1
|
||||
hitStats.byDayReal[dayKey] = (hitStats.byDayReal[dayKey] ?? 0) + 1
|
||||
}
|
||||
|
||||
// Keep legacy byPath and byDay for backward compatibility
|
||||
hitStats.byPath[safePath] = (hitStats.byPath[safePath] ?? 0) + 1
|
||||
hitStats.byDay[dayKey] = (hitStats.byDay[dayKey] ?? 0) + 1
|
||||
|
||||
@@ -1128,10 +1204,17 @@ function loadHitStatsFromDisk() {
|
||||
const parsed = JSON.parse(raw)
|
||||
hitStats = {
|
||||
totalHits: Number(parsed?.totalHits) || 0,
|
||||
realHits: Number(parsed?.realHits) || 0,
|
||||
botHits: Number(parsed?.botHits) || 0,
|
||||
firstHitAt: typeof parsed?.firstHitAt === 'string' ? parsed.firstHitAt : null,
|
||||
lastHitAt: typeof parsed?.lastHitAt === 'string' ? parsed.lastHitAt : null,
|
||||
byPath: parsed?.byPath && typeof parsed.byPath === 'object' ? parsed.byPath : {},
|
||||
byPathReal: parsed?.byPathReal && typeof parsed.byPathReal === 'object' ? parsed.byPathReal : {},
|
||||
byPathBot: parsed?.byPathBot && typeof parsed.byPathBot === 'object' ? parsed.byPathBot : {},
|
||||
byDay: parsed?.byDay && typeof parsed.byDay === 'object' ? parsed.byDay : {},
|
||||
byDayReal: parsed?.byDayReal && typeof parsed.byDayReal === 'object' ? parsed.byDayReal : {},
|
||||
byDayBot: parsed?.byDayBot && typeof parsed.byDayBot === 'object' ? parsed.byDayBot : {},
|
||||
botReasons: parsed?.botReasons && typeof parsed.botReasons === 'object' ? parsed.botReasons : {},
|
||||
}
|
||||
})
|
||||
.catch(() => {
|
||||
@@ -1615,15 +1698,54 @@ app.get('/api/admin-stats', requireAdminAuth, (_req, res) => {
|
||||
.slice(0, 10)
|
||||
.map(([pathKey, hits]) => ({ path: pathKey, hits }))
|
||||
|
||||
const topPathsReal = Object.entries(hitStats.byPathReal)
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 10)
|
||||
.map(([pathKey, hits]) => ({ path: pathKey, hits }))
|
||||
|
||||
const topPathsBot = Object.entries(hitStats.byPathBot)
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 10)
|
||||
.map(([pathKey, hits]) => ({ path: pathKey, hits }))
|
||||
|
||||
const last7Days = buildLastNDaysStats(7)
|
||||
const last7DaysReal = last7Days.map(item => ({
|
||||
day: item.day,
|
||||
hits: hitStats.byDayReal?.[item.day] ?? 0
|
||||
}))
|
||||
const last7DaysBot = last7Days.map(item => ({
|
||||
day: item.day,
|
||||
hits: hitStats.byDayBot?.[item.day] ?? 0
|
||||
}))
|
||||
|
||||
const last30Days = buildLastNDaysStats(30)
|
||||
const last30DaysTotal = last30Days.reduce((sum, item) => sum + item.hits, 0)
|
||||
const last30DaysRealTotal = last30Days.reduce((sum, item) => sum + (hitStats.byDayReal?.[item.day] ?? 0), 0)
|
||||
const last30DaysBotTotal = last30Days.reduce((sum, item) => sum + (hitStats.byDayBot?.[item.day] ?? 0), 0)
|
||||
|
||||
const botReasons = Object.entries(hitStats.botReasons ?? {})
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 10)
|
||||
.map(([reason, count]) => ({ reason, count }))
|
||||
|
||||
const recentVisitorRows = visitorStats.recentVisits.slice(0, 100)
|
||||
|
||||
res.json({
|
||||
totalHits: hitStats.totalHits,
|
||||
realHits: hitStats.realHits ?? 0,
|
||||
botHits: hitStats.botHits ?? 0,
|
||||
firstHitAt: hitStats.firstHitAt,
|
||||
lastHitAt: hitStats.lastHitAt,
|
||||
topPaths,
|
||||
last7Days: buildLastNDaysStats(7),
|
||||
last30DaysTotal: buildLastNDaysStats(30).reduce((sum, item) => sum + item.hits, 0),
|
||||
topPathsReal,
|
||||
topPathsBot,
|
||||
last7Days,
|
||||
last7DaysReal,
|
||||
last7DaysBot,
|
||||
last30DaysTotal,
|
||||
last30DaysRealTotal,
|
||||
last30DaysBotTotal,
|
||||
botReasons,
|
||||
visitors: {
|
||||
totalVisits: visitorStats.totalVisits,
|
||||
uniqueVisitors: visitorStats.uniqueVisitors,
|
||||
@@ -1878,8 +2000,10 @@ app.post('/api/admin-stats/restore', requireAdminAuth, async (req, res) => {
|
||||
|
||||
app.use((req, res, next) => {
|
||||
if (shouldCountHit(req)) {
|
||||
recordHit(req.path)
|
||||
if (hasVisitorConsent(req)) {
|
||||
const ua = sanitizeUserAgent(req.get('user-agent'))
|
||||
const botDetection = detectBot(ua)
|
||||
recordHit(req.path, botDetection.isBot, botDetection.reason)
|
||||
if (hasVisitorConsent(req) && !botDetection.isBot) {
|
||||
recordVisitor(req, res).catch(err => {
|
||||
console.error('[visitor-stats] failed to record visitor:', err)
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user