Siteforge/scripts/evaluate-chatbot.mjs

import fs from 'node:fs/promises'
import path from 'node:path'

const root = process.cwd()
const dataPath = path.join(root, 'data', 'chatbot-content.json')
const evalPath = path.join(root, 'data', 'chatbot-eval.json')

const STOP_WORDS = new Set([
  'a','an','the','is','are','was','were','be','been','being','have','has','had','do','does','did',
  'will','would','could','should','may','might','shall','can','i','you','he','she','it','we','they',
  'me','him','her','us','them','my','your','his','its','our','their','this','that','these','those',
  'and','but','or','nor','so','yet','for','of','in','on','at','to','from','with','by','about',
  'what','how','why','when','where','who','which','if','then','than','as','just','not',
])

const TOKEN_ALIASES = {
  bible: ['translation', 'version', 'scripture', 'bsb', 'berean'],
  translation: ['version', 'bsb', 'berean', 'bible'],
  version: ['translation', 'bsb', 'berean', 'bible'],
  elders: ['elder', 'leadership', 'leaders', 'overseer', 'pastor'],
  leadership: ['elders', 'elder', 'overseer', 'leaders'],
  grace: ['salvation', 'saved', 'godliness', 'mercy'],
  salvation: ['saved', 'grace', 'mercy', 'gospel'],
  saved: ['salvation', 'grace', 'mercy', 'gospel'],
  hope: ['blessed', 'appearing', 'return', 'coming'],
  politics: ['public', 'government', 'authorities'],
}

function tokenize(text) {
  return text
    .toLowerCase()
    .replace(/[^a-z0-9\s]/g, ' ')
    .split(/\s+/)
    .filter(token => token.length > 2 && !STOP_WORDS.has(token))
}

function expandTokens(tokens) {
  const expanded = new Set(tokens)
  for (const token of tokens) {
    const aliases = TOKEN_ALIASES[token] ?? []
    for (const alias of aliases) {
      for (const aliasToken of tokenize(alias)) expanded.add(aliasToken)
    }
  }
  return [...expanded]
}

function literalTerms(text) {
  return [...new Set(
    text
      .toLowerCase()
      .replace(/[^a-z0-9:\-\s']/g, ' ')
      .split(/\s+/)
      .map(term => term.trim())
      .filter(term => term.length >= 2 && !STOP_WORDS.has(term)),
  )]
}

function scoreEntry(entry, query) {
  const indexText = `${entry.title} ${entry.content} ${entry.keywords.join(' ')}`.toLowerCase()
  const queryTokens = expandTokens(tokenize(query))
  const terms = literalTerms(query)

  let score = 0

  for (const token of queryTokens) {
    if (entry.title.toLowerCase().includes(token)) score += 4
    else if (indexText.includes(token)) score += 2
  }

  for (const term of terms) {
    if (entry.title.toLowerCase().includes(term)) score += 2
    else if (indexText.includes(term)) score += 1
  }

  const phrase = query.toLowerCase().replace(/[^a-z0-9:\-\s']/g, ' ').replace(/\s+/g, ' ').trim()
  if (phrase.length > 6) {
    if (entry.title.toLowerCase().includes(phrase)) score += 10
    else if (indexText.includes(phrase)) score += 6
  }

  return score
}

async function main() {
  const [contentRaw, evalRaw] = await Promise.all([
    fs.readFile(dataPath, 'utf8'),
    fs.readFile(evalPath, 'utf8'),
  ])

  const entries = JSON.parse(contentRaw)
  const tests = JSON.parse(evalRaw)

  let pass = 0
  const failures = []

  for (const test of tests) {
    const ranked = entries
      .map(entry => ({ entry, score: scoreEntry(entry, test.query) }))
      .sort((a, b) => b.score - a.score)

    const topK = ranked.slice(0, test.expectedTopK)
    const hit = topK.some(item => item.entry.id === test.expectedEntryId)

    if (hit) {
      pass += 1
      continue
    }

    failures.push({
      query: test.query,
      expectedEntryId: test.expectedEntryId,
      expectedTopK: test.expectedTopK,
      actualTop: topK.map(item => ({ id: item.entry.id, title: item.entry.title, score: item.score })),
    })
  }

  const total = tests.length
  const pct = ((pass / total) * 100).toFixed(1)

  console.log(`Chatbot eval: ${pass}/${total} (${pct}%) passed`)

  if (failures.length > 0) {
    console.log('\nFailures:')
    for (const failure of failures) {
      console.log(`- Query: ${failure.query}`)
      console.log(`  Expected: ${failure.expectedEntryId} in top ${failure.expectedTopK}`)
      console.log(`  Actual: ${failure.actualTop.map(item => `${item.id} (${item.score})`).join(', ')}`)
    }
    process.exitCode = 1
  }
}

main().catch(error => {
  console.error(error)
  process.exitCode = 1
})