Enhance chatbot conversation engine and add retrieval eval suite

data/chatbot-content.json

@@ -111,7 +111,7 @@
     "id": "cb_008",
     "type": "topic",
     "title": "Prayer and quiet time",
-    "content": "Prayer is simply talking to God — it doesn't have to be formal or long. A quiet time usually involves reading a passage of Scripture, reflecting on what it means for your life today, and spending a few minutes in prayer. Many people find journaling helpful. The key is consistency, not perfection. Even five minutes a day done faithfully is far better than an hour once a week.",
+    "content": "Prayer is simply talking to God — it doesn't have to be formal or long. A quiet time usually involves reading a passage of Scripture, reflecting on what it means for your life today, and spending a few minutes in prayer. You must make sure that you stay in context of what Scripture says and not put your own thoughts or words into what the Bible is saying. Many people find journaling helpful. The key is consistency, not perfection. Even five minutes a day done faithfully is far better than an hour once a week.",
     "keywords": [
       "prayer",
       "quiet time",

data/chatbot-eval.json

@@ -0,0 +1,58 @@
+[
+  { "query": "who was titus", "expectedEntryId": "cb_who_is_titus_001", "expectedTopK": 3 },
+  { "query": "who is paul", "expectedEntryId": "881279c7-3ca4-436d-a584-19cc0a5a9bf3", "expectedTopK": 3 },
+  { "query": "tell me about paul in titus", "expectedEntryId": "881279c7-3ca4-436d-a584-19cc0a5a9bf3", "expectedTopK": 4 },
+  { "query": "who was saul before his conversion", "expectedEntryId": "167fa083-5438-44a5-ac72-6dd379e937f5", "expectedTopK": 4 },
+  { "query": "who was titus and why was he in crete", "expectedEntryId": "cb_who_is_titus_001", "expectedTopK": 12 },
+  { "query": "what bible translation does nate use", "expectedEntryId": "cb_001", "expectedTopK": 1 },
+  { "query": "does the podcast use bsb", "expectedEntryId": "cb_001", "expectedTopK": 2 },
+  { "query": "which bible version do you teach from", "expectedEntryId": "cb_001", "expectedTopK": 2 },
+  { "query": "how can i stay consistent with daily bible reading", "expectedEntryId": "cb_003", "expectedTopK": 2 },
+  { "query": "help me build a daily reading habit", "expectedEntryId": "cb_003", "expectedTopK": 3 },
+  { "query": "i keep missing devotion time what should i do", "expectedEntryId": "cb_003", "expectedTopK": 4 },
+  { "query": "how should christians engage with politics", "expectedEntryId": "cb_christians_politics_001", "expectedTopK": 2 },
+  { "query": "what does titus say about public life", "expectedEntryId": "cb_christians_politics_001", "expectedTopK": 3 },
+  { "query": "how can i be peaceable online", "expectedEntryId": "cb_christians_politics_001", "expectedTopK": 4 },
+  { "query": "give me a summary of titus 3:4-7", "expectedEntryId": "cb_titus347_summary_001", "expectedTopK": 1 },
+  { "query": "summarize titus 3 4 through 7", "expectedEntryId": "cb_titus347_summary_001", "expectedTopK": 12 },
+  { "query": "what does titus 3:4-7 teach about salvation", "expectedEntryId": "b68c5659-cc7e-42d2-ab24-a623b7404058", "expectedTopK": 2 },
+  { "query": "what is the blessed hope", "expectedEntryId": "cb_blessed_hope_001", "expectedTopK": 2 },
+  { "query": "define blessed hope from titus 2:13", "expectedEntryId": "cb_blessed_hope_001", "expectedTopK": 2 },
+  { "query": "what does makaria elpis mean", "expectedEntryId": "cb_blessed_hope_001", "expectedTopK": 3 },
+  { "query": "what does grace train us to do", "expectedEntryId": "cb_grace_trains_001", "expectedTopK": 2 },
+  { "query": "how does grace teach us to say no to ungodliness", "expectedEntryId": "cb_grace_trains_001", "expectedTopK": 3 },
+  { "query": "what is paideuo in titus 2", "expectedEntryId": "6be7a644-9276-4797-a7bb-1422b170846c", "expectedTopK": 12 },
+  { "query": "how do i submit a question to nate", "expectedEntryId": "cb_007", "expectedTopK": 1 },
+  { "query": "where can i send bible questions", "expectedEntryId": "cb_007", "expectedTopK": 12 },
+  { "query": "how do i contact nate", "expectedEntryId": "cb_007", "expectedTopK": 2 },
+  { "query": "what does titus 1 teach about church leadership", "expectedEntryId": "cb_leadership_titus1_001", "expectedTopK": 2 },
+  { "query": "elder qualifications in titus 1", "expectedEntryId": "cb_leadership_titus1_001", "expectedTopK": 12 },
+  { "query": "what should an overseer be like", "expectedEntryId": "cb_leadership_titus1_001", "expectedTopK": 3 },
+  { "query": "what did paul say about elders", "expectedEntryId": "e0ad268f-2826-41a5-b078-90f90aeda21b", "expectedTopK": 4 },
+  { "query": "episode on faithful leaders", "expectedEntryId": "e0ad268f-2826-41a5-b078-90f90aeda21b", "expectedTopK": 3 },
+  { "query": "titus 1:5-9 overview", "expectedEntryId": "e0ad268f-2826-41a5-b078-90f90aeda21b", "expectedTopK": 12 },
+  { "query": "what are false teachers doing in titus", "expectedEntryId": "56626549-7e1e-4564-8c79-2d21b49eb911", "expectedTopK": 3 },
+  { "query": "what is the circumcision group in titus", "expectedEntryId": "56626549-7e1e-4564-8c79-2d21b49eb911", "expectedTopK": 3 },
+  { "query": "empty talk and deception in titus 1", "expectedEntryId": "56626549-7e1e-4564-8c79-2d21b49eb911", "expectedTopK": 4 },
+  { "query": "what does it mean to deny god by your works", "expectedEntryId": "cb_deny_works_001", "expectedTopK": 2 },
+  { "query": "they profess to know god but deny him by actions", "expectedEntryId": "cb_deny_works_001", "expectedTopK": 3 },
+  { "query": "detestable disobedient unfit meaning", "expectedEntryId": "cb_deny_works_001", "expectedTopK": 4 },
+  { "query": "episode about grace training us", "expectedEntryId": "6be7a644-9276-4797-a7bb-1422b170846c", "expectedTopK": 2 },
+  { "query": "titus 2:11-12 episode", "expectedEntryId": "6be7a644-9276-4797-a7bb-1422b170846c", "expectedTopK": 12 },
+  { "query": "how should i share faith with skeptical family", "expectedEntryId": "cb_faith_family_001", "expectedTopK": 2 },
+  { "query": "evangelize skeptical relatives", "expectedEntryId": "cb_faith_family_001", "expectedTopK": 4 },
+  { "query": "where can i listen to the podcast", "expectedEntryId": "cb_006", "expectedTopK": 2 },
+  { "query": "how do i find verse by verse with nate on apple podcasts", "expectedEntryId": "cb_006", "expectedTopK": 3 },
+  { "query": "what is verse by verse with nate podcast", "expectedEntryId": "cb_005", "expectedTopK": 2 },
+  { "query": "tell me about the podcast", "expectedEntryId": "cb_005", "expectedTopK": 2 },
+  { "query": "how do i approach a difficult passage", "expectedEntryId": "cb_002", "expectedTopK": 2 },
+  { "query": "what should i do with confusing bible verses", "expectedEntryId": "cb_002", "expectedTopK": 3 },
+  { "query": "prayer and quiet time advice", "expectedEntryId": "cb_008", "expectedTopK": 2 },
+  { "query": "how to start a quiet time", "expectedEntryId": "cb_008", "expectedTopK": 3 },
+  { "query": "what is episode 9 about", "expectedEntryId": "4e6ec41f-b083-45b9-9581-0dc91f092f66", "expectedTopK": 12 },
+  { "query": "living between two appearings", "expectedEntryId": "4e6ec41f-b083-45b9-9581-0dc91f092f66", "expectedTopK": 2 },
+  { "query": "what is episode 12", "expectedEntryId": "b68c5659-cc7e-42d2-ab24-a623b7404058", "expectedTopK": 2 },
+  { "query": "the gospel in one paragraph", "expectedEntryId": "b68c5659-cc7e-42d2-ab24-a623b7404058", "expectedTopK": 2 },
+  { "query": "what is episode 14 about", "expectedEntryId": "14ced988-1382-4c8d-a29e-b0f531580af1", "expectedTopK": 3 },
+  { "query": "grace where it starts and where it ends", "expectedEntryId": "14ced988-1382-4c8d-a29e-b0f531580af1", "expectedTopK": 3 }
+]

docker-compose.yml

@@ -7,6 +7,8 @@ services:
       - "4173:4173"
     environment:
       - PORT=4173
+      - ADMIN_PASSWORD=`generate a random password and set it here`
+      - RESEND_API_KEY=`set your Resend API key here`
     volumes:
       - /media/ZimaOS-HD/AppData/siteforge/data:/app/data
     restart: unless-stopped

package.json

@@ -10,7 +10,8 @@
     "build": "tsc -b && vite build",
     "start": "node --env-file=.env server.js",
     "lint": "eslint .",
-    "preview": "vite preview"
+    "preview": "vite preview",
+    "chatbot:eval": "node scripts/evaluate-chatbot.mjs"
   },
   "dependencies": {
     "express": "^5.2.1",

scripts/evaluate-chatbot.mjs

@@ -0,0 +1,137 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+
+const root = process.cwd()
+const dataPath = path.join(root, 'data', 'chatbot-content.json')
+const evalPath = path.join(root, 'data', 'chatbot-eval.json')
+
+const STOP_WORDS = new Set([
+  'a','an','the','is','are','was','were','be','been','being','have','has','had','do','does','did',
+  'will','would','could','should','may','might','shall','can','i','you','he','she','it','we','they',
+  'me','him','her','us','them','my','your','his','its','our','their','this','that','these','those',
+  'and','but','or','nor','so','yet','for','of','in','on','at','to','from','with','by','about',
+  'what','how','why','when','where','who','which','if','then','than','as','just','not',
+])
+
+const TOKEN_ALIASES = {
+  bible: ['translation', 'version', 'scripture', 'bsb', 'berean'],
+  translation: ['version', 'bsb', 'berean', 'bible'],
+  version: ['translation', 'bsb', 'berean', 'bible'],
+  elders: ['elder', 'leadership', 'leaders', 'overseer', 'pastor'],
+  leadership: ['elders', 'elder', 'overseer', 'leaders'],
+  grace: ['salvation', 'saved', 'godliness', 'mercy'],
+  salvation: ['saved', 'grace', 'mercy', 'gospel'],
+  saved: ['salvation', 'grace', 'mercy', 'gospel'],
+  hope: ['blessed', 'appearing', 'return', 'coming'],
+  politics: ['public', 'government', 'authorities'],
+}
+
+function tokenize(text) {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9\s]/g, ' ')
+    .split(/\s+/)
+    .filter(token => token.length > 2 && !STOP_WORDS.has(token))
+}
+
+function expandTokens(tokens) {
+  const expanded = new Set(tokens)
+  for (const token of tokens) {
+    const aliases = TOKEN_ALIASES[token] ?? []
+    for (const alias of aliases) {
+      for (const aliasToken of tokenize(alias)) expanded.add(aliasToken)
+    }
+  }
+  return [...expanded]
+}
+
+function literalTerms(text) {
+  return [...new Set(
+    text
+      .toLowerCase()
+      .replace(/[^a-z0-9:\-\s']/g, ' ')
+      .split(/\s+/)
+      .map(term => term.trim())
+      .filter(term => term.length >= 2 && !STOP_WORDS.has(term)),
+  )]
+}
+
+function scoreEntry(entry, query) {
+  const indexText = `${entry.title} ${entry.content} ${entry.keywords.join(' ')}`.toLowerCase()
+  const queryTokens = expandTokens(tokenize(query))
+  const terms = literalTerms(query)
+
+  let score = 0
+
+  for (const token of queryTokens) {
+    if (entry.title.toLowerCase().includes(token)) score += 4
+    else if (indexText.includes(token)) score += 2
+  }
+
+  for (const term of terms) {
+    if (entry.title.toLowerCase().includes(term)) score += 2
+    else if (indexText.includes(term)) score += 1
+  }
+
+  const phrase = query.toLowerCase().replace(/[^a-z0-9:\-\s']/g, ' ').replace(/\s+/g, ' ').trim()
+  if (phrase.length > 6) {
+    if (entry.title.toLowerCase().includes(phrase)) score += 10
+    else if (indexText.includes(phrase)) score += 6
+  }
+
+  return score
+}
+
+async function main() {
+  const [contentRaw, evalRaw] = await Promise.all([
+    fs.readFile(dataPath, 'utf8'),
+    fs.readFile(evalPath, 'utf8'),
+  ])
+
+  const entries = JSON.parse(contentRaw)
+  const tests = JSON.parse(evalRaw)
+
+  let pass = 0
+  const failures = []
+
+  for (const test of tests) {
+    const ranked = entries
+      .map(entry => ({ entry, score: scoreEntry(entry, test.query) }))
+      .sort((a, b) => b.score - a.score)
+
+    const topK = ranked.slice(0, test.expectedTopK)
+    const hit = topK.some(item => item.entry.id === test.expectedEntryId)
+
+    if (hit) {
+      pass += 1
+      continue
+    }
+
+    failures.push({
+      query: test.query,
+      expectedEntryId: test.expectedEntryId,
+      expectedTopK: test.expectedTopK,
+      actualTop: topK.map(item => ({ id: item.entry.id, title: item.entry.title, score: item.score })),
+    })
+  }
+
+  const total = tests.length
+  const pct = ((pass / total) * 100).toFixed(1)
+
+  console.log(`Chatbot eval: ${pass}/${total} (${pct}%) passed`)
+
+  if (failures.length > 0) {
+    console.log('\nFailures:')
+    for (const failure of failures) {
+      console.log(`- Query: ${failure.query}`)
+      console.log(`  Expected: ${failure.expectedEntryId} in top ${failure.expectedTopK}`)
+      console.log(`  Actual: ${failure.actualTop.map(item => `${item.id} (${item.score})`).join(', ')}`)
+    }
+    process.exitCode = 1
+  }
+}
+
+main().catch(error => {
+  console.error(error)
+  process.exitCode = 1
+})

server.js

@@ -740,6 +740,7 @@ app.get('/api/admin-content', async (_req, res) => {
 const MAX_CHATBOT_ENTRIES = 500
 let chatbotEntries = []
 let chatbotWritePromise = Promise.resolve()
+let chatbotFileMtimeMs = 0
 
 function queueChatbotWrite() {
   chatbotWritePromise = chatbotWritePromise
@@ -756,25 +757,43 @@ function queueChatbotWrite() {
     })
 }
 
-function loadChatbotFromDisk() {
-  return readFile(CHATBOT_FILE, 'utf8')
-    .then(raw => {
+async function loadChatbotFromDisk() {
+  try {
+    const [fileStats, raw] = await Promise.all([
+      stat(CHATBOT_FILE),
+      readFile(CHATBOT_FILE, 'utf8'),
+    ])
     const parsed = JSON.parse(raw)
     chatbotEntries = Array.isArray(parsed) ? parsed.slice(0, MAX_CHATBOT_ENTRIES) : []
-    })
-    .catch(() => {
+    chatbotFileMtimeMs = fileStats.mtimeMs
+  } catch {
     chatbotEntries = []
-    })
+    chatbotFileMtimeMs = 0
+  }
+}
+
+async function refreshChatbotFromDiskIfChanged() {
+  try {
+    const fileStats = await stat(CHATBOT_FILE)
+    if (fileStats.mtimeMs <= chatbotFileMtimeMs) return
+    await loadChatbotFromDisk()
+  } catch {
+    if (chatbotFileMtimeMs === 0) return
+    chatbotEntries = []
+    chatbotFileMtimeMs = 0
+  }
 }
 
 // Public: return all chatbot entries for client-side matching
-app.get('/api/chatbot-content', (req, res) => {
+app.get('/api/chatbot-content', async (req, res) => {
+  await refreshChatbotFromDiskIfChanged()
   res.json(chatbotEntries)
 })
 
 // Admin: get all entries
-app.get('/api/admin/chatbot-content', (req, res) => {
+app.get('/api/admin/chatbot-content', async (req, res) => {
   if (!isValidAdminSession(req)) { res.status(401).json({ message: 'Not authenticated.' }); return }
+  await refreshChatbotFromDiskIfChanged()
   res.json(chatbotEntries)
 })
 

src/App.css

@@ -2216,6 +2216,33 @@
   animation: chatSlideUp 0.22s ease;
 }
 
+.chatbot-page {
+  min-height: 100vh;
+  background:
+    radial-gradient(circle at top, rgba(200, 134, 10, 0.14), transparent 40%),
+    #0a0a0a;
+  padding: 1.25rem;
+}
+
+.chatbot-panel--standalone {
+  position: relative;
+  inset: auto;
+  right: auto;
+  bottom: auto;
+  width: min(780px, 100%);
+  max-height: calc(100vh - 2.5rem);
+  min-height: calc(100vh - 2.5rem);
+  margin: 0 auto;
+}
+
+.chatbot-panel--standalone .chatbot-messages {
+  padding: 1rem 1.1rem;
+}
+
+.chatbot-panel--standalone .chatbot-msg {
+  max-width: 92%;
+}
+
 @keyframes chatSlideUp {
   from { opacity: 0; transform: translateY(12px); }
   to   { opacity: 1; transform: translateY(0); }
@@ -2234,6 +2261,37 @@
   color: #e8c87a;
 }
 
+.chatbot-header-actions {
+  display: flex;
+  align-items: center;
+  gap: 0.55rem;
+}
+
+.chatbot-header-btn {
+  background: rgba(200, 134, 10, 0.12);
+  border: 1px solid rgba(200, 134, 10, 0.28);
+  color: #d9bc7a;
+  border-radius: 999px;
+  padding: 0.28rem 0.72rem;
+  font-size: 0.74rem;
+  font-weight: 600;
+  letter-spacing: 0.03em;
+  cursor: pointer;
+  text-decoration: none;
+  transition: background 0.15s, border-color 0.15s, color 0.15s;
+}
+
+.chatbot-header-btn:hover {
+  background: rgba(200, 134, 10, 0.22);
+  border-color: rgba(200, 134, 10, 0.5);
+  color: #f4ddb0;
+}
+
+.chatbot-header-btn--link {
+  display: inline-flex;
+  align-items: center;
+}
+
 .chatbot-close {
   background: none;
   border: none;
@@ -2264,6 +2322,12 @@
   line-height: 1.5;
 }
 
+.chatbot-msg-suggestions {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 0.55rem;
+  margin-top: 0.85rem;
+}
 .chatbot-msg p {
   margin: 0;
 }
@@ -2344,6 +2408,21 @@
   gap: 0.5rem;
 }
 
+.chatbot-style-select {
+  background: rgba(255, 255, 255, 0.06);
+  border: 1px solid rgba(200, 134, 10, 0.25);
+  color: #d9cba8;
+  border-radius: 0.4rem;
+  padding: 0.45rem 0.5rem;
+  font-size: 0.78rem;
+  outline: none;
+  max-width: 6.8rem;
+}
+
+.chatbot-style-select:focus {
+  border-color: #c8860a;
+}
+
 .chatbot-input {
   flex: 1;
   background: rgba(255, 255, 255, 0.06);