Add grounded Ollama rewrite flow for chatbot

2026-04-14 12:46:56 -04:00
parent 50459b7234
commit 87967bf149
3 changed files with 333 additions and 9 deletions
@@ -295,6 +295,8 @@ interface IntentAnalysis {
  subject: string
  excludedSubject: string
  isCorrection: boolean
+  needsClarification: boolean
+  clarificationPrompt: string
  verseRefs: string[]
  episodeNumber: number | null
  isFollowUp: boolean
@@ -700,6 +702,25 @@ function buildConversationOnlyReply(cue: ConversationCue, context: ChatContextSt
  return null
 }

+function buildClarificationReply(analysis: IntentAnalysis, context: ChatContextState): ChatResponse {
+  const followupSuggestions = context.lastChunks.length > 0
+    ? buildSmartFollowUpPrompts(
+        context.lastChunks,
+        analyzeIntent(context.lastPrompt || 'go deeper', context),
+        context.lastChunks[0]?.content ?? '',
+      )
+    : [
+        { label: 'Who was Titus?', prompt: 'Who was Titus?' },
+        { label: 'Give me a summary of Titus 3:4-7', prompt: 'Give me a summary of Titus 3:4-7' },
+      ]
+
+  return {
+    text: `${analysis.clarificationPrompt}\n\nI want to answer from Nate's notes accurately, so one more detail will help me match the right content.`,
+    suggestions: followupSuggestions.slice(0, 3),
+    sources: context.lastChunks.length > 0 ? buildSourceCitations(context.lastChunks) : undefined,
+  }
+}
+
 function analyzeIntent(query: string, context: ChatContextState): IntentAnalysis {
  const rawQuery = query.trim()
  const normalized = rawQuery.toLowerCase()
@@ -711,12 +732,32 @@ function analyzeIntent(query: string, context: ChatContextState): IntentAnalysis
  const wantsDefinition = /\bmean\b|\bmeaning\b|\bdefine\b|\bwhat is\b|\bwhat does\b/.test(normalized)
  const wantsIdentity = /\bwho is\b|\bwho was\b|\btell me about\b/.test(normalized)
  const wantsPractice = /\bhow can i\b|\bhow do i\b|\bhow should i\b|\bpray\b|\bpractice\b/.test(normalized)
+  const wantsComparison = /\bdifference\b|\bcompare\b|\bvs\.?\b|\bversus\b/.test(normalized)
  const correctionSubjects = extractCorrectionSubjects(normalized)
  const isFollowUp = isContextualFollowUp(rawQuery) || correctionSubjects !== null
  const explicitSubject = extractIdentitySubject(normalized)
  const subject = correctionSubjects?.expected || explicitSubject || (isFollowUp ? context.lastSubject : '')
  const excludedSubject = correctionSubjects?.rejected ?? ''
  const isCorrection = correctionSubjects !== null
+  const tokenCount = tokenize(rawQuery).length
+  const hasStandalonePronoun = /\b(this|that|it|he|she|they|them|those|these|him|her)\b/.test(normalized)
+
+  let needsClarification = false
+  let clarificationPrompt = ''
+
+  if (!isFollowUp && hasStandalonePronoun && context.turnCount === 0) {
+    needsClarification = true
+    clarificationPrompt = 'Can you name the specific person, verse, or episode you mean?'
+  } else if (!isFollowUp && tokenCount < 2 && verseRefs.length === 0 && episodeNumber === null) {
+    needsClarification = true
+    clarificationPrompt = 'Can you make that a bit more specific so I can match Nate\'s notes?'
+  } else if (/\bwho\s+(is|was)\b/.test(normalized) && !subject) {
+    needsClarification = true
+    clarificationPrompt = 'Who would you like to ask about specifically?'
+  } else if (wantsComparison && !/\band\b|\bbetween\b/.test(normalized)) {
+    needsClarification = true
+    clarificationPrompt = 'What two things would you like me to compare?'
+  }

  let type: ChatIntent = 'general'
  if (wantsEpisode) type = 'episode-lookup'
@@ -727,6 +768,7 @@ function analyzeIntent(query: string, context: ChatContextState): IntentAnalysis
  else if (wantsPractice) type = 'practice'
  else if (wantsSummary) type = 'overview'
  else if (isFollowUp) type = 'follow-up'
+  else if (wantsComparison) type = 'overview'

  let searchQuery = rawQuery
  if (type === 'identity' && subject) {
@@ -757,6 +799,8 @@ function analyzeIntent(query: string, context: ChatContextState): IntentAnalysis
    subject,
    excludedSubject,
    isCorrection,
+    needsClarification,
+    clarificationPrompt,
    verseRefs,
    episodeNumber,
    isFollowUp,
@@ -969,15 +1013,23 @@ function buildLowConfidenceReply(scored: ScoredChunk[], analysis: IntentAnalysis
  const choices = getTopDistinctChunks(scored, 3)
  const topLabels = choices.slice(0, 2).map(chunk => chunk.sourceLabel)
  const text = topLabels.length === 2
-    ? `I want to be accurate, so I need one quick clarification. Did you mean "${topLabels[0]}" or "${topLabels[1]}"?`
-    : `I want to be accurate, and I don't have enough confidence to answer yet. Pick the closest direction and I'll continue.`
+    ? `I don't want to make up an answer. I couldn't find a confident match in Nate's notes yet. Did you mean "${topLabels[0]}" or "${topLabels[1]}"?`
+    : `I don't want to make up an answer. I couldn't find this clearly in Nate's current notes yet. You can submit this question directly to Nate below.`
+
+  const choiceSuggestions = choices.map(chunk => ({
+    label: chunk.sourceLabel,
+    prompt: buildPromptForChunk(chunk, analysis),
+  }))

  return {
    text,
-    suggestions: choices.map(chunk => ({
-      label: chunk.sourceLabel,
-      prompt: buildPromptForChunk(chunk, analysis),
-    })),
+    suggestions: [
+      ...choiceSuggestions,
+      {
+        label: 'How do I submit a question to Nate?',
+        prompt: 'How do I submit a question to Nate?',
+      },
+    ].slice(0, 3),
    sources: buildSourceCitations(choices),
  }
 }
@@ -1036,10 +1088,10 @@ function buildSmartFallbackReply(analysis: IntentAnalysis, context: ChatContextS
  if (context.lastVerseRefs[0]) suggestions.add(`How does ${formatVerseRef(context.lastVerseRefs[0])} connect to the rest of the chapter?`)
  suggestions.add('What does Titus 1 teach about church leadership?')
  suggestions.add('What does grace train us to do?')
-  suggestions.add('How should I apply this today?')
+  suggestions.add('How do I submit a question to Nate?')

  return {
-    text: `I don't have a strong enough match yet to answer that clearly. Try one of these more specific prompts and I'll narrow it down.`,
+    text: `I don't want to make up an answer. I couldn't find this in Nate's current notes. Please submit this question to Nate using the contact form below.`,
    suggestions: [...suggestions].slice(0, 3).map(prompt => ({ label: prompt, prompt })),
  }
 }
@@ -1280,6 +1332,41 @@ function ChatBot({ mode = 'embedded' }: { mode?: 'embedded' | 'standalone' }) {
    }
  }

+  const maybeRewriteWithLlm = async (
+    question: string,
+    draft: ChatResponse,
+    contextChunks: SearchChunk[],
+  ): Promise<ChatResponse> => {
+    try {
+      const response = await fetch('/api/chatbot-grounded-rewrite', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          question,
+          draftAnswer: draft.text,
+          sources: draft.sources ?? [],
+          contextChunks: contextChunks.slice(0, 4).map(chunk => ({
+            title: chunk.title,
+            sourceLabel: chunk.sourceLabel,
+            content: chunk.content,
+          })),
+        }),
+      })
+
+      if (!response.ok) return draft
+      const payload = await response.json()
+      const rewritten = typeof payload?.text === 'string' ? payload.text.trim() : ''
+      if (!rewritten) return draft
+
+      return {
+        ...draft,
+        text: rewritten,
+      }
+    } catch {
+      return draft
+    }
+  }
+
  const respond = async (query: string) => {
    const userMsg: ChatMessage = { role: 'user', text: query }
    setMessages(m => [...m, userMsg])
@@ -1293,6 +1380,7 @@ function ChatBot({ mode = 'embedded' }: { mode?: 'embedded' | 'standalone' }) {
    const contextualQuery = enrichQueryWithContext(trimmedQuery, activeContext)

    setTimeout(() => {
+      void (async () => {
      const cue = detectConversationCue(trimmedQuery)
      const cueReply = buildConversationOnlyReply(cue, activeContext, responseStyle)
      if (cueReply) {
@@ -1327,6 +1415,18 @@ function ChatBot({ mode = 'embedded' }: { mode?: 'embedded' | 'standalone' }) {
      }

      const analysis = analyzeIntent(contextualQuery, activeContext)
+      if (analysis.needsClarification) {
+        const clarificationReply = buildClarificationReply(analysis, activeContext)
+        setMessages(m => [...m, {
+          role: 'bot',
+          text: formatResponseText(clarificationReply),
+          suggestions: clarificationReply.suggestions,
+          sources: clarificationReply.sources,
+        }])
+        setLoading(false)
+        return
+      }
+
      const bibleVersionIntent = isBibleVersionQuery(analysis.searchQuery, analysis.queryTokens)
      const scored = latestChunks
        .map(chunk => {
@@ -1340,17 +1440,27 @@ function ChatBot({ mode = 'embedded' }: { mode?: 'embedded' | 'standalone' }) {
        .sort((a, b) => b.score - a.score)

      let botReply: ChatResponse
+      let topChunksForRewrite: SearchChunk[] = []
+      let shouldTryLlmRewrite = false
      if (scored.length > 0) {
+        const confidence = determineConfidence(scored, analysis)
+        topChunksForRewrite = getTopDistinctChunks(scored, 3)
        botReply = synthesizeSmartReply(scored, analysis, activeContext, responseStyle)
-        if (determineConfidence(scored, analysis) !== 'low') {
+        if (confidence !== 'low') {
+          shouldTryLlmRewrite = true
          chatContextRef.current = buildNextChatContext(contextualQuery, analysis, scored, activeContext)
        }
      } else {
        botReply = buildSmartFallbackReply(analyzeIntent(contextualQuery, activeContext), activeContext)
      }

+      if (shouldTryLlmRewrite) {
+        botReply = await maybeRewriteWithLlm(trimmedQuery, botReply, topChunksForRewrite)
+      }
+
      setMessages(m => [...m, { role: 'bot', text: formatResponseText(botReply), suggestions: botReply.suggestions, sources: botReply.sources }])
      setLoading(false)
+      })()
    }, 400)
  }