diff --git a/README.md b/README.md index 8f5a3ec..95dfbde 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,50 @@ Admin auth: The app will be available at `http://localhost:4173`. +## Optional local LLM (Ollama) for grounded rewrites + +You can keep deterministic retrieval as the source-of-truth and optionally rewrite responses with a local model. + +1. Install and run Ollama on your host. +2. Pull a small model suited to older hardware, for example: + +```bash +ollama pull qwen2.5:3b-instruct +``` + +3. Start the API with these environment variables: + +```bash +CHATBOT_LLM_ENABLED=true +CHATBOT_LLM_BASE_URL=http://127.0.0.1:11434 +CHATBOT_LLM_MODEL=qwen2.5:3b-instruct +CHATBOT_LLM_TIMEOUT_MS=25000 +CHATBOT_LLM_NUM_CTX=2048 +``` + +4. Call the rewrite endpoint from your existing chat flow: + +`POST /api/chatbot-grounded-rewrite` + +Request payload shape: + +```json +{ + "question": "Who was Titus?", + "draftAnswer": "Deterministic answer produced by current retrieval/synthesis.", + "sources": ["Episode 2 - Introduction to Titus"], + "contextChunks": [ + { + "title": "Episode 2 - Introduction to Titus", + "sourceLabel": "Episode 2", + "content": "Titus was a Gentile..." + } + ] +} +``` + +If the endpoint fails or is disabled, keep your deterministic answer and existing fallback behavior. + Persistent admin saves: - Admin updates are written to `data/admin-content.json`. diff --git a/server.js b/server.js index 021184e..b38955a 100644 --- a/server.js +++ b/server.js @@ -55,6 +55,15 @@ const BACKUP_RETENTION_DAYS = 30 const BACKUP_INTERVAL_MS = 24 * 60 * 60 * 1000 const ADMIN_SESSION_TTL_MS = 7 * 24 * 60 * 60 * 1000 const ADMIN_PASSWORD = process.env.ADMIN_PASSWORD ?? 'change-me-admin-password' +const CHATBOT_LLM_ENABLED = process.env.CHATBOT_LLM_ENABLED === 'true' +const CHATBOT_LLM_BASE_URL = process.env.CHATBOT_LLM_BASE_URL ?? 'http://127.0.0.1:11434' +const CHATBOT_LLM_MODEL = process.env.CHATBOT_LLM_MODEL ?? 'qwen2.5:3b-instruct' +const CHATBOT_LLM_TIMEOUT_MS = Number(process.env.CHATBOT_LLM_TIMEOUT_MS) > 0 + ? Number(process.env.CHATBOT_LLM_TIMEOUT_MS) + : 25000 +const CHATBOT_LLM_NUM_CTX = Number(process.env.CHATBOT_LLM_NUM_CTX) > 0 + ? Number(process.env.CHATBOT_LLM_NUM_CTX) + : 2048 const EMPTY_VISITOR_STATS = { totalVisits: 0, @@ -784,12 +793,173 @@ async function refreshChatbotFromDiskIfChanged() { } } +function normalizeChatbotLlmContext(chunks) { + if (!Array.isArray(chunks)) return [] + + return chunks + .filter(chunk => chunk && typeof chunk.content === 'string') + .slice(0, 4) + .map((chunk, index) => { + const title = typeof chunk.title === 'string' ? chunk.title.trim().slice(0, 160) : `Context ${index + 1}` + const sourceLabel = typeof chunk.sourceLabel === 'string' && chunk.sourceLabel.trim() + ? chunk.sourceLabel.trim().slice(0, 200) + : title + const content = chunk.content.trim().slice(0, 1400) + + return { + title, + sourceLabel, + content, + } + }) + .filter(chunk => chunk.content.length > 0) +} + +function buildGroundedRewritePrompt({ question, draftAnswer, sources, contextChunks }) { + const sourceList = Array.isArray(sources) && sources.length > 0 + ? sources.slice(0, 6).map(source => `- ${String(source).slice(0, 220)}`).join('\n') + : '- No explicit source labels provided' + + const contextBlock = contextChunks.length > 0 + ? contextChunks + .map((chunk, index) => ( + `Context ${index + 1}: ${chunk.sourceLabel}\nTitle: ${chunk.title}\nExcerpt: ${chunk.content}` + )) + .join('\n\n') + : 'No context excerpts were provided.' + + return [ + 'You are rewriting a Bible-study chatbot answer.', + 'Hard rules:', + '1) Use ONLY facts in DRAFT ANSWER and CONTEXT EXCERPTS.', + '2) If DRAFT ANSWER is uncertain or says content is missing, keep that uncertainty and do not fill gaps.', + '3) Do not add verses, names, history, or claims not present in the provided text.', + '4) Keep a warm pastoral tone but concise.', + '5) Return plain text only. No markdown headers, no bullets unless needed for clarity.', + '', + `QUESTION:\n${question}`, + '', + `DRAFT ANSWER:\n${draftAnswer}`, + '', + `SOURCES:\n${sourceList}`, + '', + `CONTEXT EXCERPTS:\n${contextBlock}`, + ].join('\n') +} + +async function rewriteWithChatbotLlm({ question, draftAnswer, sources, contextChunks }) { + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), CHATBOT_LLM_TIMEOUT_MS) + + try { + const prompt = buildGroundedRewritePrompt({ + question, + draftAnswer, + sources, + contextChunks, + }) + + const response = await fetch(`${CHATBOT_LLM_BASE_URL.replace(/\/$/, '')}/api/chat`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + signal: controller.signal, + body: JSON.stringify({ + model: CHATBOT_LLM_MODEL, + stream: false, + options: { + temperature: 0.2, + num_ctx: CHATBOT_LLM_NUM_CTX, + }, + messages: [ + { + role: 'system', + content: 'Rewrite grounded answers faithfully. Never add information not present in the provided draft/context.', + }, + { + role: 'user', + content: prompt, + }, + ], + }), + }) + + if (!response.ok) { + const text = await response.text() + throw new Error(`LLM request failed (${response.status}): ${text.slice(0, 220)}`) + } + + const payload = await response.json() + const rewritten = typeof payload?.message?.content === 'string' + ? payload.message.content.trim() + : '' + + if (!rewritten) { + throw new Error('LLM returned an empty response') + } + + return rewritten.slice(0, 3500) + } finally { + clearTimeout(timeout) + } +} + // Public: return all chatbot entries for client-side matching app.get('/api/chatbot-content', async (req, res) => { await refreshChatbotFromDiskIfChanged() res.json(chatbotEntries) }) +// Optional: grounded rewrite endpoint for local Ollama usage. +app.post('/api/chatbot-grounded-rewrite', async (req, res) => { + if (!CHATBOT_LLM_ENABLED) { + res.status(503).json({ + ok: false, + message: 'Chatbot LLM is disabled. Set CHATBOT_LLM_ENABLED=true to enable.', + }) + return + } + + const question = typeof req.body?.question === 'string' ? req.body.question.trim() : '' + const draftAnswer = typeof req.body?.draftAnswer === 'string' ? req.body.draftAnswer.trim() : '' + const sources = Array.isArray(req.body?.sources) + ? req.body.sources.filter(source => typeof source === 'string').slice(0, 8) + : [] + const contextChunks = normalizeChatbotLlmContext(req.body?.contextChunks) + + if (!question) { + res.status(400).json({ ok: false, message: 'Missing required field: question' }) + return + } + + if (!draftAnswer) { + res.status(400).json({ ok: false, message: 'Missing required field: draftAnswer' }) + return + } + + try { + const rewrittenAnswer = await rewriteWithChatbotLlm({ + question, + draftAnswer, + sources, + contextChunks, + }) + + res.json({ + ok: true, + text: rewrittenAnswer, + model: CHATBOT_LLM_MODEL, + grounded: true, + }) + } catch (err) { + console.error('[chatbot-llm] rewrite failed:', err) + res.status(502).json({ + ok: false, + message: 'LLM rewrite failed. Returning deterministic draft is recommended.', + error: String(err), + }) + } +}) + // Admin: get all entries app.get('/api/admin/chatbot-content', async (req, res) => { if (!isValidAdminSession(req)) { res.status(401).json({ message: 'Not authenticated.' }); return } diff --git a/src/App.tsx b/src/App.tsx index 0358833..380d3db 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -295,6 +295,8 @@ interface IntentAnalysis { subject: string excludedSubject: string isCorrection: boolean + needsClarification: boolean + clarificationPrompt: string verseRefs: string[] episodeNumber: number | null isFollowUp: boolean @@ -700,6 +702,25 @@ function buildConversationOnlyReply(cue: ConversationCue, context: ChatContextSt return null } +function buildClarificationReply(analysis: IntentAnalysis, context: ChatContextState): ChatResponse { + const followupSuggestions = context.lastChunks.length > 0 + ? buildSmartFollowUpPrompts( + context.lastChunks, + analyzeIntent(context.lastPrompt || 'go deeper', context), + context.lastChunks[0]?.content ?? '', + ) + : [ + { label: 'Who was Titus?', prompt: 'Who was Titus?' }, + { label: 'Give me a summary of Titus 3:4-7', prompt: 'Give me a summary of Titus 3:4-7' }, + ] + + return { + text: `${analysis.clarificationPrompt}\n\nI want to answer from Nate's notes accurately, so one more detail will help me match the right content.`, + suggestions: followupSuggestions.slice(0, 3), + sources: context.lastChunks.length > 0 ? buildSourceCitations(context.lastChunks) : undefined, + } +} + function analyzeIntent(query: string, context: ChatContextState): IntentAnalysis { const rawQuery = query.trim() const normalized = rawQuery.toLowerCase() @@ -711,12 +732,32 @@ function analyzeIntent(query: string, context: ChatContextState): IntentAnalysis const wantsDefinition = /\bmean\b|\bmeaning\b|\bdefine\b|\bwhat is\b|\bwhat does\b/.test(normalized) const wantsIdentity = /\bwho is\b|\bwho was\b|\btell me about\b/.test(normalized) const wantsPractice = /\bhow can i\b|\bhow do i\b|\bhow should i\b|\bpray\b|\bpractice\b/.test(normalized) + const wantsComparison = /\bdifference\b|\bcompare\b|\bvs\.?\b|\bversus\b/.test(normalized) const correctionSubjects = extractCorrectionSubjects(normalized) const isFollowUp = isContextualFollowUp(rawQuery) || correctionSubjects !== null const explicitSubject = extractIdentitySubject(normalized) const subject = correctionSubjects?.expected || explicitSubject || (isFollowUp ? context.lastSubject : '') const excludedSubject = correctionSubjects?.rejected ?? '' const isCorrection = correctionSubjects !== null + const tokenCount = tokenize(rawQuery).length + const hasStandalonePronoun = /\b(this|that|it|he|she|they|them|those|these|him|her)\b/.test(normalized) + + let needsClarification = false + let clarificationPrompt = '' + + if (!isFollowUp && hasStandalonePronoun && context.turnCount === 0) { + needsClarification = true + clarificationPrompt = 'Can you name the specific person, verse, or episode you mean?' + } else if (!isFollowUp && tokenCount < 2 && verseRefs.length === 0 && episodeNumber === null) { + needsClarification = true + clarificationPrompt = 'Can you make that a bit more specific so I can match Nate\'s notes?' + } else if (/\bwho\s+(is|was)\b/.test(normalized) && !subject) { + needsClarification = true + clarificationPrompt = 'Who would you like to ask about specifically?' + } else if (wantsComparison && !/\band\b|\bbetween\b/.test(normalized)) { + needsClarification = true + clarificationPrompt = 'What two things would you like me to compare?' + } let type: ChatIntent = 'general' if (wantsEpisode) type = 'episode-lookup' @@ -727,6 +768,7 @@ function analyzeIntent(query: string, context: ChatContextState): IntentAnalysis else if (wantsPractice) type = 'practice' else if (wantsSummary) type = 'overview' else if (isFollowUp) type = 'follow-up' + else if (wantsComparison) type = 'overview' let searchQuery = rawQuery if (type === 'identity' && subject) { @@ -757,6 +799,8 @@ function analyzeIntent(query: string, context: ChatContextState): IntentAnalysis subject, excludedSubject, isCorrection, + needsClarification, + clarificationPrompt, verseRefs, episodeNumber, isFollowUp, @@ -969,15 +1013,23 @@ function buildLowConfidenceReply(scored: ScoredChunk[], analysis: IntentAnalysis const choices = getTopDistinctChunks(scored, 3) const topLabels = choices.slice(0, 2).map(chunk => chunk.sourceLabel) const text = topLabels.length === 2 - ? `I want to be accurate, so I need one quick clarification. Did you mean "${topLabels[0]}" or "${topLabels[1]}"?` - : `I want to be accurate, and I don't have enough confidence to answer yet. Pick the closest direction and I'll continue.` + ? `I don't want to make up an answer. I couldn't find a confident match in Nate's notes yet. Did you mean "${topLabels[0]}" or "${topLabels[1]}"?` + : `I don't want to make up an answer. I couldn't find this clearly in Nate's current notes yet. You can submit this question directly to Nate below.` + + const choiceSuggestions = choices.map(chunk => ({ + label: chunk.sourceLabel, + prompt: buildPromptForChunk(chunk, analysis), + })) return { text, - suggestions: choices.map(chunk => ({ - label: chunk.sourceLabel, - prompt: buildPromptForChunk(chunk, analysis), - })), + suggestions: [ + ...choiceSuggestions, + { + label: 'How do I submit a question to Nate?', + prompt: 'How do I submit a question to Nate?', + }, + ].slice(0, 3), sources: buildSourceCitations(choices), } } @@ -1036,10 +1088,10 @@ function buildSmartFallbackReply(analysis: IntentAnalysis, context: ChatContextS if (context.lastVerseRefs[0]) suggestions.add(`How does ${formatVerseRef(context.lastVerseRefs[0])} connect to the rest of the chapter?`) suggestions.add('What does Titus 1 teach about church leadership?') suggestions.add('What does grace train us to do?') - suggestions.add('How should I apply this today?') + suggestions.add('How do I submit a question to Nate?') return { - text: `I don't have a strong enough match yet to answer that clearly. Try one of these more specific prompts and I'll narrow it down.`, + text: `I don't want to make up an answer. I couldn't find this in Nate's current notes. Please submit this question to Nate using the contact form below.`, suggestions: [...suggestions].slice(0, 3).map(prompt => ({ label: prompt, prompt })), } } @@ -1280,6 +1332,41 @@ function ChatBot({ mode = 'embedded' }: { mode?: 'embedded' | 'standalone' }) { } } + const maybeRewriteWithLlm = async ( + question: string, + draft: ChatResponse, + contextChunks: SearchChunk[], + ): Promise => { + try { + const response = await fetch('/api/chatbot-grounded-rewrite', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + question, + draftAnswer: draft.text, + sources: draft.sources ?? [], + contextChunks: contextChunks.slice(0, 4).map(chunk => ({ + title: chunk.title, + sourceLabel: chunk.sourceLabel, + content: chunk.content, + })), + }), + }) + + if (!response.ok) return draft + const payload = await response.json() + const rewritten = typeof payload?.text === 'string' ? payload.text.trim() : '' + if (!rewritten) return draft + + return { + ...draft, + text: rewritten, + } + } catch { + return draft + } + } + const respond = async (query: string) => { const userMsg: ChatMessage = { role: 'user', text: query } setMessages(m => [...m, userMsg]) @@ -1293,6 +1380,7 @@ function ChatBot({ mode = 'embedded' }: { mode?: 'embedded' | 'standalone' }) { const contextualQuery = enrichQueryWithContext(trimmedQuery, activeContext) setTimeout(() => { + void (async () => { const cue = detectConversationCue(trimmedQuery) const cueReply = buildConversationOnlyReply(cue, activeContext, responseStyle) if (cueReply) { @@ -1327,6 +1415,18 @@ function ChatBot({ mode = 'embedded' }: { mode?: 'embedded' | 'standalone' }) { } const analysis = analyzeIntent(contextualQuery, activeContext) + if (analysis.needsClarification) { + const clarificationReply = buildClarificationReply(analysis, activeContext) + setMessages(m => [...m, { + role: 'bot', + text: formatResponseText(clarificationReply), + suggestions: clarificationReply.suggestions, + sources: clarificationReply.sources, + }]) + setLoading(false) + return + } + const bibleVersionIntent = isBibleVersionQuery(analysis.searchQuery, analysis.queryTokens) const scored = latestChunks .map(chunk => { @@ -1340,17 +1440,27 @@ function ChatBot({ mode = 'embedded' }: { mode?: 'embedded' | 'standalone' }) { .sort((a, b) => b.score - a.score) let botReply: ChatResponse + let topChunksForRewrite: SearchChunk[] = [] + let shouldTryLlmRewrite = false if (scored.length > 0) { + const confidence = determineConfidence(scored, analysis) + topChunksForRewrite = getTopDistinctChunks(scored, 3) botReply = synthesizeSmartReply(scored, analysis, activeContext, responseStyle) - if (determineConfidence(scored, analysis) !== 'low') { + if (confidence !== 'low') { + shouldTryLlmRewrite = true chatContextRef.current = buildNextChatContext(contextualQuery, analysis, scored, activeContext) } } else { botReply = buildSmartFallbackReply(analyzeIntent(contextualQuery, activeContext), activeContext) } + if (shouldTryLlmRewrite) { + botReply = await maybeRewriteWithLlm(trimmedQuery, botReply, topChunksForRewrite) + } + setMessages(m => [...m, { role: 'bot', text: formatResponseText(botReply), suggestions: botReply.suggestions, sources: botReply.sources }]) setLoading(false) + })() }, 400) }