Make all site content editable from admin; add per-page content blocks
This commit is contained in:
nmemmert committed
1 parent
79eb72635e
commit
7c02062663
11 files changed
+1762
-2502
No files matched your search
@@ -1,92 +0,0 @@
|
||||
import { execSync } from 'child_process'
|
||||
import { readFileSync, writeFileSync } from 'fs'
|
||||
import { randomUUID } from 'crypto'
|
||||
|
||||
const BASE = "/Users/nate.emmert/Documents/github/Siteforge/Verse by Verse with Nate Complete Series"
|
||||
const CHATBOT_FILE = "/Users/nate.emmert/Documents/github/Siteforge/data/chatbot-content.json"
|
||||
|
||||
const FILES = [
|
||||
{ file: `${BASE}/Done/Verse_by_Verse_with_Nate_Episode02.docx`, ep: 2 },
|
||||
{ file: `${BASE}/Done/Verse_by_Verse_with_Nate_Episode03.docx`, ep: 3 },
|
||||
{ file: `${BASE}/Done/Verse_by_Verse_with_Nate_Episode04_Updated.docx`, ep: 4 },
|
||||
{ file: `${BASE}/Verse_by_Verse_with_Nate_Episode05.docx`, ep: 5 },
|
||||
{ file: `${BASE}/Verse_by_Verse_with_Nate_Episode06_expanded.docx`, ep: 6 },
|
||||
{ file: `${BASE}/Verse_by_Verse_with_Nate_Episode07.docx`, ep: 7 },
|
||||
{ file: `${BASE}/Verse_by_Verse_with_Nate_Episode08.docx`, ep: 8 },
|
||||
{ file: `${BASE}/Verse_by_Verse_with_Nate_Episode09.docx`, ep: 9 },
|
||||
{ file: `${BASE}/Verse_by_Verse_with_Nate_Episode10.docx`, ep: 10 },
|
||||
{ file: `${BASE}/Verse_by_Verse_with_Nate_Episode11.docx`, ep: 11 },
|
||||
{ file: `${BASE}/Verse_by_Verse_with_Nate_Episode12.docx`, ep: 12 },
|
||||
{ file: `${BASE}/Verse_by_Verse_with_Nate_Episode13.docx`, ep: 13 },
|
||||
{ file: `${BASE}/Verse_by_Verse_with_Nate_Episode14.docx`, ep: 14 },
|
||||
]
|
||||
|
||||
function extractText(filePath) {
|
||||
const xml = execSync(`unzip -p "${filePath}" word/document.xml 2>/dev/null`, { encoding: 'utf8' })
|
||||
return xml
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"').replace(/'/g, "'")
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
}
|
||||
|
||||
function parseEpisode(raw, epNum) {
|
||||
// Extract episode subtitle and passage reference from header
|
||||
const headerMatch = raw.match(/EPISODE\s+\d+\s*[—\-\u2013\u2014]+\s*(.+?)\s+(Titus\s+[\d:]+(?:\s*[\-\u2013\u2014]+\s*[\d:]+)?)\s*·/i)
|
||||
const subtitle = headerMatch ? headerMatch[1].trim().replace(/\s+/g, ' ') : ''
|
||||
const passageRef = headerMatch ? headerMatch[2].trim() : 'Titus'
|
||||
const episodeTitle = `Episode ${epNum} — ${subtitle || 'Verse by Verse with Nate'}`
|
||||
|
||||
// Find where the actual teaching content starts
|
||||
let contentStart = raw.indexOf('SEGMENT 1')
|
||||
if (contentStart === -1) contentStart = raw.indexOf('WHO WAS PAUL')
|
||||
if (contentStart === -1) contentStart = raw.indexOf('COLD OPEN')
|
||||
if (contentStart === -1) contentStart = 400
|
||||
|
||||
const rawContent = raw.slice(contentStart, contentStart + 4000)
|
||||
const content = rawContent
|
||||
.replace(/\[[^\]]{0,100}\]/g, '') // remove [stage directions]
|
||||
.replace(/[✝🎙️📖💬🧠💡🔑✅◀▶]/gu, '') // remove emoji
|
||||
.replace(/SEGMENT\s+\d+\s*[—\-\u2013]+\s*/g, '\n\n') // turn SEGMENT headers into breaks
|
||||
.replace(/\s{2,}/g, ' ')
|
||||
.trim()
|
||||
|
||||
// Build keyword list
|
||||
const verseRefs = [...new Set((raw.match(/Titus\s+\d+:\d+/g) || []))].slice(0, 5).map(k => k.toLowerCase())
|
||||
const titleWords = subtitle.toLowerCase().replace(/[^a-z0-9\s]/g, ' ').split(/\s+/).filter(w => w.length > 3)
|
||||
const keywords = [...new Set([
|
||||
'titus', `episode ${epNum}`, passageRef.toLowerCase(),
|
||||
...verseRefs, ...titleWords
|
||||
])].slice(0, 20)
|
||||
|
||||
return {
|
||||
id: randomUUID(),
|
||||
type: 'episode',
|
||||
title: `${episodeTitle} (${passageRef})`,
|
||||
content: content.slice(0, 3900),
|
||||
keywords,
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
}
|
||||
}
|
||||
|
||||
// Load existing entries (keep the 8 hand-written ones)
|
||||
const existing = JSON.parse(readFileSync(CHATBOT_FILE, 'utf8'))
|
||||
// Remove any previously generated episode entries to avoid duplication
|
||||
const baseEntries = existing.filter(e => e.type !== 'episode')
|
||||
|
||||
const newEntries = []
|
||||
for (const { file, ep } of FILES) {
|
||||
try {
|
||||
const raw = extractText(file)
|
||||
const entry = parseEpisode(raw, ep)
|
||||
newEntries.push(entry)
|
||||
console.log(`✓ Ep ${ep}: ${entry.title.slice(0, 80)}`)
|
||||
} catch (err) {
|
||||
console.error(`✗ Ep ${ep}: ${err.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
const combined = [...baseEntries, ...newEntries]
|
||||
writeFileSync(CHATBOT_FILE, JSON.stringify(combined, null, 2), 'utf8')
|
||||
console.log(`\nDone. ${newEntries.length} episode entries added. Total: ${combined.length}`)
|
||||
@@ -1,137 +0,0 @@
|
||||
import fs from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
|
||||
const root = process.cwd()
|
||||
const dataPath = path.join(root, 'data', 'chatbot-content.json')
|
||||
const evalPath = path.join(root, 'data', 'chatbot-eval.json')
|
||||
|
||||
const STOP_WORDS = new Set([
|
||||
'a','an','the','is','are','was','were','be','been','being','have','has','had','do','does','did',
|
||||
'will','would','could','should','may','might','shall','can','i','you','he','she','it','we','they',
|
||||
'me','him','her','us','them','my','your','his','its','our','their','this','that','these','those',
|
||||
'and','but','or','nor','so','yet','for','of','in','on','at','to','from','with','by','about',
|
||||
'what','how','why','when','where','who','which','if','then','than','as','just','not',
|
||||
])
|
||||
|
||||
const TOKEN_ALIASES = {
|
||||
bible: ['translation', 'version', 'scripture', 'bsb', 'berean'],
|
||||
translation: ['version', 'bsb', 'berean', 'bible'],
|
||||
version: ['translation', 'bsb', 'berean', 'bible'],
|
||||
elders: ['elder', 'leadership', 'leaders', 'overseer', 'pastor'],
|
||||
leadership: ['elders', 'elder', 'overseer', 'leaders'],
|
||||
grace: ['salvation', 'saved', 'godliness', 'mercy'],
|
||||
salvation: ['saved', 'grace', 'mercy', 'gospel'],
|
||||
saved: ['salvation', 'grace', 'mercy', 'gospel'],
|
||||
hope: ['blessed', 'appearing', 'return', 'coming'],
|
||||
politics: ['public', 'government', 'authorities'],
|
||||
}
|
||||
|
||||
function tokenize(text) {
|
||||
return text
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9\s]/g, ' ')
|
||||
.split(/\s+/)
|
||||
.filter(token => token.length > 2 && !STOP_WORDS.has(token))
|
||||
}
|
||||
|
||||
function expandTokens(tokens) {
|
||||
const expanded = new Set(tokens)
|
||||
for (const token of tokens) {
|
||||
const aliases = TOKEN_ALIASES[token] ?? []
|
||||
for (const alias of aliases) {
|
||||
for (const aliasToken of tokenize(alias)) expanded.add(aliasToken)
|
||||
}
|
||||
}
|
||||
return [...expanded]
|
||||
}
|
||||
|
||||
function literalTerms(text) {
|
||||
return [...new Set(
|
||||
text
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9:\-\s']/g, ' ')
|
||||
.split(/\s+/)
|
||||
.map(term => term.trim())
|
||||
.filter(term => term.length >= 2 && !STOP_WORDS.has(term)),
|
||||
)]
|
||||
}
|
||||
|
||||
function scoreEntry(entry, query) {
|
||||
const indexText = `${entry.title} ${entry.content} ${entry.keywords.join(' ')}`.toLowerCase()
|
||||
const queryTokens = expandTokens(tokenize(query))
|
||||
const terms = literalTerms(query)
|
||||
|
||||
let score = 0
|
||||
|
||||
for (const token of queryTokens) {
|
||||
if (entry.title.toLowerCase().includes(token)) score += 4
|
||||
else if (indexText.includes(token)) score += 2
|
||||
}
|
||||
|
||||
for (const term of terms) {
|
||||
if (entry.title.toLowerCase().includes(term)) score += 2
|
||||
else if (indexText.includes(term)) score += 1
|
||||
}
|
||||
|
||||
const phrase = query.toLowerCase().replace(/[^a-z0-9:\-\s']/g, ' ').replace(/\s+/g, ' ').trim()
|
||||
if (phrase.length > 6) {
|
||||
if (entry.title.toLowerCase().includes(phrase)) score += 10
|
||||
else if (indexText.includes(phrase)) score += 6
|
||||
}
|
||||
|
||||
return score
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const [contentRaw, evalRaw] = await Promise.all([
|
||||
fs.readFile(dataPath, 'utf8'),
|
||||
fs.readFile(evalPath, 'utf8'),
|
||||
])
|
||||
|
||||
const entries = JSON.parse(contentRaw)
|
||||
const tests = JSON.parse(evalRaw)
|
||||
|
||||
let pass = 0
|
||||
const failures = []
|
||||
|
||||
for (const test of tests) {
|
||||
const ranked = entries
|
||||
.map(entry => ({ entry, score: scoreEntry(entry, test.query) }))
|
||||
.sort((a, b) => b.score - a.score)
|
||||
|
||||
const topK = ranked.slice(0, test.expectedTopK)
|
||||
const hit = topK.some(item => item.entry.id === test.expectedEntryId)
|
||||
|
||||
if (hit) {
|
||||
pass += 1
|
||||
continue
|
||||
}
|
||||
|
||||
failures.push({
|
||||
query: test.query,
|
||||
expectedEntryId: test.expectedEntryId,
|
||||
expectedTopK: test.expectedTopK,
|
||||
actualTop: topK.map(item => ({ id: item.entry.id, title: item.entry.title, score: item.score })),
|
||||
})
|
||||
}
|
||||
|
||||
const total = tests.length
|
||||
const pct = ((pass / total) * 100).toFixed(1)
|
||||
|
||||
console.log(`Chatbot eval: ${pass}/${total} (${pct}%) passed`)
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.log('\nFailures:')
|
||||
for (const failure of failures) {
|
||||
console.log(`- Query: ${failure.query}`)
|
||||
console.log(` Expected: ${failure.expectedEntryId} in top ${failure.expectedTopK}`)
|
||||
console.log(` Actual: ${failure.actualTop.map(item => `${item.id} (${item.score})`).join(', ')}`)
|
||||
}
|
||||
process.exitCode = 1
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(error => {
|
||||
console.error(error)
|
||||
process.exitCode = 1
|
||||
})
|
||||
@@ -1,26 +0,0 @@
|
||||
import { readFileSync, writeFileSync } from 'fs'
|
||||
|
||||
const path = '/Users/nate.emmert/Documents/github/Siteforge/data/chatbot-content.json'
|
||||
const data = JSON.parse(readFileSync(path, 'utf8'))
|
||||
|
||||
const fixes = {
|
||||
2: { title: 'Episode 2 — Introduction to Titus (Background & Overview)', extra: ['introduction', 'background', 'overview', 'crete', 'letter'] },
|
||||
5: { title: 'Episode 5 — The Danger of Empty Words (Titus 1:10–13a)', extra: ['danger', 'empty', 'words', 'false', 'teacher', 'titus 1:10', 'titus 1:13'] },
|
||||
6: { title: 'Episode 6 — Words That Deny What We Claim to Believe (Titus 1:13b–16)', extra: ['deny', 'claim', 'believe', 'titus 1:13', 'titus 1:16'] },
|
||||
14: { title: 'Episode 14 — Grace: Where It Starts and Where It Ends (Titus 3:12–15)', extra: ['grace', 'starts', 'ends', 'review', 'titus 3:12', 'titus 3:15'] },
|
||||
}
|
||||
|
||||
let count = 0
|
||||
for (const entry of data) {
|
||||
const m = entry.title.match(/^Episode (\d+)/)
|
||||
if (!m) continue
|
||||
const ep = Number(m[1])
|
||||
if (fixes[ep]) {
|
||||
entry.title = fixes[ep].title
|
||||
entry.keywords = [...new Set([...entry.keywords, ...fixes[ep].extra])].slice(0, 20)
|
||||
count++
|
||||
}
|
||||
}
|
||||
|
||||
writeFileSync(path, JSON.stringify(data, null, 2), 'utf8')
|
||||
console.log(`Fixed ${count} entries. Total: ${data.length}`)
|
||||
Reference in new issue
Block a user