nate/Siteforge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance chatbot conversation engine and add retrieval eval suite

Browse Source
This commit is contained in:
nmemmert
2026-04-14 10:36:22 -04:00
parent 36594d5e16
commit 50459b7234
8 changed files with 1451 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data/chatbot-eval.json
+58
View File
@@ -0,0 +1,58 @@
[
{ "query": "who was titus", "expectedEntryId": "cb_who_is_titus_001", "expectedTopK": 3 },
{ "query": "who is paul", "expectedEntryId": "881279c7-3ca4-436d-a584-19cc0a5a9bf3", "expectedTopK": 3 },
{ "query": "tell me about paul in titus", "expectedEntryId": "881279c7-3ca4-436d-a584-19cc0a5a9bf3", "expectedTopK": 4 },
{ "query": "who was saul before his conversion", "expectedEntryId": "167fa083-5438-44a5-ac72-6dd379e937f5", "expectedTopK": 4 },
{ "query": "who was titus and why was he in crete", "expectedEntryId": "cb_who_is_titus_001", "expectedTopK": 12 },
{ "query": "what bible translation does nate use", "expectedEntryId": "cb_001", "expectedTopK": 1 },
{ "query": "does the podcast use bsb", "expectedEntryId": "cb_001", "expectedTopK": 2 },
{ "query": "which bible version do you teach from", "expectedEntryId": "cb_001", "expectedTopK": 2 },
{ "query": "how can i stay consistent with daily bible reading", "expectedEntryId": "cb_003", "expectedTopK": 2 },
{ "query": "help me build a daily reading habit", "expectedEntryId": "cb_003", "expectedTopK": 3 },
{ "query": "i keep missing devotion time what should i do", "expectedEntryId": "cb_003", "expectedTopK": 4 },
{ "query": "how should christians engage with politics", "expectedEntryId": "cb_christians_politics_001", "expectedTopK": 2 },
{ "query": "what does titus say about public life", "expectedEntryId": "cb_christians_politics_001", "expectedTopK": 3 },
{ "query": "how can i be peaceable online", "expectedEntryId": "cb_christians_politics_001", "expectedTopK": 4 },
{ "query": "give me a summary of titus 3:4-7", "expectedEntryId": "cb_titus347_summary_001", "expectedTopK": 1 },
{ "query": "summarize titus 3 4 through 7", "expectedEntryId": "cb_titus347_summary_001", "expectedTopK": 12 },
{ "query": "what does titus 3:4-7 teach about salvation", "expectedEntryId": "b68c5659-cc7e-42d2-ab24-a623b7404058", "expectedTopK": 2 },
{ "query": "what is the blessed hope", "expectedEntryId": "cb_blessed_hope_001", "expectedTopK": 2 },
{ "query": "define blessed hope from titus 2:13", "expectedEntryId": "cb_blessed_hope_001", "expectedTopK": 2 },
{ "query": "what does makaria elpis mean", "expectedEntryId": "cb_blessed_hope_001", "expectedTopK": 3 },
{ "query": "what does grace train us to do", "expectedEntryId": "cb_grace_trains_001", "expectedTopK": 2 },
{ "query": "how does grace teach us to say no to ungodliness", "expectedEntryId": "cb_grace_trains_001", "expectedTopK": 3 },
{ "query": "what is paideuo in titus 2", "expectedEntryId": "6be7a644-9276-4797-a7bb-1422b170846c", "expectedTopK": 12 },
{ "query": "how do i submit a question to nate", "expectedEntryId": "cb_007", "expectedTopK": 1 },
{ "query": "where can i send bible questions", "expectedEntryId": "cb_007", "expectedTopK": 12 },
{ "query": "how do i contact nate", "expectedEntryId": "cb_007", "expectedTopK": 2 },
{ "query": "what does titus 1 teach about church leadership", "expectedEntryId": "cb_leadership_titus1_001", "expectedTopK": 2 },
{ "query": "elder qualifications in titus 1", "expectedEntryId": "cb_leadership_titus1_001", "expectedTopK": 12 },
{ "query": "what should an overseer be like", "expectedEntryId": "cb_leadership_titus1_001", "expectedTopK": 3 },
{ "query": "what did paul say about elders", "expectedEntryId": "e0ad268f-2826-41a5-b078-90f90aeda21b", "expectedTopK": 4 },
{ "query": "episode on faithful leaders", "expectedEntryId": "e0ad268f-2826-41a5-b078-90f90aeda21b", "expectedTopK": 3 },
{ "query": "titus 1:5-9 overview", "expectedEntryId": "e0ad268f-2826-41a5-b078-90f90aeda21b", "expectedTopK": 12 },
{ "query": "what are false teachers doing in titus", "expectedEntryId": "56626549-7e1e-4564-8c79-2d21b49eb911", "expectedTopK": 3 },
{ "query": "what is the circumcision group in titus", "expectedEntryId": "56626549-7e1e-4564-8c79-2d21b49eb911", "expectedTopK": 3 },
{ "query": "empty talk and deception in titus 1", "expectedEntryId": "56626549-7e1e-4564-8c79-2d21b49eb911", "expectedTopK": 4 },
{ "query": "what does it mean to deny god by your works", "expectedEntryId": "cb_deny_works_001", "expectedTopK": 2 },
{ "query": "they profess to know god but deny him by actions", "expectedEntryId": "cb_deny_works_001", "expectedTopK": 3 },
{ "query": "detestable disobedient unfit meaning", "expectedEntryId": "cb_deny_works_001", "expectedTopK": 4 },
{ "query": "episode about grace training us", "expectedEntryId": "6be7a644-9276-4797-a7bb-1422b170846c", "expectedTopK": 2 },
{ "query": "titus 2:11-12 episode", "expectedEntryId": "6be7a644-9276-4797-a7bb-1422b170846c", "expectedTopK": 12 },
{ "query": "how should i share faith with skeptical family", "expectedEntryId": "cb_faith_family_001", "expectedTopK": 2 },
{ "query": "evangelize skeptical relatives", "expectedEntryId": "cb_faith_family_001", "expectedTopK": 4 },
{ "query": "where can i listen to the podcast", "expectedEntryId": "cb_006", "expectedTopK": 2 },
{ "query": "how do i find verse by verse with nate on apple podcasts", "expectedEntryId": "cb_006", "expectedTopK": 3 },
{ "query": "what is verse by verse with nate podcast", "expectedEntryId": "cb_005", "expectedTopK": 2 },
{ "query": "tell me about the podcast", "expectedEntryId": "cb_005", "expectedTopK": 2 },
{ "query": "how do i approach a difficult passage", "expectedEntryId": "cb_002", "expectedTopK": 2 },
{ "query": "what should i do with confusing bible verses", "expectedEntryId": "cb_002", "expectedTopK": 3 },
{ "query": "prayer and quiet time advice", "expectedEntryId": "cb_008", "expectedTopK": 2 },
{ "query": "how to start a quiet time", "expectedEntryId": "cb_008", "expectedTopK": 3 },
{ "query": "what is episode 9 about", "expectedEntryId": "4e6ec41f-b083-45b9-9581-0dc91f092f66", "expectedTopK": 12 },
{ "query": "living between two appearings", "expectedEntryId": "4e6ec41f-b083-45b9-9581-0dc91f092f66", "expectedTopK": 2 },
{ "query": "what is episode 12", "expectedEntryId": "b68c5659-cc7e-42d2-ab24-a623b7404058", "expectedTopK": 2 },
{ "query": "the gospel in one paragraph", "expectedEntryId": "b68c5659-cc7e-42d2-ab24-a623b7404058", "expectedTopK": 2 },
{ "query": "what is episode 14 about", "expectedEntryId": "14ced988-1382-4c8d-a29e-b0f531580af1", "expectedTopK": 3 },
{ "query": "grace where it starts and where it ends", "expectedEntryId": "14ced988-1382-4c8d-a29e-b0f531580af1", "expectedTopK": 3 }
]