From 6ab158eaf3cd7a404ebf09ceaf6f62caddc90b8d Mon Sep 17 00:00:00 2001 From: Haritha Lakshmi Date: Mon, 19 Oct 2026 16:01:59 +0530 Subject: [PATCH 1/2] Add langchain-flan example (Harithalakshmikt) --- .../.gitignore | 27 ++ .../RESULTS.md | 66 +++++ .../run_example.py | 260 ++++++++++++++++++ 3 files changed, 353 insertions(+) create mode 100644 examples/Harithalakshmikt_langchain_flan/.gitignore create mode 100644 examples/Harithalakshmikt_langchain_flan/RESULTS.md create mode 100644 examples/Harithalakshmikt_langchain_flan/run_example.py diff --git a/examples/Harithalakshmikt_langchain_flan/.gitignore b/examples/Harithalakshmikt_langchain_flan/.gitignore new file mode 100644 index 0000000..06c0fbe --- /dev/null +++ b/examples/Harithalakshmikt_langchain_flan/.gitignore @@ -0,0 +1,27 @@ + +# Python +__pycache__/ +*.py[cod] +*.pyo + +# Virtual environments +.venv/ +venv/ +env/ + +# Environment variables and secrets +.env +.env.* + +# Logs +*.log + +# Local vector indexes and caches +faiss_index/ +.cache/ +.cache_huggingface/ + +# Test and coverage output +.pytest_cache/ +.coverage +htmlcov/ diff --git a/examples/Harithalakshmikt_langchain_flan/RESULTS.md b/examples/Harithalakshmikt_langchain_flan/RESULTS.md new file mode 100644 index 0000000..d756e3f --- /dev/null +++ b/examples/Harithalakshmikt_langchain_flan/RESULTS.md @@ -0,0 +1,66 @@ +# BOOTH + LangChain + FAISS + FLAN-T5-base + +## Setup +- **Generator:** google/flan-t5-base, run locally via transformers (no API key) +- **Retrieval:** LangChain + FAISS, embeddings from sentence-transformers/all-MiniLM-L6-v2 +- **Evidence comparison:** cross-encoder/nli-deberta-v3-base +- **BOOTH function used:** `check_with_evidence()` with `evidence_threshold=0.70` +- **Date run:** 2026-10-19 +- **BOOTH version:** boothpy 0.5.4 +- **Other versions:** langchain-text-splitters 1.1.3, langchain-community 0.4.2, faiss-cpu 1.15.1, transformers 5.20.0, sentence-transformers 6.1.0, torch 2.14.1 +- **Python / OS:** Python 3.11.0 / Windows (version 26H2) + +## What the example does +Five short library-policy sentences are defined inline and indexed in FAISS. For a question, the top-3 chunks are retrieved and FLAN-T5 answers from them. BOOTH then checks that answer against the same retrieved chunks using my own NLI-based `compare_fn`. After each question, the script also checks two hardcoded answers (one wrong, one correct) for the same question, to see how the checker treats a simulated hallucination and a correct full sentence. + +BOOTH only validates here. The answer comes from FLAN-T5 (or is hardcoded), and the score comes from my `compare_fn`. + +## compare_fn used +An NLI cross-encoder scores whether the evidence entails each sentence of the answer. + +1. The answer is split into sentences (regex on `.`, `!`, `?`), and each sentence is a claim. +2. Each claim is paired with each evidence passage (premise = evidence, hypothesis = claim). +3. Entailment probability is computed with a softmax over the logits. +4. Each claim takes its best score across passages, and the final score is the lowest claim score. + +**Weak spots:** +- Entailment is an estimate of support, not proof of truth. +- NLI models expect full-sentence hypotheses. Short answers like "No" or "9 AM to 5 PM" score near zero. +- The question is never given to the NLI model, so a fragment loses the context that gives it meaning. +- Claims that need a small inference (closing time from opening hours) are scored low. +- Sentence splitting is naive. +- The 0.70 threshold was not tuned, and only a few checks were run. + +## Runs +| # | Question | Answer | Agreement | Status | Answer quality | Expected? | +|---|----------|--------|-----------|--------|----------------|-----------| +| 1 | What time does the library close on weekdays? | 9 AM to 5 PM (FLAN-T5) | 0.0013 | BLOCKED | Imprecise: gives opening hours, not the closing time | Debatable | +| 2 | How many books can a member borrow? | up to five (FLAN-T5) | 0.882 | ACCEPTED | Correct | Yes | +| 3 | Can members borrow ten books at a time? | No (FLAN-T5) | 0.0047 | BLOCKED | Correct | **No (false block)** | +| 4 | What time does the library close on weekdays? | The library closes at 9 PM on weekdays. (hardcoded, simulates a hallucination) | 0.0043 | BLOCKED | Wrong | Yes | +| 5 | What time does the library close on weekdays? | The library closes at 5 PM on weekdays. (hardcoded control) | 0.168 | BLOCKED | Correct | **No (false block)** | + +**Total:** 5 checks: 1 accepted, 4 blocked. The deliberately wrong answer was blocked as intended. Two correct answers (runs 3 and 5) were also blocked. No wrong answer was accepted, and the only acceptance was a correct one. Scores were identical across repeated runs of the same input, so scoring is deterministic. + +## What worked well +- `check_with_evidence()` was easy to connect to a local model, and the `compare_fn` contract is simple to satisfy. +- Status, score and detail make it clear why an answer was blocked (for example `score 0.0013 below evidence_threshold 0.7`). +- The deliberately wrong answer was BLOCKED, and "up to five" was ACCEPTED at 0.88. +- The scores rank the correct sentence above the wrong one (0.168 vs 0.0043), so the NLI model has some signal. +- Everything runs locally with no API key. + +## Problems and edge cases +- **Over-strict compare_fn.** At threshold 0.70, only a near-verbatim answer ("up to five", 0.88) passed. A correct full sentence that needs a small inference scored 0.168 and was blocked. A likely cause is that the evidence says "open from 9 AM to 5 PM" while the claim says "closes at 5 PM". Another possible cause is the five-sentence premise diluting the signal. Neither was tested. +- **Terse answers.** FLAN-T5 tends to answer with fragments. "No" (run 3) is correct, but no passage entails a bare "No", so it was blocked. +- **Generator imprecision (run 1).** FLAN-T5 returned the opening hours instead of the closing time. BOOTH blocked it, but the score (0.0013) is almost the same as for the wrong sentence (0.0043). The `compare_fn` penalizes any fragment, so it cannot tell an imprecise answer from a wrong one, and the block was not due to the checker understanding the problem. +- **Weak separation at the low end.** The scores are ordered correctly, but at a 0.70 threshold a score of 0.168 and a score of 0.004 get the same BLOCKED outcome. +- **Single evidence passage.** k=3 returned one passage in every run, because the whole knowledge base fits in one 300-character chunk. +- **Tiny sample.** 5 checks on one 5-sentence knowledge base. Nothing here supports a claim about general accuracy. +- **Not tried:** scoring against single sentences instead of whole chunks, rewriting short answers into full sentences before comparing, and tuning the threshold. I did not lower the threshold, to avoid tuning to a handful of examples. +- The "Decision" line printed by the script is my own wording, not a BOOTH field. + +## Tips for reproducing +- Install and run commands are in the `run_example.py` docstring. +- The first run downloads FLAN-T5-base, MiniLM and the NLI model, so it needs internet and some disk space. +- Type a real question when prompted. An empty question exits before the checks run. +- If correct short answers are blocked, try rewriting them as full sentences before comparing, using sentence-level premises, or adjusting the threshold. \ No newline at end of file diff --git a/examples/Harithalakshmikt_langchain_flan/run_example.py b/examples/Harithalakshmikt_langchain_flan/run_example.py new file mode 100644 index 0000000..43f96e3 --- /dev/null +++ b/examples/Harithalakshmikt_langchain_flan/run_example.py @@ -0,0 +1,260 @@ +""" +BOOTH evidence-checking example with a local RAG pipeline. + +Provider/model: +- Embeddings: sentence-transformers/all-MiniLM-L6-v2 +- Answer generation: google/flan-t5-base +- Evidence comparison: cross-encoder/nli-deberta-v3-base + +Run from the BOOTH repository root: + pip install -e . + pip install langchain-text-splitters langchain-community \ + sentence-transformers transformers torch faiss-cpu + python examples/Harithalakshmikt_langchain_flan/run_example.py + +No API key is required. The Hugging Face models are downloaded +on first use and then run locally. + +After your question is answered, the script also runs one deliberately +wrong answer through BOOTH to show that it gets BLOCKED. +""" + +import re + +try: + import numpy as np + import booth + + from langchain_text_splitters import RecursiveCharacterTextSplitter + from langchain_community.embeddings import HuggingFaceEmbeddings + from langchain_community.vectorstores import FAISS + from langchain_community.vectorstores.utils import DistanceStrategy + from sentence_transformers import CrossEncoder + from transformers import AutoModelForSeq2SeqLM, AutoTokenizer +except ImportError as exc: + raise SystemExit( + f"Missing dependency: {exc}.\n" + "Install the packages listed in this file's docstring, then " + "run the script again." + ) from exc + + +EMBEDDING_MODEL = "sentence-transformers/all-MiniLM-L6-v2" +GENERATION_MODEL = "google/flan-t5-base" +NLI_MODEL = "cross-encoder/nli-deberta-v3-base" +EVIDENCE_THRESHOLD = 0.70 + +# Small sample knowledge base; no extra data file is needed. +KNOWLEDGE_BASE = """ +The library is open from Monday to Friday, 9 AM to 5 PM. +The library is closed on Saturdays and Sundays. +Members can borrow up to five books at a time. +Books must be returned within fourteen days. +A member who returns a book late may need to pay a late fee. +""" + + +def build_vector_store(text): + """Split the notes and index them using FAISS.""" + splitter = RecursiveCharacterTextSplitter( + chunk_size=300, + chunk_overlap=100, + separators=["\n\n", "\n", ". ", " ", ""], + ) + chunks = splitter.split_text(text) + + embeddings = HuggingFaceEmbeddings( + model_name=EMBEDDING_MODEL, + encode_kwargs={"normalize_embeddings": True}, + ) + + return FAISS.from_texts( + chunks, + embeddings, + distance_strategy=DistanceStrategy.COSINE, + ) + + +class FlanGenerator: + """Generate an answer using FLAN-T5-base.""" + + def __init__(self): + self.tokenizer = AutoTokenizer.from_pretrained(GENERATION_MODEL) + self.model = AutoModelForSeq2SeqLM.from_pretrained(GENERATION_MODEL) + + def generate(self, context, question): + prompt = f"""Answer the question using only the context. +If the answer is not in the context, say "I don't know". + +Context: +{context} + +Question: {question} +Answer:""" + + inputs = self.tokenizer( + prompt, + return_tensors="pt", + truncation=True, + max_length=512, + ) + + outputs = self.model.generate( + **inputs, + max_new_tokens=150, + ) + + return self.tokenizer.decode( + outputs[0], + skip_special_tokens=True, + ).strip() + + +def make_compare_fn(): + """ + Return a comparison function for BOOTH. + + Each answer sentence is checked against the retrieved evidence. + The final score is the lowest sentence-level entailment score. + This is an estimate of support, not a guarantee of truth. + """ + nli_model = CrossEncoder(NLI_MODEL) + + labels = { + int(index): str(label).lower() + for index, label in nli_model.model.config.id2label.items() + } + entailment_index = next( + (index for index, label in labels.items() if "entail" in label), + None, + ) + + if entailment_index is None: + raise RuntimeError( + f"Could not identify the entailment label: {labels}" + ) + + def compare_fn(answer, evidence): + if not isinstance(answer, str) or not answer.strip(): + return 0.0 + + passages = [ + item.strip() + for item in evidence + if isinstance(item, str) and item.strip() + ] + if not passages: + return 0.0 + + claims = [ + sentence.strip() + for sentence in re.split(r"(?<=[.!?])\s+", answer.strip()) + if sentence.strip() + ] + if not claims: + return 0.0 + + # NLI takes (premise, hypothesis): evidence supports the claim. + pairs = [ + (passage, claim) + for claim in claims + for passage in passages + ] + + logits = np.asarray(nli_model.predict(pairs)) + logits = logits - logits.max(axis=1, keepdims=True) + probabilities = np.exp(logits) + probabilities /= probabilities.sum(axis=1, keepdims=True) + + entailment_scores = probabilities[:, entailment_index].reshape( + len(claims), len(passages) + ) + + # Each claim must have support from at least one retrieved passage. + best_score_per_claim = entailment_scores.max(axis=1) + return float(best_score_per_claim.min()) + + return compare_fn + + +def run_wrong_answer_demo(vector_store, compare_fn): + """Check one deliberately wrong and one correct hardcoded answer.""" + question = "What time does the library close on weekdays?" + + documents = vector_store.similarity_search(question, k=3) + evidence = [doc.page_content for doc in documents] + + answers = [ + ("wrong (hardcoded)", "The library closes at 9 PM on weekdays."), + ("correct (hardcoded)", "The library closes at 5 PM on weekdays."), + ] + + print("\n=== Hardcoded answer demo ===") + print("Question:", question) + + for label, answer in answers: + result = booth.check_with_evidence( + answer=answer, + evidence=evidence, + compare_fn=compare_fn, + evidence_threshold=EVIDENCE_THRESHOLD, + ) + print(f"\n[{label}] {answer}") + print("Status:", result.status) + print("Evidence agreement:", result.evidence_agreement) + print("Detail:", getattr(result, "detail", None)) + + +def main(): + vector_store = build_vector_store(KNOWLEDGE_BASE) + generator = FlanGenerator() + compare_fn = make_compare_fn() + + print("Local RAG + BOOTH evidence check") + print("Ask about the sample library notes.") + question = input("\nQuestion: ").strip() + + if not question: + print("Please enter a question.") + return + + # Retrieve the actual evidence chunks from FAISS. + documents = vector_store.similarity_search(question, k=3) + evidence = [doc.page_content for doc in documents] + context = "\n".join(evidence) + + print("\n--- Retrieved evidence ---") + for number, passage in enumerate(evidence, start=1): + print(f"{number}. {passage}") + + answer = generator.generate(context, question) + print("\n--- FLAN-T5 answer ---") + print(answer) + + # BOOTH checks the answer against the same chunks used by FLAN-T5. + result = booth.check_with_evidence( + answer=answer, + evidence=evidence, + compare_fn=compare_fn, + evidence_threshold=EVIDENCE_THRESHOLD, + ) + + print("\n--- BOOTH result ---") + print("Status:", result.status) + print("Evidence agreement:", result.evidence_agreement) + print("Detail:", getattr(result, "detail", None)) + + if result.status == booth.BLOCKED: + print("Decision: The answer was blocked by the evidence check.") + else: + print( + "Decision: Review the status and evidence agreement before " + "using the answer." + ) + + # Show that a deliberately wrong answer is blocked. + run_wrong_answer_demo(vector_store, compare_fn) + + +if __name__ == "__main__": + main() \ No newline at end of file From ef053b778e5fef8ff404cd20177b812a05e1f264 Mon Sep 17 00:00:00 2001 From: Haritha Lakshmi Date: Tue, 20 Oct 2026 15:36:16 +0530 Subject: [PATCH 2/2] Fix docstring, use pip install boothpy, add default question --- .../RESULTS.md | 4 +- .../run_example.py | 38 ++++++++++--------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/examples/Harithalakshmikt_langchain_flan/RESULTS.md b/examples/Harithalakshmikt_langchain_flan/RESULTS.md index d756e3f..b75169f 100644 --- a/examples/Harithalakshmikt_langchain_flan/RESULTS.md +++ b/examples/Harithalakshmikt_langchain_flan/RESULTS.md @@ -11,7 +11,7 @@ - **Python / OS:** Python 3.11.0 / Windows (version 26H2) ## What the example does -Five short library-policy sentences are defined inline and indexed in FAISS. For a question, the top-3 chunks are retrieved and FLAN-T5 answers from them. BOOTH then checks that answer against the same retrieved chunks using my own NLI-based `compare_fn`. After each question, the script also checks two hardcoded answers (one wrong, one correct) for the same question, to see how the checker treats a simulated hallucination and a correct full sentence. +Five short library-policy sentences are defined inline and indexed in FAISS. For a question, the top-3 chunks are retrieved and FLAN-T5 answers from them. BOOTH then checks that answer against the same retrieved chunks using my own NLI-based `compare_fn`. After that, the script checks two hardcoded answers (one wrong, one correct) to the default question "What time does the library close on weekdays?", whatever question was typed, to see how the checker treats a simulated hallucination and a correct full sentence. BOOTH only validates here. The answer comes from FLAN-T5 (or is hardcoded), and the score comes from my `compare_fn`. @@ -62,5 +62,5 @@ An NLI cross-encoder scores whether the evidence entails each sentence of the an ## Tips for reproducing - Install and run commands are in the `run_example.py` docstring. - The first run downloads FLAN-T5-base, MiniLM and the NLI model, so it needs internet and some disk space. -- Type a real question when prompted. An empty question exits before the checks run. +- Press Enter at the prompt to use the default question ("What time does the library close on weekdays?"), or type your own. - If correct short answers are blocked, try rewriting them as full sentences before comparing, using sentence-level premises, or adjusting the threshold. \ No newline at end of file diff --git a/examples/Harithalakshmikt_langchain_flan/run_example.py b/examples/Harithalakshmikt_langchain_flan/run_example.py index 43f96e3..5f710e3 100644 --- a/examples/Harithalakshmikt_langchain_flan/run_example.py +++ b/examples/Harithalakshmikt_langchain_flan/run_example.py @@ -1,4 +1,4 @@ -""" +r""" BOOTH evidence-checking example with a local RAG pipeline. Provider/model: @@ -6,17 +6,24 @@ - Answer generation: google/flan-t5-base - Evidence comparison: cross-encoder/nli-deberta-v3-base -Run from the BOOTH repository root: - pip install -e . - pip install langchain-text-splitters langchain-community \ +Install: + pip install boothpy langchain-text-splitters langchain-community \ sentence-transformers transformers torch faiss-cpu + +Run from the repository root: python examples/Harithalakshmikt_langchain_flan/run_example.py +At the prompt, press Enter to use the default question or type your own. +FLAN-T5 answers it from the retrieved notes, and BOOTH checks that answer +with check_with_evidence(). + +The script then checks two hardcoded answers to the default question +("What time does the library close on weekdays?"), whatever you typed: +one deliberately wrong (should be BLOCKED) and one correct full sentence +(a control). + No API key is required. The Hugging Face models are downloaded on first use and then run locally. - -After your question is answered, the script also runs one deliberately -wrong answer through BOOTH to show that it gets BLOCKED. """ import re @@ -43,6 +50,7 @@ GENERATION_MODEL = "google/flan-t5-base" NLI_MODEL = "cross-encoder/nli-deberta-v3-base" EVIDENCE_THRESHOLD = 0.70 +DEFAULT_QUESTION = "What time does the library close on weekdays?" # Small sample knowledge base; no extra data file is needed. KNOWLEDGE_BASE = """ @@ -177,10 +185,8 @@ def compare_fn(answer, evidence): return compare_fn -def run_wrong_answer_demo(vector_store, compare_fn): +def run_hardcoded_answer_demo(vector_store, compare_fn, question): """Check one deliberately wrong and one correct hardcoded answer.""" - question = "What time does the library close on weekdays?" - documents = vector_store.similarity_search(question, k=3) evidence = [doc.page_content for doc in documents] @@ -212,11 +218,9 @@ def main(): print("Local RAG + BOOTH evidence check") print("Ask about the sample library notes.") - question = input("\nQuestion: ").strip() - - if not question: - print("Please enter a question.") - return + question = input( + f"\nQuestion (press Enter for: {DEFAULT_QUESTION}): " + ).strip() or DEFAULT_QUESTION # Retrieve the actual evidence chunks from FAISS. documents = vector_store.similarity_search(question, k=3) @@ -252,8 +256,8 @@ def main(): "using the answer." ) - # Show that a deliberately wrong answer is blocked. - run_wrong_answer_demo(vector_store, compare_fn) + # Check one wrong and one correct hardcoded answer to the default question. + run_hardcoded_answer_demo(vector_store, compare_fn, DEFAULT_QUESTION) if __name__ == "__main__":