100+ AI Agents, Agent Skills and RAG Apps - Free and Open Source.
fix(agentic_rag_math_agent): use few-shot guardrail, tolerant verdicts, robust web fallback
prince03101ee4f2db883b
diff --git a/rag_tutorials/agentic_rag_math_agent/app/benchmark.py b/rag_tutorials/agentic_rag_math_agent/app/benchmark.pyindex e20874f3..094d6085 100644--- a/rag_tutorials/agentic_rag_math_agent/app/benchmark.py+++ b/rag_tutorials/agentic_rag_math_agent/app/benchmark.py@@ -3,12 +3,23 @@ import sysimport ossys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))+import reimport pandas as pdimport timefrom datetime import datetimefrom rag.query_router import answer_math_questionfrom data.load_gsm8k_data import load_jeebench_dataset+def is_answer_correct(expected, response: str) -> bool:+ """Check that the gold answer appears as a standalone token in the response.++ A plain substring test counts gold "C" as correct for any reply containing+ the letter c, and gold "12" as correct inside "112" or "0.12".+ """+ pattern = rf"(?<![A-Za-z0-9.]){re.escape(str(expected).strip())}(?![A-Za-z0-9]|\.\d)"+ return re.search(pattern, response) is not None++def benchmark_math_agent(limit: int = 10):# ✅ Always filter math-only questionsdf = load_jeebench_dataset()@@ -25,7 +36,7 @@ def benchmark_math_agent(limit: int = 10):try:response = answer_math_question(question)- is_correct = expected.lower() in response.lower()+ is_correct = is_answer_correct(expected, response)if is_correct:correct += 1diff --git a/rag_tutorials/agentic_rag_math_agent/rag/guardrails.py b/rag_tutorials/agentic_rag_math_agent/rag/guardrails.pyindex ba6948ab..c0f29930 100644--- a/rag_tutorials/agentic_rag_math_agent/rag/guardrails.py+++ b/rag_tutorials/agentic_rag_math_agent/rag/guardrails.py@@ -1,5 +1,7 @@import dspyimport os+import re+from typing import Optionalfrom dotenv import load_dotenv# Load API key@@ -24,58 +26,77 @@ class ClassifyMath(dspy.Signature):+def parse_verdict(verdict) -> Optional[bool]:+ """Read a Yes/No verdict from the model.++ Accepts "Yes", "yes.", "**Yes**", "Yes, this is math." etc. Returns None+ when the reply is neither, so callers can log it instead of guessing.+ """+ match = re.match(r"\W*(yes|no)\b", str(verdict or ""), re.IGNORECASE)+ if not match:+ return None+ return match.group(1).lower() == "yes"+++# Few-shot examples for the input guard+MATH_CLASSIFIER_EXAMPLES = [+ ("What is the derivative of x^2?", "Yes"),+ ("Explain the chain rule in calculus.", "Yes"),+ ("Why do I need to learn algebra?", "Yes"),+ ("What is the Pythagorean theorem?", "Yes"),+ ("How do I solve a quadratic equation?", "Yes"),+ ("What is the area of a circle?", "Yes"),+ ("How is math used in real life?", "Yes"),+ ("What is the purpose of trigonometry?", "Yes"),+ ("What is the Fibonacci sequence?", "Yes"),+ ("can you tell me about rhombus?", "Yes"),+ ("what is a circle?", "Yes"),+ ("What is the formula for the area of a circle?", "Yes"),+ ("What is the formula for the circumference of a circle?", "Yes"),+ ("What is the formula for the volume of a cone?", "Yes"),+ ("What is the formula for the area of a parallelogram?", "Yes"),+ ("What is the formula for the area of a trapezoid?", "Yes"),+ ("What is the formula for the surface area of a cube?", "Yes"),+ ("What is the area of parallelogram?", "Yes"),+ ("What is a square?", "Yes"),+ ("Explain rectangle?", "Yes"),+ ("can you tell me about pentagon?", "Yes"),+ ("What is the formula for the volume of a sphere?", "Yes"),+ ("What is the formula for the area of a triangle?", "Yes"),+ ("What is the difference between a permutation and a combination?", "Yes"),+ ("What is the formula for the slope of a line?", "Yes"),+ ("What is the difference between a rational and irrational number?", "Yes"),+ ("What is the formula for the area of a rectangle?", "Yes"),+ ("What is the formula for the volume of a cylinder?", "Yes"),+ ("What is the formula for the surface area of a sphere?", "Yes"),+ ("What is the formula for the surface area of a cylinder?", "Yes"),+ ("What is the integral of sin(x)?", "Yes"),+ ("What is the difference between mean and median?", "Yes"),+ ("What is the quadratic formula?", "Yes"),+ ("Tell me a good movie to watch.", "No"),+ ("What is AI?", "No"),+]++# ✅ Input Validatorclass InputValidator(dspy.Module):def __init__(self):super().__init__()self.classifier = dspy.Predict(ClassifyMath)- self.validate_question = dspy.ChainOfThought(- ClassifyMath,- examples=[- {"question": "What is the derivative of x^2?", "verdict": "Yes"},- {"question": "Explain the chain rule in calculus.", "verdict": "Yes"},- {"question": "Why do I need to learn algebra?", "verdict": "Yes"},- {"question": "What is the Pythagorean theorem?", "verdict": "Yes"},- {"question": "How do I solve a quadratic equation?", "verdict": "Yes"},- {"question": "What is the area of a circle?", "verdict": "Yes"},- {"question": "How is math used in real life?", "verdict": "Yes"},- {"question": "What is the purpose of trigonometry?", "verdict": "Yes"},- {"question": "What is the Fibonacci sequence?", "verdict": "Yes"},- {"question": "can you tell me about rhombus?", "verdict": "Yes"},- {"question": "what is a circle?", "verdict": "Yes"},- {"question": "What is the formula for the area of a circle?", "verdict": "Yes"},- {"question": "What is the formula for the circumference of a circle?", "verdict": "Yes"},- {"question": "What is the formula for the volume of a cone?", "verdict": "Yes"},- {"question": "What is the formula for the area of a parallelogram?", "verdict": "Yes"},- {"question": "What is the formula for the area of a trapezoid?", "verdict": "Yes"},- {"question": "What is the formula for the surface area of a cube?", "verdict": "Yes"},- {"question": "What is the area of parallelogram?", "verdict": "Yes"},- {"question": "What is a square?", "verdict": "Yes"},- {"question": "Explain rectangle?", "verdict": "Yes"},- {"question": "can you tell me about pentagon?", "verdict": "Yes"},- {"question": "What is the formula for the volume of a sphere?", "verdict": "Yes"},- {"question": "What is the difference between a mean and median?", "verdict": "Yes"},- {"question": "What is the formula for the area of a triangle?", "verdict": "Yes"},- {"question": "What is the difference between a permutation and a combination?", "verdict": "Yes"},- {"question": "What is the formula for the slope of a line?", "verdict": "Yes"},- {"question": "What is the difference between a rational and irrational number?", "verdict": "Yes"},- {"question": "What is the formula for the area of a rectangle?", "verdict": "Yes"},- {"question": "What is the formula for the volume of a cylinder?", "verdict": "Yes"},- {"question": "What is the formula for the area of a trapezoid?", "verdict": "Yes"},- {"question": "What is the formula for the surface area of a sphere?", "verdict": "Yes"},- {"question": "What is the formula for the surface area of a cylinder?", "verdict": "Yes"},- {"question": "What is the integral of sin(x)?", "verdict": "Yes"},- {"question": "What is the difference between mean and median?", "verdict": "Yes"},- {"question": "What is the formula for the circumference of a circle?", "verdict": "Yes"},- {"question": "What is the quadratic formula?", "verdict": "Yes"},- {"question": "Tell me a good movie to watch.", "verdict": "No"},- {"question": "What is AI?", "verdict": "No"},- ]- )+ # Attach the examples as demos so they are actually sent to the model+ self.classifier.demos = [+ dspy.Example(question=q, verdict=v).with_inputs("question")+ for q, v in MATH_CLASSIFIER_EXAMPLES+ ]+def forward(self, question):response = self.classifier(question=question)print("🧠 InputValidator Response:", response.verdict)- return response.verdict.lower().strip() == "yes"+ is_math = parse_verdict(response.verdict)+ if is_math is None:+ print(f"⚠️ InputValidator got an unclear verdict {response.verdict!r}; treating as not math.")+ return False+ return is_math# ✅ Output Validator (no change unless needed)class OutputValidator(dspy.Module):@@ -95,7 +116,11 @@ class OutputValidator(dspy.Module):answer=answer)print("🧠 OutputValidator Response:", response.verdict)- return response.verdict.lower().strip() == "yes"+ is_valid = parse_verdict(response.verdict)+ if is_valid is None:+ print(f"⚠️ OutputValidator got an unclear verdict {response.verdict!r}; treating as failed.")+ return False+ return is_valid# Initialize validatorsinput_validator = InputValidator()diff --git a/rag_tutorials/agentic_rag_math_agent/rag/query_router.py b/rag_tutorials/agentic_rag_math_agent/rag/query_router.pyindex 8b1421ec..6f98020e 100644--- a/rag_tutorials/agentic_rag_math_agent/rag/query_router.py+++ b/rag_tutorials/agentic_rag_math_agent/rag/query_router.py@@ -9,6 +9,7 @@ import requestsimport openaiimport jsonimport inspect+from typing import Optionalfrom llama_index.core import StorageContext,load_index_from_storagefrom dotenv import load_dotenvfrom llama_index.vector_stores.qdrant import QdrantVectorStore@@ -48,7 +49,15 @@ def query_kb(question: str):return matched_text, similarity-def query_web(question: str):+WEB_SEARCH_TIMEOUT = 20 # seconds+++def query_web(question: str) -> Optional[str]:+ """Return Tavily's answer, or None if the search failed or found nothing."""+ if not TAVILY_API_KEY:+ print("⚠️ Web search skipped: TAVILY_API_KEY is not set")+ return None+url = "https://api.tavily.com/search"headers = {"Content-Type": "application/json"}payload = {@@ -58,11 +67,23 @@ def query_web(question: str):"include_answer": True,"include_raw_content": False}- response = requests.post(url, json=payload, headers=headers)- data = response.json()- return data.get("answer", "No answer found.")+ try:+ response = requests.post(url, json=payload, headers=headers, timeout=WEB_SEARCH_TIMEOUT)+ response.raise_for_status()+ data = response.json()+ except (requests.RequestException, ValueError) as e:+ print(f"⚠️ Web search failed: {e}")+ return None++ answer = data.get("answer") if isinstance(data, dict) else None+ if not isinstance(answer, str) or not answer.strip():+ print("⚠️ Web search returned no answer")+ return None+ return answer-def explain_with_openai(question: str, web_content: str):+def explain_with_openai(question: str, web_content: Optional[str]):+ if not web_content:+ web_content = "(No web content was found. Answer from your own math knowledge.)"prompt = f"""You are a friendly and precise math tutor.@@ -83,6 +104,13 @@ Only include valid math steps — do not guess or make up answers.return response.text+def answer_from_web(question: str):+ """Answer via the web fallback. Returns (answer, source label)."""+ web_content = query_web(question)+ source = "Web" if web_content else "Model only (web search unavailable)"+ return explain_with_openai(question, web_content), source++# query_kb returns a cosine similarity (higher = better); with OpenAI embeddings# almost any two English texts score well above 0, so `> 0.` accepted every query# and served an arbitrary KB answer as authoritative. Require a real match.@@ -96,7 +124,7 @@ def answer_math_question(question: str):return "⚠️ This assistant only answers math-related academic questions."answer = ""- from_kb = False+ source = "KB"try:kb_answer, similarity = query_kb(question)@@ -126,25 +154,28 @@ Use the KB content as your only source. Do not guess or recalculate.llm = OpenAI(api_key=OPENAI_API_KEY, model="gpt-4o")answer = llm.complete(prompt).text- from_kb = Trueelse:raise ValueError("Low similarity match or empty")except Exception as e:print("⚠️ Using Web fallback because:", e)- web_content = query_web(question)- answer = explain_with_openai(question, web_content)- from_kb = False+ answer, source = answer_from_web(question)- print(f"📦 Answer Source: {'KB' if from_kb else 'Web'}")+ print(f"📦 Answer Source: {source}")# Final Output Guardrail Checkif not output_validator.forward(question, answer):print("⚠️ Final answer failed validation — retrying with web content...")-- web_content = query_web(question)- answer = explain_with_openai(question, web_content)- from_kb = False+ answer, source = answer_from_web(question)+ print(f"📦 Answer Source: {source}")++ # Re-check the retry instead of returning it unvalidated+ if not output_validator.forward(question, answer):+ print("⚠️ Retried answer also failed validation.")+ answer = (+ "⚠️ This answer could not be verified by the output guardrail. "+ "Please double-check it.\n\n" + answer+ )return answer