Skip to content

100+ AI Agents, Agent Skills and RAG Apps - Free and Open Source.

fix(agentic_rag_math_agent): use few-shot guardrail, tolerant verdicts, robust web fallback

prince03101ee4f2db883b
diff --git a/rag_tutorials/agentic_rag_math_agent/app/benchmark.py b/rag_tutorials/agentic_rag_math_agent/app/benchmark.py
index e20874f3..094d6085 100644
--- a/rag_tutorials/agentic_rag_math_agent/app/benchmark.py
+++ b/rag_tutorials/agentic_rag_math_agent/app/benchmark.py
@@ -3,12 +3,23 @@ import sys
import os
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+import re
import pandas as pd
import time
from datetime import datetime
from rag.query_router import answer_math_question
from data.load_gsm8k_data import load_jeebench_dataset
+def is_answer_correct(expected, response: str) -> bool:
+ """Check that the gold answer appears as a standalone token in the response.
+
+ A plain substring test counts gold "C" as correct for any reply containing
+ the letter c, and gold "12" as correct inside "112" or "0.12".
+ """
+ pattern = rf"(?<![A-Za-z0-9.]){re.escape(str(expected).strip())}(?![A-Za-z0-9]|\.\d)"
+ return re.search(pattern, response) is not None
+
+
def benchmark_math_agent(limit: int = 10):
# ✅ Always filter math-only questions
df = load_jeebench_dataset()
@@ -25,7 +36,7 @@ def benchmark_math_agent(limit: int = 10):
try:
response = answer_math_question(question)
- is_correct = expected.lower() in response.lower()
+ is_correct = is_answer_correct(expected, response)
if is_correct:
correct += 1
diff --git a/rag_tutorials/agentic_rag_math_agent/rag/guardrails.py b/rag_tutorials/agentic_rag_math_agent/rag/guardrails.py
index ba6948ab..c0f29930 100644
--- a/rag_tutorials/agentic_rag_math_agent/rag/guardrails.py
+++ b/rag_tutorials/agentic_rag_math_agent/rag/guardrails.py
@@ -1,5 +1,7 @@
import dspy
import os
+import re
+from typing import Optional
from dotenv import load_dotenv
# Load API key
@@ -24,58 +26,77 @@ class ClassifyMath(dspy.Signature):
+def parse_verdict(verdict) -> Optional[bool]:
+ """Read a Yes/No verdict from the model.
+
+ Accepts "Yes", "yes.", "**Yes**", "Yes, this is math." etc. Returns None
+ when the reply is neither, so callers can log it instead of guessing.
+ """
+ match = re.match(r"\W*(yes|no)\b", str(verdict or ""), re.IGNORECASE)
+ if not match:
+ return None
+ return match.group(1).lower() == "yes"
+
+
+# Few-shot examples for the input guard
+MATH_CLASSIFIER_EXAMPLES = [
+ ("What is the derivative of x^2?", "Yes"),
+ ("Explain the chain rule in calculus.", "Yes"),
+ ("Why do I need to learn algebra?", "Yes"),
+ ("What is the Pythagorean theorem?", "Yes"),
+ ("How do I solve a quadratic equation?", "Yes"),
+ ("What is the area of a circle?", "Yes"),
+ ("How is math used in real life?", "Yes"),
+ ("What is the purpose of trigonometry?", "Yes"),
+ ("What is the Fibonacci sequence?", "Yes"),
+ ("can you tell me about rhombus?", "Yes"),
+ ("what is a circle?", "Yes"),
+ ("What is the formula for the area of a circle?", "Yes"),
+ ("What is the formula for the circumference of a circle?", "Yes"),
+ ("What is the formula for the volume of a cone?", "Yes"),
+ ("What is the formula for the area of a parallelogram?", "Yes"),
+ ("What is the formula for the area of a trapezoid?", "Yes"),
+ ("What is the formula for the surface area of a cube?", "Yes"),
+ ("What is the area of parallelogram?", "Yes"),
+ ("What is a square?", "Yes"),
+ ("Explain rectangle?", "Yes"),
+ ("can you tell me about pentagon?", "Yes"),
+ ("What is the formula for the volume of a sphere?", "Yes"),
+ ("What is the formula for the area of a triangle?", "Yes"),
+ ("What is the difference between a permutation and a combination?", "Yes"),
+ ("What is the formula for the slope of a line?", "Yes"),
+ ("What is the difference between a rational and irrational number?", "Yes"),
+ ("What is the formula for the area of a rectangle?", "Yes"),
+ ("What is the formula for the volume of a cylinder?", "Yes"),
+ ("What is the formula for the surface area of a sphere?", "Yes"),
+ ("What is the formula for the surface area of a cylinder?", "Yes"),
+ ("What is the integral of sin(x)?", "Yes"),
+ ("What is the difference between mean and median?", "Yes"),
+ ("What is the quadratic formula?", "Yes"),
+ ("Tell me a good movie to watch.", "No"),
+ ("What is AI?", "No"),
+]
+
+
# ✅ Input Validator
class InputValidator(dspy.Module):
def __init__(self):
super().__init__()
self.classifier = dspy.Predict(ClassifyMath)
- self.validate_question = dspy.ChainOfThought(
- ClassifyMath,
- examples=[
- {"question": "What is the derivative of x^2?", "verdict": "Yes"},
- {"question": "Explain the chain rule in calculus.", "verdict": "Yes"},
- {"question": "Why do I need to learn algebra?", "verdict": "Yes"},
- {"question": "What is the Pythagorean theorem?", "verdict": "Yes"},
- {"question": "How do I solve a quadratic equation?", "verdict": "Yes"},
- {"question": "What is the area of a circle?", "verdict": "Yes"},
- {"question": "How is math used in real life?", "verdict": "Yes"},
- {"question": "What is the purpose of trigonometry?", "verdict": "Yes"},
- {"question": "What is the Fibonacci sequence?", "verdict": "Yes"},
- {"question": "can you tell me about rhombus?", "verdict": "Yes"},
- {"question": "what is a circle?", "verdict": "Yes"},
- {"question": "What is the formula for the area of a circle?", "verdict": "Yes"},
- {"question": "What is the formula for the circumference of a circle?", "verdict": "Yes"},
- {"question": "What is the formula for the volume of a cone?", "verdict": "Yes"},
- {"question": "What is the formula for the area of a parallelogram?", "verdict": "Yes"},
- {"question": "What is the formula for the area of a trapezoid?", "verdict": "Yes"},
- {"question": "What is the formula for the surface area of a cube?", "verdict": "Yes"},
- {"question": "What is the area of parallelogram?", "verdict": "Yes"},
- {"question": "What is a square?", "verdict": "Yes"},
- {"question": "Explain rectangle?", "verdict": "Yes"},
- {"question": "can you tell me about pentagon?", "verdict": "Yes"},
- {"question": "What is the formula for the volume of a sphere?", "verdict": "Yes"},
- {"question": "What is the difference between a mean and median?", "verdict": "Yes"},
- {"question": "What is the formula for the area of a triangle?", "verdict": "Yes"},
- {"question": "What is the difference between a permutation and a combination?", "verdict": "Yes"},
- {"question": "What is the formula for the slope of a line?", "verdict": "Yes"},
- {"question": "What is the difference between a rational and irrational number?", "verdict": "Yes"},
- {"question": "What is the formula for the area of a rectangle?", "verdict": "Yes"},
- {"question": "What is the formula for the volume of a cylinder?", "verdict": "Yes"},
- {"question": "What is the formula for the area of a trapezoid?", "verdict": "Yes"},
- {"question": "What is the formula for the surface area of a sphere?", "verdict": "Yes"},
- {"question": "What is the formula for the surface area of a cylinder?", "verdict": "Yes"},
- {"question": "What is the integral of sin(x)?", "verdict": "Yes"},
- {"question": "What is the difference between mean and median?", "verdict": "Yes"},
- {"question": "What is the formula for the circumference of a circle?", "verdict": "Yes"},
- {"question": "What is the quadratic formula?", "verdict": "Yes"},
- {"question": "Tell me a good movie to watch.", "verdict": "No"},
- {"question": "What is AI?", "verdict": "No"},
- ]
- )
+ # Attach the examples as demos so they are actually sent to the model
+ self.classifier.demos = [
+ dspy.Example(question=q, verdict=v).with_inputs("question")
+ for q, v in MATH_CLASSIFIER_EXAMPLES
+ ]
+
def forward(self, question):
response = self.classifier(question=question)
print("🧠 InputValidator Response:", response.verdict)
- return response.verdict.lower().strip() == "yes"
+ is_math = parse_verdict(response.verdict)
+ if is_math is None:
+ print(f"⚠️ InputValidator got an unclear verdict {response.verdict!r}; treating as not math.")
+ return False
+ return is_math
# ✅ Output Validator (no change unless needed)
class OutputValidator(dspy.Module):
@@ -95,7 +116,11 @@ class OutputValidator(dspy.Module):
answer=answer
)
print("🧠 OutputValidator Response:", response.verdict)
- return response.verdict.lower().strip() == "yes"
+ is_valid = parse_verdict(response.verdict)
+ if is_valid is None:
+ print(f"⚠️ OutputValidator got an unclear verdict {response.verdict!r}; treating as failed.")
+ return False
+ return is_valid
# Initialize validators
input_validator = InputValidator()
diff --git a/rag_tutorials/agentic_rag_math_agent/rag/query_router.py b/rag_tutorials/agentic_rag_math_agent/rag/query_router.py
index 8b1421ec..6f98020e 100644
--- a/rag_tutorials/agentic_rag_math_agent/rag/query_router.py
+++ b/rag_tutorials/agentic_rag_math_agent/rag/query_router.py
@@ -9,6 +9,7 @@ import requests
import openai
import json
import inspect
+from typing import Optional
from llama_index.core import StorageContext,load_index_from_storage
from dotenv import load_dotenv
from llama_index.vector_stores.qdrant import QdrantVectorStore
@@ -48,7 +49,15 @@ def query_kb(question: str):
return matched_text, similarity
-def query_web(question: str):
+WEB_SEARCH_TIMEOUT = 20 # seconds
+
+
+def query_web(question: str) -> Optional[str]:
+ """Return Tavily's answer, or None if the search failed or found nothing."""
+ if not TAVILY_API_KEY:
+ print("⚠️ Web search skipped: TAVILY_API_KEY is not set")
+ return None
+
url = "https://api.tavily.com/search"
headers = {"Content-Type": "application/json"}
payload = {
@@ -58,11 +67,23 @@ def query_web(question: str):
"include_answer": True,
"include_raw_content": False
}
- response = requests.post(url, json=payload, headers=headers)
- data = response.json()
- return data.get("answer", "No answer found.")
+ try:
+ response = requests.post(url, json=payload, headers=headers, timeout=WEB_SEARCH_TIMEOUT)
+ response.raise_for_status()
+ data = response.json()
+ except (requests.RequestException, ValueError) as e:
+ print(f"⚠️ Web search failed: {e}")
+ return None
+
+ answer = data.get("answer") if isinstance(data, dict) else None
+ if not isinstance(answer, str) or not answer.strip():
+ print("⚠️ Web search returned no answer")
+ return None
+ return answer
-def explain_with_openai(question: str, web_content: str):
+def explain_with_openai(question: str, web_content: Optional[str]):
+ if not web_content:
+ web_content = "(No web content was found. Answer from your own math knowledge.)"
prompt = f"""
You are a friendly and precise math tutor.
@@ -83,6 +104,13 @@ Only include valid math steps — do not guess or make up answers.
return response.text
+def answer_from_web(question: str):
+ """Answer via the web fallback. Returns (answer, source label)."""
+ web_content = query_web(question)
+ source = "Web" if web_content else "Model only (web search unavailable)"
+ return explain_with_openai(question, web_content), source
+
+
# query_kb returns a cosine similarity (higher = better); with OpenAI embeddings
# almost any two English texts score well above 0, so `> 0.` accepted every query
# and served an arbitrary KB answer as authoritative. Require a real match.
@@ -96,7 +124,7 @@ def answer_math_question(question: str):
return "⚠️ This assistant only answers math-related academic questions."
answer = ""
- from_kb = False
+ source = "KB"
try:
kb_answer, similarity = query_kb(question)
@@ -126,25 +154,28 @@ Use the KB content as your only source. Do not guess or recalculate.
llm = OpenAI(api_key=OPENAI_API_KEY, model="gpt-4o")
answer = llm.complete(prompt).text
- from_kb = True
else:
raise ValueError("Low similarity match or empty")
except Exception as e:
print("⚠️ Using Web fallback because:", e)
- web_content = query_web(question)
- answer = explain_with_openai(question, web_content)
- from_kb = False
+ answer, source = answer_from_web(question)
- print(f"📦 Answer Source: {'KB' if from_kb else 'Web'}")
+ print(f"📦 Answer Source: {source}")
# Final Output Guardrail Check
if not output_validator.forward(question, answer):
print("⚠️ Final answer failed validation — retrying with web content...")
-
- web_content = query_web(question)
- answer = explain_with_openai(question, web_content)
- from_kb = False
+ answer, source = answer_from_web(question)
+ print(f"📦 Answer Source: {source}")
+
+ # Re-check the retry instead of returning it unvalidated
+ if not output_validator.forward(question, answer):
+ print("⚠️ Retried answer also failed validation.")
+ answer = (
+ "⚠️ This answer could not be verified by the output guardrail. "
+ "Please double-check it.\n\n" + answer
+ )
return answer