Skip to content

100+ AI Agents, Agent Skills and RAG Apps - Free and Open Source.

1.9 KB
Raw Blame
# Add the project root to the Python path
import sys
import os
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))

import re
import pandas as pd
import time
from datetime import datetime
from rag.query_router import answer_math_question
from data.load_gsm8k_data import load_jeebench_dataset

def is_answer_correct(expected, response: str) -> bool:
    """Check that the gold answer appears as a standalone token in the response.

    A plain substring test counts gold "C" as correct for any reply containing
    the letter c, and gold "12" as correct inside "112" or "0.12".
    """
    pattern = rf"(?<![A-Za-z0-9.]){re.escape(str(expected).strip())}(?![A-Za-z0-9]|\.\d)"
    return re.search(pattern, response) is not None


def benchmark_math_agent(limit: int = 10):
    # ✅ Always filter math-only questions
    df = load_jeebench_dataset()
    df = df.head(limit)  # Limit the number of questions for benchmarking

    total = len(df)
    correct = 0
    results = []

    for idx, row in df.iterrows():
        question = row["question"]
        expected = row["gold"]
        start = time.time()

        try:
            response = answer_math_question(question)
            is_correct = is_answer_correct(expected, response)
            if is_correct:
                correct += 1

            results.append({
                "Question": question,
                "Expected": expected,
                "Predicted": response,
                "Correct": is_correct,
                "TimeTakenSec": round(time.time() - start, 2)
            })

        except Exception as e:
            results.append({
                "Question": question,
                "Expected": expected,
                "Predicted": f"Error: {e}",
                "Correct": False,
                "TimeTakenSec": None
            })

    df_result = pd.DataFrame(results)
    accuracy = correct / total * 100
    return df_result, accuracy