Skip to content
Python
10.4 KB
Raw Blame
#!/usr/bin/env python3
"""
Evaluate two-turn ball path results from batch1 and batch2.
"""

import json
import re
from pathlib import Path
from typing import Dict, Optional
import pandas as pd

# Regex patterns for parsing bucket numbers from $\boxed{}$ format
_BOXED_RE = re.compile(r"\$\\boxed\{\s*(\d+)\s*\}\$", re.IGNORECASE)
_LOOSE_RE = re.compile(r"\\boxed\{\s*(\d+)\s*\}", re.IGNORECASE)


def parse_bucket_from_answer(text: Optional[str]) -> Optional[int]:
    """Parse bucket number from answer field."""
    if not text:
        return None

    # Try strict $\boxed{} format first
    m = _BOXED_RE.search(text)
    if m:
        return int(m.group(1))

    # Try without $ signs
    m = _LOOSE_RE.search(text)
    if m:
        return int(m.group(1))

    return None


def load_ground_truth(dataset_dir: Path) -> Dict[str, int]:
    """
    Load ground truth labels from dataset metadata.

    Args:
        dataset_dir: Path to datasets/large_run_split

    Returns:
        Dictionary mapping image_name -> bucket_hit (1-4)
    """
    bucket_dict = {}

    # Iterate through all subdirectories
    for subdir in dataset_dir.iterdir():
        if not subdir.is_dir():
            continue

        image_name = subdir.name  # e.g., "run_001_1"
        metadata_path = subdir / "random_scene_metadata.json"

        if not metadata_path.exists():
            continue

        try:
            with open(metadata_path, 'r') as f:
                data = json.load(f)

            simulation = data.get('simulation', {})
            bucket_hit = simulation.get('bucket_hit')

            # Store with .png extension to match source_image format
            img_key = f"{image_name}.png"

            if bucket_hit is not None and isinstance(bucket_hit, int):
                bucket_dict[img_key] = bucket_hit

        except Exception as e:
            print(f"Error loading {metadata_path}: {e}")
            continue

    return bucket_dict


def evaluate_batch(batch_dir: Path, batch_name: str, gt_dict: Dict[str, int]) -> pd.DataFrame:
    """
    Evaluate a single batch of results.

    Args:
        batch_dir: Path to the batch results directory
        batch_name: Name of the batch (for reporting)
        gt_dict: Ground truth dictionary

    Returns:
        DataFrame with evaluation results
    """
    rows = []

    # Find all JSON files
    json_files = sorted(batch_dir.glob("item_*.json"))

    for json_file in json_files:
        try:
            with open(json_file, 'r') as f:
                data = json.load(f)

            # Extract source image name
            source_image = data.get('source_image', '')
            if source_image:
                source_image = source_image.replace('\\', '/')
                image_name = Path(source_image).name
            else:
                continue

            # Extract model answer from 'answer' field
            answer_text = data.get('answer', '')
            prediction = parse_bucket_from_answer(answer_text)

            # Get ground truth
            gold = gt_dict.get(image_name)

            # Compute correctness
            correct = (prediction == gold) if (prediction is not None and gold is not None) else False

            rows.append({
                'batch': batch_name,
                'item': json_file.stem,
                'image': image_name,
                'answer_text': answer_text,
                'prediction': prediction,
                'gold': gold,
                'correct': correct
            })

        except Exception as e:
            print(f"Error processing {json_file}: {e}")
            continue

    return pd.DataFrame(rows)


def main():
    # Define paths
    base_dir = Path(__file__).parent.parent.parent
    results_base = base_dir / "results" / "mix_eval" / "ball_paths"
    dataset_dir1 = base_dir / "datasets" / "large_run_split"
    dataset_dir2 = base_dir / "datasets" / "large_second_batch"
    output_dir = base_dir / "analysis" / "ball_physics"

    print("="*80)
    print("TWO-TURN BALL PATH EVALUATION - ALL MODELS")
    print("="*80)

    # Load ground truth from both datasets
    print("\nLoading ground truth from dataset metadata...")
    print("  Loading from large_run_split...")
    gt_dict1 = load_ground_truth(dataset_dir1)
    print(f"  Loaded {len(gt_dict1)} labels from batch 1 dataset")

    print("  Loading from large_second_batch...")
    gt_dict2 = load_ground_truth(dataset_dir2)
    print(f"  Loaded {len(gt_dict2)} labels from batch 2 dataset")

    # Combine both ground truth dictionaries
    gt_dict = {**gt_dict1, **gt_dict2}
    print(f"Total ground truth labels: {len(gt_dict)}")

    # Define all models to evaluate
    models = [
        ('gemini25_flash_ball_paths', 'gemini25_flash_ball_paths_batch2', 'Gemini 2.5 Flash'),
        ('gemini25_pro_ball_paths', 'gemini25_pro_ball_paths_batch2', 'Gemini 2.5 Pro'),
        ('gpt5_low_paths', 'gpt5_low_ball_paths_batch2', 'GPT-5 Low'),
        ('gpt5_med_ball_paths', 'gpt5_med_ball_paths_batch2', 'GPT-5 Med'),
    ]

    all_dfs = []

    # Evaluate each model
    for batch1_name, batch2_name, model_label in models:
        print("\n" + "="*80)
        print(f"EVALUATING {model_label.upper()}")
        print("="*80)

        # Batch 1
        batch1_dir = results_base / "batch1" / "two_turn" / batch1_name
        if batch1_dir.exists():
            print(f"Batch 1: Loading from {batch1_name}...")
            df = evaluate_batch(batch1_dir, f"batch1", gt_dict)
            df['model'] = model_label
            all_dfs.append(df)
            print(f"  Loaded {len(df)} samples")
        else:
            print(f"Warning: {batch1_dir} not found")

        # Batch 2
        batch2_dir = results_base / "batch2" / "two_turn" / batch2_name
        if batch2_dir.exists():
            print(f"Batch 2: Loading from {batch2_name}...")
            df = evaluate_batch(batch2_dir, f"batch2", gt_dict)
            df['model'] = model_label
            all_dfs.append(df)
            print(f"  Loaded {len(df)} samples")
        else:
            print(f"Warning: {batch2_dir} not found")

    # Combine all results
    if not all_dfs:
        print("Error: No data loaded!")
        return

    combined_df = pd.concat(all_dfs, ignore_index=True)

    # Save detailed results
    output_path = output_dir / "two_turn_detailed_results.csv"
    combined_df.to_csv(output_path, index=False)
    print(f"\nSaved detailed results to: {output_path}")

    # Compute accuracy statistics
    print("\n" + "="*80)
    print("ACCURACY RESULTS")
    print("="*80)

    # Overall accuracy
    valid_df = combined_df[combined_df['gold'].notna()].copy()

    overall_total = len(valid_df)
    overall_correct = valid_df['correct'].sum()
    overall_accuracy = (overall_correct / overall_total * 100) if overall_total > 0 else 0.0

    print(f"\nOVERALL (All Models, Both Batches):")
    print(f"  Total:    {overall_total}")
    print(f"  Correct:  {overall_correct}")
    print(f"  Accuracy: {overall_accuracy:.2f}%")

    # Per-model accuracy
    print("\n" + "="*80)
    print("PER-MODEL RESULTS (Both Batches Combined)")
    print("="*80)
    for model_name in valid_df['model'].unique():
        model_df = valid_df[valid_df['model'] == model_name]
        model_total = len(model_df)
        model_correct = model_df['correct'].sum()
        model_accuracy = (model_correct / model_total * 100) if model_total > 0 else 0.0

        print(f"\n{model_name}:")
        print(f"  Total:    {model_total}")
        print(f"  Correct:  {int(model_correct)}")
        print(f"  Accuracy: {model_accuracy:.2f}%")

    # Per-model per-batch breakdown
    print("\n" + "="*80)
    print("PER-MODEL PER-BATCH BREAKDOWN")
    print("="*80)
    for model_name in valid_df['model'].unique():
        print(f"\n{model_name}:")
        for batch_name in ['batch1', 'batch2']:
            batch_df = valid_df[(valid_df['model'] == model_name) & (valid_df['batch'] == batch_name)]
            batch_total = len(batch_df)
            batch_correct = batch_df['correct'].sum()
            batch_accuracy = (batch_correct / batch_total * 100) if batch_total > 0 else 0.0

            print(f"  {batch_name}: {int(batch_correct)}/{batch_total} = {batch_accuracy:.2f}%")

    # Show some examples of errors
    print("\n" + "="*80)
    print("SAMPLE ERRORS (First 10)")
    print("="*80)

    errors = valid_df[~valid_df['correct']].head(10)
    if len(errors) > 0:
        for idx, row in errors.iterrows():
            print(f"\nItem: {row['item']} ({row['batch']})")
            print(f"  Image:      {row['image']}")
            print(f"  Predicted:  {row['prediction']}")
            print(f"  Gold:       {row['gold']}")
            print(f"  Answer:     {row['answer_text']}")
    else:
        print("\nNo errors found!")

    # Create summary CSV with per-model per-batch breakdown
    summary_data = []

    # Per-model per-batch
    for model_name in sorted(valid_df['model'].unique()):
        for batch_name in ['batch1', 'batch2']:
            batch_df = valid_df[(valid_df['model'] == model_name) & (valid_df['batch'] == batch_name)]
            if len(batch_df) > 0:
                summary_data.append({
                    'model': model_name,
                    'batch': batch_name,
                    'total': len(batch_df),
                    'correct': int(batch_df['correct'].sum()),
                    'incorrect': int((~batch_df['correct']).sum()),
                    'accuracy': (batch_df['correct'].sum() / len(batch_df) * 100)
                })

        # Per-model overall (both batches)
        model_df = valid_df[valid_df['model'] == model_name]
        summary_data.append({
            'model': model_name,
            'batch': 'both',
            'total': len(model_df),
            'correct': int(model_df['correct'].sum()),
            'incorrect': int((~model_df['correct']).sum()),
            'accuracy': (model_df['correct'].sum() / len(model_df) * 100)
        })

    # Add overall (all models, all batches)
    summary_data.append({
        'model': 'ALL',
        'batch': 'both',
        'total': overall_total,
        'correct': int(overall_correct),
        'incorrect': int(overall_total - overall_correct),
        'accuracy': overall_accuracy
    })

    summary_df = pd.DataFrame(summary_data)
    summary_path = output_dir / "two_turn_accuracy_summary.csv"
    summary_df.to_csv(summary_path, index=False)
    print(f"\nSaved summary to: {summary_path}")

    return combined_df, summary_df


if __name__ == "__main__":
    combined_df, summary_df = main()