Skip to content
Python
5.4 KB
Raw Blame
#!/usr/bin/env python3
"""
Compile all JSON response files from a directory into a CSV file.
"""

import os
import re
import csv
import json
from pathlib import Path
from typing import Optional

# Same regex patterns from gpt_baseline_ball_drop.py
_BOXED_RE = re.compile(r"\$\\boxed\{s*(1|2|3|4|none)\s*\}\$", re.IGNORECASE)
_LOOSE_RE = re.compile(r"\\boxed\{\s*(1|2|3|4|none)\s*\}", re.IGNORECASE)
_FALLBACK_RE = re.compile(r"\b(1|2|3|4|none)\b", re.IGNORECASE)

def parse_bucket(text: Optional[str]) -> Optional[str]:
    """Parse bucket number from text."""
    if not text:
        return None

    # Try strict $\boxed{} format first
    m = _BOXED_RE.search(text)
    if m:
        return m.group(1)

    # Try without $ signs
    m = _LOOSE_RE.search(text)
    if m:
        return m.group(1)

    # Fallback to any digit
    m = _FALLBACK_RE.search(text)
    if m:
        return m.group(1)

    return None

def extract_text_from_response(response_data: dict) -> str:
    """Extract the output text from the response JSON."""
    try:
        output = response_data.get("output", [])
        for item in output:
            if item.get("type") == "message":
                content = item.get("content", [])
                for content_item in content:
                    if content_item.get("type") == "output_text":
                        return content_item.get("text", "")
    except Exception:
        pass
    return ""

def get_model_name(response_data: dict, reasoning_effort: str = "medium") -> str:
    """Extract model name and reasoning effort."""
    model = response_data.get("model", "gpt-5")
    if "gpt-5" in model:
        # Extract reasoning effort if available
        reasoning = response_data.get("reasoning", {})
        effort = reasoning.get("effort", reasoning_effort)
        return f"openai::gpt-5-{effort}"
    return f"openai::{model}"

def compile_jsons_to_csv(json_dir: str, output_csv: str, labels_dict: Optional[dict] = None):
    """
    Compile all .response.json files from json_dir into a CSV file.

    Args:
        json_dir: Directory containing the JSON response files
        output_csv: Output CSV file path
        labels_dict: Optional dictionary mapping image names to gold labels
    """
    json_path = Path(json_dir)

    # Find all response.json files
    response_files = sorted(json_path.glob("*.response.json"))

    if not response_files:
        print(f"No .response.json files found in {json_dir}")
        return

    print(f"Found {len(response_files)} response files")

    # Prepare data rows
    rows = []

    for response_file in response_files:
        # Extract image name from filename (remove .response.json)
        image_name = response_file.stem.replace(".response", "") + ".png"

        try:
            # Read JSON file
            with open(response_file, "r", encoding="utf-8") as f:
                response_data = json.load(f)

            # Extract raw text
            raw_text = extract_text_from_response(response_data)

            # Parse bucket number
            parsed_label = parse_bucket(raw_text)
            parsed_int = int(parsed_label) if parsed_label and parsed_label.isdigit() else None

            # Get model name
            model = get_model_name(response_data)

            # Get gold labels if available
            gold = ""
            gold_int = ""
            correct = ""

            if labels_dict and image_name in labels_dict:
                gold = labels_dict[image_name]
                gold_int = int(gold) if gold.isdigit() else ""
                if parsed_int is not None and gold_int != "":
                    correct = int(parsed_int == gold_int)

            rows.append({
                "image": image_name,
                "model": model,
                "raw_text": raw_text,
                "parsed_label": parsed_label or "",
                "parsed_int": parsed_int or "",
                "gold": gold,
                "gold_int": gold_int,
                "correct": correct
            })

        except Exception as e:
            print(f"Error processing {response_file}: {e}")
            continue

    # Write to CSV
    output_path = Path(output_csv)
    output_path.parent.mkdir(parents=True, exist_ok=True)

    with open(output_path, "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=[
            "image", "model", "raw_text", "parsed_label", "parsed_int",
            "gold", "gold_int", "correct"
        ])
        writer.writeheader()
        writer.writerows(rows)

    print(f"Wrote {len(rows)} rows to {output_csv}")

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser(description="Compile JSON response files to CSV")
    parser.add_argument("--json-dir", required=True, help="Directory containing .response.json files")
    parser.add_argument("--output", required=True, help="Output CSV file path")
    parser.add_argument("--labels", help="Optional labels CSV file (image,label columns)")

    args = parser.parse_args()

    # Load labels if provided
    labels_dict = None
    if args.labels:
        labels_dict = {}
        with open(args.labels, "r", encoding="utf-8") as f:
            reader = csv.DictReader(f)
            for row in reader:
                image = row.get("image", "")
                label = row.get("label", "") or row.get("gold", "")
                if image and label:
                    labels_dict[image] = label

    compile_jsons_to_csv(args.json_dir, args.output, labels_dict)