Skip to content
Python
21.5 KB
Raw Blame
#!/usr/bin/env python3
"""
Combined Analysis for All Models in batch2
Processes both sketch (ball_paths) and direct_vqa results for all models.
"""

import pandas as pd
import json
import re
from pathlib import Path
from typing import Dict, List, Tuple, Optional


def parse_boxed_answer(text: str) -> Optional[int]:
    r"""Extract answer from $\boxed{...}$ or \boxed{...} or \(\boxed{...}\) or <answer> format."""
    if not text:
        return None

    # Try <answer> tags first (for ViLaSR)
    answer_match = re.search(r'<answer>\s*(\d+|none)\s*</answer>', text, re.IGNORECASE)
    if answer_match:
        content = answer_match.group(1).strip().lower()
        if content == 'none':
            return 0
        try:
            return int(content)
        except ValueError:
            return None

    # Look for various boxed patterns
    # Try $\boxed{...}$ first
    match = re.search(r'\$\\boxed\{([^}]+)\}\$', text)
    if not match:
        # Try \boxed{...}
        match = re.search(r'\\boxed\{([^}]+)\}', text)
    if not match:
        # Try \(\boxed{...}\)
        match = re.search(r'\\\(\\boxed\{([^}]+)\}\\\)', text)

    if match:
        content = match.group(1).strip().lower()
        if content == 'none':
            return 0  # Use 0 for "none"
        try:
            return int(content)
        except ValueError:
            return None
    return None


def extract_boxed_text(text: str) -> str:
    r"""Extract the full boxed answer text including the boxed notation."""
    if not text:
        return ""

    # Look for various boxed patterns and return the full match
    # Try $\boxed{...}$ first
    match = re.search(r'\$\\boxed\{[^}]+\}\$', text)
    if match:
        return match.group(0)

    # Try \boxed{...}
    match = re.search(r'\\boxed\{[^}]+\}', text)
    if match:
        return match.group(0)

    # Try \(\boxed{...}\)
    match = re.search(r'\\\(\\boxed\{[^}]+\}\\\)', text)
    if match:
        return match.group(0)

    return ""


def load_ground_truth_from_metadata() -> Dict[str, int]:
    """Load ground truth from random_scene_metadata.json files."""
    ground_truth = {}

    # Path to the dataset metadata
    metadata_base = Path('/Users/log/Github/sketchvlm/datasets/large_second_batch')

    if not metadata_base.exists():
        print(f"Warning: Metadata directory not found: {metadata_base}")
        return ground_truth

    # Iterate through all run directories
    for run_dir in sorted(metadata_base.glob('run_b2_*')):
        metadata_file = run_dir / 'random_scene_metadata.json'
        if metadata_file.exists():
            try:
                with open(metadata_file, 'r') as f:
                    data = json.load(f)

                # Extract bucket_hit from simulation data
                bucket_hit = data.get('simulation', {}).get('bucket_hit')
                if bucket_hit is not None:
                    image_name = run_dir.name + '.png'
                    ground_truth[image_name] = int(bucket_hit)
            except Exception as e:
                print(f"Error loading ground truth from {metadata_file}: {e}")

    return ground_truth


def process_sketch_results(model_dir: Path, model_name: str, ground_truth: Dict[str, int]) -> List[Dict]:
    """Process sketch/ball_paths results from individual JSON files."""
    results = []

    # Process all item_*.json files
    for json_file in sorted(model_dir.glob("item_*.json")):
        if json_file.name == "summary.json":
            continue

        try:
            with open(json_file, 'r') as f:
                data = json.load(f)

            # Extract image name from source_image (original filename)
            source_image = data.get('source_image', '')
            if source_image:
                # Handle both forward and backward slashes
                source_image = source_image.replace('\\', '/')
                image_name = Path(source_image).name
            else:
                # Fallback to raw_image or grid_image path
                image_path = data.get('raw_image', data.get('grid_image', ''))
                image_path = image_path.replace('\\', '/')
                image_name = Path(image_path).stem.replace('_orig', '').replace('_grid', '') + '.png'

            # Parse the answer
            answer = data.get('answer')
            # If answer is a string (e.g., "$\boxed{3}$"), parse it
            if isinstance(answer, str):
                answer = parse_boxed_answer(answer)
            # If answer is still None, try other fields
            if answer is None:
                # Try model_output_full first (for gemini3 image results)
                model_output_full = data.get('model_output_full', '')
                answer = parse_boxed_answer(model_output_full)
                if answer is None:
                    # Fallback to model_output
                    model_output = data.get('model_output', '')
                    answer = parse_boxed_answer(model_output)

            if answer is not None:
                gold = ground_truth.get(image_name)
                if gold is not None:
                    # Get full model output (prefer model_output_full, fallback to model_output)
                    full_output = data.get('model_output_full', data.get('model_output', ''))
                    boxed_answer = extract_boxed_text(full_output)
                    results.append({
                        'image': image_name,
                        'model': model_name,
                        'type': 'paths',
                        'prediction': int(answer),
                        'gold': gold,
                        'correct': int(answer) == gold,
                        'boxed_answer': boxed_answer,
                        'model_output': full_output
                    })
        except Exception as e:
            print(f"Error processing {json_file}: {e}")

    return results


def process_jsonl_results(jsonl_file: Path, model_name: str, result_type: str, ground_truth: Dict[str, int]) -> List[Dict]:
    """Process ViLaSR results from a JSONL file."""
    results = []

    if not jsonl_file.exists():
        return results

    try:
        with open(jsonl_file, 'r') as f:
            for line_num, line in enumerate(f, 1):
                try:
                    data = json.loads(line.strip())

                    # Extract image name from image_path
                    image_path = data.get('image_path', [])
                    if isinstance(image_path, list) and len(image_path) > 0:
                        image_name = Path(image_path[0]).name
                    else:
                        image_name = Path(image_path).name

                    # Parse the answer from model_output
                    model_output = data.get('model_output', '')
                    answer = parse_boxed_answer(model_output)

                    if answer is not None:
                        gold = ground_truth.get(image_name)
                        if gold is not None:
                            boxed_answer = extract_boxed_text(model_output)
                            results.append({
                                'image': image_name,
                                'model': model_name,
                                'type': result_type,
                                'prediction': int(answer),
                                'gold': gold,
                                'correct': int(answer) == gold,
                                'boxed_answer': boxed_answer,
                                'model_output': model_output
                            })
                except json.JSONDecodeError as e:
                    print(f"    Warning: Error decoding JSON at line {line_num}: {e}")
                    continue
                except Exception as e:
                    print(f"    Warning: Error processing line {line_num}: {e}")
                    continue

    except Exception as e:
        print(f"  Error reading {jsonl_file}: {e}")

    return results


def process_thinkmorph_results(model_dir: Path, model_name: str, ground_truth: Dict[str, int]) -> List[Dict]:
    """Process ThinkMorph results from directories with text_data.json files.

    ThinkMorph has a unique structure where each result is in a directory named
    like 'sample_YYYYMMDD_HHMMSS_run_b2_XXX' and contains a text_data.json file.
    The run name (e.g., run_b2_XXX) must be extracted from the directory name.
    """
    results = []

    if not model_dir.exists():
        return results

    # Process all sample directories
    for sample_dir in sorted(model_dir.iterdir()):
        if not sample_dir.is_dir():
            continue

        # Extract run name from directory name
        # Pattern: sample_YYYYMMDD_HHMMSS_<run_name>
        # e.g., sample_20251203_182246_run_b2_001 -> run_b2_001
        dir_name = sample_dir.name
        parts = dir_name.split('_')

        # Find the index where 'run' starts
        run_idx = None
        for i, part in enumerate(parts):
            if part == 'run':
                run_idx = i
                break

        if run_idx is None:
            continue

        # Extract run name (everything from 'run' onwards)
        run_name = '_'.join(parts[run_idx:])
        image_name = run_name + '.png'

        # Look for text_data.json
        json_file = sample_dir / 'text_data.json'
        if not json_file.exists():
            continue

        try:
            with open(json_file, 'r') as f:
                data = json.load(f)

            # Extract answer from text_outputs
            text_outputs = data.get('text_outputs', [])
            model_output = '\n'.join(text_outputs) if text_outputs else ''

            # Parse the boxed answer
            answer = parse_boxed_answer(model_output)

            if answer is not None:
                gold = ground_truth.get(image_name)
                if gold is not None:
                    boxed_answer = extract_boxed_text(model_output)
                    results.append({
                        'image': image_name,
                        'model': model_name,
                        'type': 'paths',
                        'prediction': int(answer),
                        'gold': gold,
                        'correct': int(answer) == gold,
                        'boxed_answer': boxed_answer,
                        'model_output': model_output
                    })
        except Exception as e:
            print(f"Error processing {json_file}: {e}")

    return results


def process_direct_vqa_individual_json(model_dir: Path, model_name: str, ground_truth: Dict[str, int]) -> List[Dict]:
    """Process direct_vqa results from individual JSON files."""
    results = []

    # Process all item_*.json files
    for json_file in sorted(model_dir.glob("item_*.json")):
        if json_file.name == "summary.json":
            continue

        try:
            with open(json_file, 'r') as f:
                data = json.load(f)

            # Extract image name from source_image (original filename)
            source_image = data.get('source_image', '')
            if source_image:
                # Handle both forward and backward slashes
                source_image = source_image.replace('\\', '/')
                image_name = Path(source_image).name
            else:
                # Fallback to raw_image or grid_image path
                image_path = data.get('raw_image', data.get('grid_image', ''))
                image_path = image_path.replace('\\', '/')
                image_name = Path(image_path).stem.replace('_orig', '').replace('_grid', '') + '.png'

            # Parse the answer from model_output_full or model_output
            model_output = data.get('model_output_full', data.get('model_output', ''))
            answer = parse_boxed_answer(model_output)

            if answer is not None:
                gold = ground_truth.get(image_name)
                if gold is not None:
                    # Get full model output (prefer model_output_full, fallback to model_output)
                    full_output = data.get('model_output_full', data.get('model_output', ''))
                    boxed_answer = extract_boxed_text(full_output)
                    results.append({
                        'image': image_name,
                        'model': model_name,
                        'type': 'direct_vqa',
                        'prediction': answer,
                        'gold': gold,
                        'correct': answer == gold,
                        'boxed_answer': boxed_answer,
                        'model_output': full_output
                    })
        except Exception as e:
            print(f"Error processing {json_file}: {e}")

    return results


def load_gemini3_nb_results_with_mapping(json_file: Path, gemini3_dir: Path,
                                         ground_truth: Dict[str, int]) -> List[Dict]:
    """
    Load Gemini-3-Pro consistency check results and map them to original images.

    Args:
        json_file: Path to batch2_gemini3_pro_nb_results.json
        gemini3_dir: Path to gemini3_image_two_turn_batch2 directory with original results
        ground_truth: Ground truth dictionary

    Returns:
        List of result dictionaries
    """
    if not json_file.exists():
        print(f"Warning: {json_file} not found")
        return []

    try:
        with open(json_file, 'r') as f:
            nb_data = json.load(f)
    except Exception as e:
        print(f"Error loading {json_file}: {e}")
        return []

    # Build mapping from item index to original image name
    index_to_image = {}

    if gemini3_dir.exists():
        for item_file in sorted(gemini3_dir.glob("item_*.json")):
            try:
                with open(item_file, 'r') as f:
                    item_data = json.load(f)

                # Extract item index from filename
                match = re.search(r'item_(\d+)\.json', item_file.name)
                if not match:
                    continue

                item_idx = int(match.group(1))

                # Get source image name
                source_image = item_data.get('source_image', '')
                if source_image:
                    image_name = Path(source_image).name
                    index_to_image[item_idx] = image_name
            except Exception as e:
                print(f"Error processing {item_file}: {e}")
                continue

    # Now process the consistency check results
    results = []

    for item in nb_data:
        # Skip failed items
        if not item.get('success', False):
            continue

        # Get the index
        item_idx = item.get('index')
        if item_idx is None:
            continue

        # Get the response
        response = item.get('consistency_check_response', '')
        if not response:
            continue

        # Parse the answer
        answer = parse_boxed_answer(response)
        if answer is None:
            continue

        # Map to original image
        image_name = index_to_image.get(item_idx)
        if not image_name:
            continue

        # Get ground truth
        gold = ground_truth.get(image_name)
        if gold is None:
            continue

        boxed_answer = extract_boxed_text(response)
        results.append({
            'image': image_name,
            'model': 'Gemini-3-Pro-NB',
            'type': 'paths',
            'prediction': int(answer),
            'gold': gold,
            'correct': int(answer) == gold,
            'boxed_answer': boxed_answer,
            'model_output': response
        })

    return results


def process_all_models():
    """Process all models in batch2 directory."""
    base_dir = Path(__file__).parent.parent.parent
    batch2_dir = base_dir / "results" / "mix_eval" / "ball_paths" / "batch2"
    output_dir = base_dir / "analysis" / "ball_physics"
    output_dir.mkdir(parents=True, exist_ok=True)

    # Load ground truth from metadata files
    print("Loading ground truth from random_scene_metadata.json files...")
    ground_truth = load_ground_truth_from_metadata()
    print(f"Loaded {len(ground_truth)} ground truth labels")

    all_results = []

    # Define models to process
    models_config = [
        # Sketch (ball_paths) models
        ('gemini_25_flash_ball_paths_batch2', 'Gemini-2.5-Flash', 'paths', False),
        ('gemini_25_pro_ball_paths_batch2', 'Gemini-2.5-Pro', 'paths', False),
        ('gpt5_low_ball_paths_batch2', 'GPT-5-low', 'paths', False),
        ('20260121_205501_gpt5low_ball_batch2', 'GPT-5-low-NEW', 'paths', False),
        ('gpt5_med_ball_paths_batch2', 'GPT-5-med', 'paths', False),
        ('qwen25_7b_ball_paths_batch2', 'Qwen2.5-7B', 'paths', False),
        ('vilasr_ball_paths_batch2', 'ViLaSR', 'paths', True),  # JSONL format
        ('gemini3_image_two_turn_batch2', 'nano_banana_pro', 'paths', False),
        ('gemini3_ball_paths_batch2', 'Gemini-3-Pro', 'paths', False),
        ('gemini3pro_batch2_ball_paths_no_grid_0_to_1000', 'Gemini-3-Pro-0-1000', 'paths', False),
        ('20260114_111423_gem3pro_meta_second_batch_copy', 'Gemini-3-Pro-Meta', 'paths', False),
        ('20260123_232913_gpt_meta_batch2_nogrid_sketch', 'GPT-Meta-NoGrid', 'paths', False),
        ('20260124_183213_gpt_meta_grid_vqa_2', 'GPT-Meta-Grid-VQA', 'paths', False),

    ]

    # Process sketch/ball_paths models
    print("\nProcessing sketch (ball_paths) results...")
    for dir_name, model_name, result_type, is_jsonl in models_config:
        model_dir = batch2_dir / dir_name
        if model_dir.exists() and model_dir.is_dir():
            print(f"  Processing {model_name}...")
            if is_jsonl:
                # Process JSONL file
                jsonl_file = model_dir / 'results.jsonl'
                results = process_jsonl_results(jsonl_file, model_name, result_type, ground_truth)
            else:
                # Process individual JSON files
                results = process_sketch_results(model_dir, model_name, ground_truth)
            all_results.extend(results)
            print(f"    Found {len(results)} results")

    # Process ThinkMorph (special handling due to different directory structure)
    thinkmorph_dir = batch2_dir / "thinkmorph_ball_paths_batch2"
    if thinkmorph_dir.exists():
        print(f"  Processing ThinkMorph...")
        results = process_thinkmorph_results(thinkmorph_dir, "ThinkMorph", ground_truth)
        all_results.extend(results)
        print(f"    Found {len(results)} results")

    # Process Gemini-3-Pro consistency check results
    print(f"  Processing Gemini-3-Pro-NB (consistency check)...")
    gemini3_nb_json = batch2_dir / "batch2_gemini3_pro_nb_results.json"
    gemini3_orig_dir = batch2_dir / "gemini3_image_two_turn_batch2"
    results = load_gemini3_nb_results_with_mapping(gemini3_nb_json, gemini3_orig_dir, ground_truth)
    all_results.extend(results)
    print(f"    Found {len(results)} results")

    # Process direct_vqa models
    print("\nProcessing direct_vqa results...")
    direct_vqa_dir = batch2_dir / "direct_vqa"

    # Direct VQA individual JSONs
    vqa_dirs = [
        ('gemini_25_flash_no_sketch_batch2', 'Gemini-2.5-Flash'),
        ('gemini_25_pro_no_sketch_batch_2', 'Gemini-2.5-Pro'),
        ('gpt5_low_no_sketch_batch2', 'GPT-5-low'),
        ('gpt5_med_no_sketch_batch2', 'GPT-5-med'),
        ('qwen25_7b_no_sketch_batch2', 'Qwen2.5-7B'),
        ('gemini3_no_sketch_batch2', 'Gemini-3-Pro'),
    ]

    for dir_name, model_name in vqa_dirs:
        vqa_model_dir = direct_vqa_dir / dir_name
        if vqa_model_dir.exists():
            print(f"  Processing {model_name} (direct_vqa)...")
            results = process_direct_vqa_individual_json(vqa_model_dir, model_name, ground_truth)
            all_results.extend(results)
            print(f"    Found {len(results)} results")

    # Create DataFrame
    df = pd.DataFrame(all_results)

    # Save to CSV
    output_file = output_dir / "batch2_all_models_results.csv"
    df.to_csv(output_file, index=False)
    print(f"\n✓ Saved all results to: {output_file}")
    print(f"  Total results: {len(df)}")

    # Compute and display accuracy
    print("\n" + "="*80)
    print("ACCURACY SUMMARY")
    print("="*80)

    # Overall accuracy by model and type
    summary = df.groupby(['model', 'type']).agg({
        'correct': ['sum', 'count', 'mean']
    }).round(4)

    print("\nAccuracy by Model and Type:")
    print("-" * 80)
    for (model, type_), row in summary.iterrows():
        correct = int(row['correct']['sum'])
        total = int(row['correct']['count'])
        accuracy = row['correct']['mean'] * 100
        type_label = 'SketchVLM' if type_ == 'paths' else 'Direct VQA'
        print(f"{model:20s} | {type_label:12s} | {correct:3d}/{total:3d} = {accuracy:5.1f}%")

    # Create pivot table for easier comparison
    pivot = df.groupby(['model', 'type'])['correct'].mean().unstack(fill_value=0) * 100

    # Save pivot table
    pivot_file = output_dir / "batch2_accuracy_pivot.csv"
    pivot.to_csv(pivot_file)
    print(f"\n✓ Saved accuracy pivot table to: {pivot_file}")

    # Display pivot table
    print("\n" + "="*80)
    print("ACCURACY PIVOT TABLE (%)")
    print("="*80)
    print(pivot.to_string())

    # Model-level summary (best of both approaches)
    print("\n" + "="*80)
    print("MODEL COMPARISON")
    print("="*80)
    model_summary = df.groupby('model')['correct'].agg(['sum', 'count', 'mean'])
    model_summary['accuracy'] = (model_summary['mean'] * 100).round(1)
    model_summary = model_summary.sort_values('accuracy', ascending=False)

    print("\nOverall Accuracy (combining both direct_vqa and paths):")
    for model, row in model_summary.iterrows():
        print(f"  {model:20s}: {int(row['sum']):3d}/{int(row['count']):3d} = {row['accuracy']:5.1f}%")

    return df, pivot


if __name__ == "__main__":
    df, pivot = process_all_models()