Skip to content
Python
27.6 KB
Raw Blame
#!/usr/bin/env python3
"""
Create a comprehensive CSV file with all maze results.
Each row represents a unique maze + validity combination.
Columns include ground truth and all model responses.
"""

import json
import re
import csv
from pathlib import Path
from typing import Dict, List, Optional


PROJECT_ROOT = Path('/Users/log/Github/sketchvlm')
OUTPUT_TAIL_CHARS = 50


def normalize_path(path_value: Optional[str]) -> str:
    """Return a repository-relative POSIX path if possible."""
    if not path_value:
        return ''

    path_str = str(path_value).replace('\\', '/')
    path_obj = Path(path_str)

    if path_obj.is_absolute():
        try:
            return str(path_obj.relative_to(PROJECT_ROOT)).replace('\\', '/')
        except ValueError:
            return path_str

    return path_str


def get_output_tail(text: Optional[str], limit: int = OUTPUT_TAIL_CHARS) -> str:
    """Return the trailing portion of a model output for quick inspection."""
    if not text:
        return ''

    text = str(text)
    if len(text) <= limit:
        return text
    return text[-limit:]


def build_result_entry(answer: str,
                       model_output: Optional[str] = None,
                       source_image: Optional[str] = None,
                       annotated_image: Optional[str] = None) -> Dict[str, str]:
    """Package grading information and lightweight metadata for CSV export."""
    return {
        'answer': answer,
        'output_tail': get_output_tail(model_output),
        'source_image': normalize_path(source_image),
        'annotated_image': normalize_path(annotated_image)
    }


def extract_answer_from_response(response_text: str) -> str:
    """Extract answer from model response."""
    if not response_text or response_text.strip() == '':
        return 'unknown'

    # Try <final_answer> tags first
    final_answer_match = re.search(r'<final_answer>\s*(.*?)\s*</final_answer>',
                                   response_text, re.IGNORECASE | re.DOTALL)
    if final_answer_match:
        answer_text = final_answer_match.group(1).strip()
        if 'valid' in answer_text.lower():
            if 'invalid' in answer_text.lower():
                return 'invalid'
            else:
                return 'valid'
        return 'unknown'

    # Try <answer> tags (for ViLaSR)
    answer_match = re.search(r'<answer>\s*(.*?)\s*</answer>',
                            response_text, re.IGNORECASE | re.DOTALL)
    if answer_match:
        answer_text = answer_match.group(1).strip()
        if 'valid' in answer_text.lower():
            if 'invalid' in answer_text.lower():
                return 'invalid'
            else:
                return 'valid'
        return 'unknown'

    # Try \boxed{} format (for ViLaSR)
    boxed_match = re.search(r'\$?\\boxed\{(.*?)\}\$?',
                           response_text, re.IGNORECASE | re.DOTALL)
    if boxed_match:
        answer_text = boxed_match.group(1).strip()
        if 'valid' in answer_text.lower():
            if 'invalid' in answer_text.lower():
                return 'invalid'
            else:
                return 'valid'
        return 'unknown'

    # Fallback: check last 30 characters
    last_chars = response_text[-30:].lower()
    if 'invalid' in last_chars:
        return 'invalid'
    elif 'valid' in last_chars:
        return 'valid'

    return 'unknown'


def extract_maze_id(source_image_path: str) -> str:
    """Extract maze ID from source image path."""
    if not source_image_path:
        return None
    # Normalize path separators to handle Windows paths on Unix systems
    normalized_path = str(source_image_path).replace('\\', '/')
    filename = Path(normalized_path).stem
    return filename


def get_path_length_from_maze_id(maze_id: str, maze_to_path_length: Dict[str, int]) -> Optional[int]:
    """Get path length for a maze ID."""
    return maze_to_path_length.get(maze_id)


def build_maze_to_path_length_mapping() -> Dict[str, int]:
    """Build mapping of maze_id to path_length."""
    maze_to_path_length = {}

    for path_length in range(1, 10):  # Check up to path_length_9
        path_dir = Path(f'/Users/log/Github/sketchvlm/datasets/maze_v2/path_length_{path_length}')
        if not path_dir.exists():
            continue
        for maze_dir in path_dir.iterdir():
            if maze_dir.is_dir() and maze_dir.name.startswith('maze_'):
                maze_to_path_length[maze_dir.name] = path_length

    return maze_to_path_length


def load_model_results(base_path: Path, model_prefix: str, validity: str) -> Dict[str, Dict[str, str]]:
    """Load answers plus lightweight metadata for a directory of JSON results."""
    results: Dict[str, Dict[str, str]] = {}

    results_dir = base_path / f'{model_prefix}_{validity}'
    if not results_dir.exists():
        return results

    for json_file in results_dir.glob('item_*.json'):
        try:
            with open(json_file, 'r') as f:
                data = json.load(f)

            source_image = data.get('source_image', '')
            maze_id = extract_maze_id(source_image)
            model_output = data.get('model_output_full') or data.get('model_output', '')

            if not maze_id:
                continue

            annotated_path = json_file.with_name(f'{json_file.stem}_annotated.png')
            entry = build_result_entry(
                extract_answer_from_response(model_output),
                model_output=model_output,
                source_image=source_image,
                annotated_image=str(annotated_path) if annotated_path.exists() else None,
            )

            results[maze_id] = entry

        except Exception as e:
            print(f"Error processing {json_file}: {e}")

    return results


def load_vilasr_jsonl_results(jsonl_path: Path) -> Dict[str, Dict[str, str]]:
    """Load ViLaSR answers plus snippets from JSONL files."""
    results: Dict[str, Dict[str, str]] = {}

    if not jsonl_path.exists():
        return results

    try:
        with open(jsonl_path, 'r') as f:
            for line in f:
                try:
                    data = json.loads(line.strip())

                    image_path = data.get('image_path', [])
                    if isinstance(image_path, list) and image_path:
                        source_image = image_path[0]
                    else:
                        source_image = image_path

                    maze_id = extract_maze_id(source_image)
                    if not maze_id:
                        continue

                    model_output = data.get('model_output', '')
                    results[maze_id] = build_result_entry(
                        extract_answer_from_response(model_output),
                        model_output=model_output,
                        source_image=source_image,
                    )

                except json.JSONDecodeError as e:
                    print(f"Error decoding JSON line: {e}")
                    continue

    except Exception as e:
        print(f"Error reading {jsonl_path}: {e}")

    return results


def load_consistency_check_json(json_path: Path, results_base_dir: Path) -> Dict[str, Dict[str, str]]:
    """Load Gemini consistency-check responses plus metadata.

    This function handles two JSON formats:
    1. Files with 'consistency_check_response' field (from limitedmaze_nanob_gemini3_pro_*.json)
    2. Files with 'model_answer' field (from nanob_gemini3_pro_*.json)

    For the limited maze format, we use consistency_check_response to get the answer.
    """
    results: Dict[str, Dict[str, str]] = {}

    if not json_path.exists():
        return results

    # Build mapping from item index to maze metadata
    item_meta: Dict[int, Dict[str, str]] = {}
    if results_base_dir and results_base_dir.exists():
        for jf in sorted(results_base_dir.glob('item_*.json')):
            try:
                j = json.loads(jf.read_text(encoding='utf-8'))
                source_img = str(j.get('source_image', '')).replace('\\', '/')
                maze_id = extract_maze_id(source_img)
                if not maze_id:
                    continue
                idxm = re.match(r'item_(\d+)\.json$', jf.name, re.I)
                if idxm:
                    item_idx = int(idxm.group(1))
                    item_meta[item_idx] = {
                        'maze_id': maze_id,
                        'source_image': source_img,
                        'item_idx': item_idx
                    }
            except Exception as e:
                continue

    try:
        with open(json_path, 'r') as f:
            data = json.load(f)

        if not isinstance(data, list):
            return results

        # Track seen image paths to handle duplicates (keep first occurrence)
        seen_image_paths = set()

        for entry in data:
            # Determine which format we're using
            has_consistency_response = 'consistency_check_response' in entry
            has_model_answer = 'model_answer' in entry

            image_path = entry.get('image_path', '')

            # Skip duplicates (keep first occurrence only)
            if image_path in seen_image_paths:
                continue
            seen_image_paths.add(image_path)

            maze_id = None
            meta = None

            # Preferred: map via item_XXX index using results_base_dir
            try:
                item_match = re.search(r'item_(\d+)', Path(image_path).name, re.I)
            except Exception:
                item_match = None

            annotated_image = None
            if item_match:
                idx = int(item_match.group(1))
                meta = item_meta.get(idx)
                if meta:
                    maze_id = meta.get('maze_id')
                # For Nano Banana, prefer the generated sketch image; fallback to annotated
                try:
                    gen_candidate = results_base_dir / f"item_{idx:05d}_generated_0.png"
                    ann_candidate = results_base_dir / f"item_{idx:05d}_annotated.png"
                    if gen_candidate.exists():
                        annotated_image = str(gen_candidate)
                    elif ann_candidate.exists():
                        annotated_image = str(ann_candidate)
                except Exception:
                    pass

            # Fallback: derive maze_id directly from the image_path filename
            if not maze_id:
                try:
                    maze_id = extract_maze_id(str(image_path))
                except Exception:
                    maze_id = None

            if not maze_id:
                continue

            # Extract answer based on format
            if has_consistency_response:
                # Use consistency_check_response for limited maze format
                response = entry.get('consistency_check_response', '')
                answer = extract_answer_from_response(response) if response else 'unknown'
            elif has_model_answer:
                # Use model_answer for direct format
                response = entry.get('model_answer', '')
                answer = extract_answer_from_response(response) if response else 'unknown'
            else:
                # Fallback
                response = ''
                answer = 'unknown'

            results[maze_id] = build_result_entry(
                answer,
                model_output=response,
                source_image=(meta.get('source_image') if meta else image_path),
                annotated_image=annotated_image,
            )

    except Exception as e:
        print(f"Error reading {json_path}: {e}")

    return results


def load_nanob_direct_json(json_path: Path, results_base_dir: Path) -> Dict[str, Dict[str, str]]:
    """Load NanoBanana + Gemini direct VQA results from a single JSON array file.

    This file has entries with keys like: image_path, model_answer, extracted_answer, model.
    We map item_XXXXX to maze_id using the provided results_base_dir's item_XXXXX.json to
    recover the canonical source_image/maze_id. We also attach the generated image path
    (item_XXXXX_generated_0.png) as the annotated image for visualization.
    """
    results: Dict[str, Dict[str, str]] = {}

    if not json_path.exists():
        return results

    # Build quick mapping item index -> (maze_id, source_image)
    item_meta: Dict[int, Dict[str, str]] = {}
    if results_base_dir and results_base_dir.exists():
        for jf in sorted(results_base_dir.glob('item_*.json')):
            try:
                j = json.loads(jf.read_text(encoding='utf-8'))
                source_img = str(j.get('source_image', '')).replace('\\', '/')
                maze_id = extract_maze_id(source_img)
                if not maze_id:
                    continue
                idxm = re.match(r'item_(\d+)\.json$', jf.name, re.I)
                if idxm:
                    item_idx = int(idxm.group(1))
                    item_meta[item_idx] = {
                        'maze_id': maze_id,
                        'source_image': source_img
                    }
            except Exception:
                continue

    try:
        data = json.loads(json_path.read_text(encoding='utf-8'))
        if not isinstance(data, list):
            return results

        for entry in data:
            image_path = str(entry.get('image_path', ''))
            # Extract item index from the filename
            try:
                item_match = re.search(r'item_(\d+)', Path(image_path).name, re.I)
            except Exception:
                item_match = None

            meta = None
            maze_id = None
            annotated_image = None
            if item_match:
                idx = int(item_match.group(1))
                meta = item_meta.get(idx)
                if meta:
                    maze_id = meta.get('maze_id')
                # Prefer generated image for NanoBanana
                gen_candidate = results_base_dir / f"item_{idx:05d}_generated_0.png"
                ann_candidate = results_base_dir / f"item_{idx:05d}_annotated.png"
                if gen_candidate.exists():
                    annotated_image = str(gen_candidate)
                elif ann_candidate.exists():
                    annotated_image = str(ann_candidate)

            # Fallback: derive maze_id from image_path if mapping missing
            if not maze_id:
                maze_id = extract_maze_id(image_path)

            if not maze_id:
                continue

            # Choose the answer: prefer extracted_answer, else parse model_answer
            extracted = (entry.get('extracted_answer') or '').strip()
            if extracted:
                final_answer = extracted.lower() if extracted.lower() in ('valid', 'invalid') else extract_answer_from_response(extracted)
                tail_text = entry.get('extracted_answer', '')
            else:
                model_answer = entry.get('model_answer', '')
                final_answer = extract_answer_from_response(model_answer)
                tail_text = model_answer

            results[maze_id] = build_result_entry(
                final_answer,
                model_output=tail_text,
                source_image=(meta.get('source_image') if meta else image_path),
                annotated_image=annotated_image,
            )

    except Exception as e:
        print(f"Error reading {json_path}: {e}")

    return results

def load_nanob_combined_json(json_path: Path) -> Dict[str, Dict[str, str]]:
    """Load unified NanoB + Gemini combined JSON produced by build_nanob_combined.py."""
    results: Dict[str, Dict[str, str]] = {}
    if not json_path.exists():
        return results
    try:
        data = json.loads(json_path.read_text(encoding='utf-8'))
        if not isinstance(data, list):
            return results
        for e in data:
            maze_id = e.get('maze_id')
            if not maze_id:
                continue
            results[maze_id] = build_result_entry(
                e.get('extracted_answer', 'unknown'),
                model_output=e.get('model_output', ''),
                source_image=e.get('source_image', ''),
                annotated_image=e.get('image_path', ''),
            )
    except Exception as ex:
        print(f'Error reading combined JSON {json_path}: {ex}')
    return results


def load_thinkmorph_results(thinkmorph_dir: Path) -> Dict[str, Dict[str, str]]:
    """Load ThinkMorph answers from per-sample directories."""
    results: Dict[str, Dict[str, str]] = {}

    if not thinkmorph_dir.exists():
        return results

    for sample_dir in thinkmorph_dir.iterdir():
        if not sample_dir.is_dir() or not sample_dir.name.startswith('sample_'):
            continue

        parts = sample_dir.name.split('_')
        maze_idx = None
        for i, part in enumerate(parts):
            if part == 'maze':
                maze_idx = i
                break

        if maze_idx is None:
            continue

        maze_id = '_'.join(parts[maze_idx:])
        json_file = sample_dir / 'text_data.json'
        if not json_file.exists():
            continue

        try:
            with open(json_file, 'r') as f:
                data = json.load(f)

            text_outputs = data.get('text_outputs', [])
            model_output = '\n'.join(text_outputs) if text_outputs else ''
            results[maze_id] = build_result_entry(
                extract_answer_from_response(model_output),
                model_output=model_output,
            )

        except Exception as e:
            print(f"Error processing {json_file}: {e}")

    return results


def main():
    print("Building maze to path length mapping...")
    maze_to_path_length = build_maze_to_path_length_mapping()
    print(f"Mapped {len(maze_to_path_length)} mazes from dataset")

    base_path = Path('/Users/log/Github/sketchvlm/results/mix_eval/maze_v2')

    # Define all models
    # Format: (model_name, model_base_path, model_prefix, format_type)
    # format_type: 'json' for standard, 'jsonl' for ViLaSR, 'thinkmorph' for ThinkMorph, 'consistency' for consistency check
    models = [
        ('gemini_flash_sketch', base_path / 'gemini', 'gemini25_flash', 'json'),
        ('gemini_flash_vqa', base_path / 'gemini' / 'direct_vqa', 'gemini25_flash', 'json'),
        ('gemini_flash_two_turn', base_path / 'gemini' / 'two_turn', 'gemini25_flash', 'json'),
        ('gemini_pro_sketch', base_path / 'gemini', 'gemini25_pro', 'json'),
        ('gemini_pro_vqa', base_path / 'gemini' / 'direct_vqa', 'gemini25_pro', 'json'),
        ('gemini_pro_two_turn', base_path / 'gemini' / 'two_turn', 'gemini25_pro', 'json'),
        ('gemini3_pro_sketch', base_path / 'gemini', 'gemini3_pro', 'json'),
        ('gemini3_pro_vqa', base_path / 'gemini' / 'direct_vqa', 'gemini3_pro', 'json'),
        ('gemini3_pro_0_1000_sketch', base_path / 'gemini', 'gemini3pro_gridworld_paths_0_to_1000', 'json'),
        ('gpt5_med_sketch', base_path / 'gpt5', 'gpt5_med', 'json'),
        ('gpt5_med_vqa', base_path / 'gpt5' / 'direct_vqa', 'gpt5_med', 'json'),
        ('gpt5_low_sketch', base_path / 'gpt5', 'gpt5_low', 'json'),
        ('gpt5_low_vqa', base_path / 'gpt5' / 'direct_vqa', 'gpt5_low', 'json'),
        ('gpt5_low_two_turn', base_path / 'gpt5' / 'two_turn', 'gpt5_low', 'json'),
        ('gpt5_low_1000_sketch', base_path / 'gpt5', 'gpt5_low_1000', 'json'),
        ('qwen3_235b_sketch', base_path / 'qwen3', 'qwen3_235b', 'json'),
        ('qwen3_235b_vqa', base_path / 'qwen3' / 'direct_vqa', 'qwen3_235b', 'json'),
        ('qwen25_7b_sketch', base_path / 'qwen25_7b', 'qwen25_7b', 'json'),
        ('qwen25_7b_vqa', base_path / 'qwen25_7b' / 'direct_vqa', 'qwen25_7b', 'json'),
        ('vilasr_sketch', base_path / 'vilasr', 'vilasr', 'jsonl'),
        ('thinkmorph_sketch', base_path / 'thinkmorph', 'thinkmorph', 'thinkmorph'),
        # Use consistency check format for NanoB + Gemini (from merged files)
        ('nanob_gemini3', base_path / 'gemini' / 'normal_gemini3', 'maze_nanob_gemini3pro_merged', 'consistency'),
    ]

    # Collect all maze IDs from results files (not just dataset)
    print("\nCollecting maze IDs from results...")
    all_maze_ids = set()

    # Scan one model's results to get all maze IDs
    ref_model_base = base_path / 'gemini'
    ref_model_prefix = 'gemini25_flash'

    for validity in ['invalid', 'valid']:
        results_dir = ref_model_base / f'{ref_model_prefix}_{validity}'
        if results_dir.exists():
            for json_file in results_dir.glob('item_*.json'):
                try:
                    with open(json_file, 'r') as f:
                        data = json.load(f)
                    source_image = data.get('source_image', '')
                    maze_id = extract_maze_id(source_image)
                    if maze_id:
                        all_maze_ids.add(maze_id)
                except Exception as e:
                    print(f"  Error reading {json_file}: {e}")

    print(f"Found {len(all_maze_ids)} unique mazes in results")

    # Load all model results
    print("\nLoading model results...")
    all_results = {}

    for model_name, model_base, model_prefix, format_type in models:
        print(f"  Loading {model_name}...")
        if format_type == 'jsonl':
            # Load from JSONL files
            invalid_jsonl = model_base / f'{model_prefix}_invalid' / 'results.jsonl'
            valid_jsonl = model_base / f'{model_prefix}_valid' / 'results.jsonl'
            all_results[model_name] = {
                'invalid': load_vilasr_jsonl_results(invalid_jsonl),
                'valid': load_vilasr_jsonl_results(valid_jsonl)
            }
        elif format_type == 'thinkmorph':
            # Load from ThinkMorph directories
            invalid_dir = model_base / f'{model_prefix}_invalid'
            valid_dir = model_base / f'{model_prefix}_valid'
            all_results[model_name] = {
                'invalid': load_thinkmorph_results(invalid_dir),
                'valid': load_thinkmorph_results(valid_dir)
            }
        elif format_type == 'consistency':
            # Load from consistency check JSON files
            invalid_json = model_base / f'{model_prefix}_invalid.json'
            valid_json = model_base / f'{model_prefix}_valid.json'
            # Get the original results directory for mapping
            # maze_nanob_gemini3pro_merged -> nanob_maze
            if 'nanob' in model_prefix:
                results_base_invalid = base_path / 'nano_banana' / 'nanob_maze_invalid'
                results_base_valid = base_path / 'nano_banana' / 'nanob_maze_valid'
            else:
                # Legacy handling: maze_nanob_gemini3_pro -> nanob_maze
                original_model = model_prefix.replace('_gemini3_pro', '')
                if original_model.startswith('maze_'):
                    original_model = original_model.replace('maze_', '', 1) + '_maze'
                results_base_invalid = base_path / 'nano_banana' / f'{original_model}_invalid'
                results_base_valid = base_path / 'nano_banana' / f'{original_model}_valid'
            all_results[model_name] = {
                'invalid': load_consistency_check_json(invalid_json, results_base_invalid),
                'valid': load_consistency_check_json(valid_json, results_base_valid)
            }
        elif format_type == 'nanob_combined':
            # Load from pre-built combined JSON arrays
            invalid_json = model_base / f'{model_prefix}_invalid.json'
            valid_json = model_base / f'{model_prefix}_valid.json'
            all_results[model_name] = {
                'invalid': load_nanob_combined_json(invalid_json),
                'valid': load_nanob_combined_json(valid_json)
            }
        else:
            # Load from individual JSON files
            all_results[model_name] = {
                'invalid': load_model_results(model_base, model_prefix, 'invalid'),
                'valid': load_model_results(model_base, model_prefix, 'valid')
            }

    # No in-memory combine; rely on prebuilt combined JSON

    # Expand maze set to include any IDs present in any loaded model results
    expanded_ids = set(all_maze_ids)
    for model_name, buckets in all_results.items():
        for validity in ('invalid', 'valid'):
            expanded_ids.update(buckets.get(validity, {}).keys())
    if len(expanded_ids) != len(all_maze_ids):
        print(f"\nExpanding maze set using loaded models: {len(all_maze_ids)} -> {len(expanded_ids)}")
    all_maze_ids = sorted(expanded_ids)

    # Build CSV rows
    print("\nBuilding CSV data...")
    rows = []

    # Check for mazes without path length info
    mazes_without_path_length = [m for m in all_maze_ids if m not in maze_to_path_length]
    if mazes_without_path_length:
        print(f"  Warning: {len(mazes_without_path_length)} mazes not found in dataset directories")
        print(f"  These will be marked with path_length='unknown'")

    model_names = [name for name, _, _, _ in models]

    def populate_row(row: Dict[str, str], validity: str, maze_id: str):
        row['source_image'] = ''
        for model_name in model_names:
            entry = all_results.get(model_name, {}).get(validity, {}).get(maze_id)
            if entry:
                row[model_name] = entry.get('answer', 'missing')
                row[f'{model_name}__output_tail'] = entry.get('output_tail', '')
                row[f'{model_name}__annotated_image'] = entry.get('annotated_image', '')
                if not row['source_image'] and entry.get('source_image'):
                    row['source_image'] = entry['source_image']
            else:
                row[model_name] = 'missing'
                row[f'{model_name}__output_tail'] = ''
                row[f'{model_name}__annotated_image'] = ''

    for maze_id in sorted(all_maze_ids):
        path_length = maze_to_path_length.get(maze_id, 'unknown')

        invalid_row = {
            'maze_id': maze_id,
            'path_length': path_length,
            'validity': 'invalid',
            'ground_truth': 'invalid'
        }
        populate_row(invalid_row, 'invalid', maze_id)
        rows.append(invalid_row)

        valid_row = {
            'maze_id': maze_id,
            'path_length': path_length,
            'validity': 'valid',
            'ground_truth': 'valid'
        }
        populate_row(valid_row, 'valid', maze_id)
        rows.append(valid_row)

    # Write CSV
    output_path = Path('/Users/log/Github/sketchvlm/analysis/maze/maze_v2_combined_results.csv')

    print(f"\nWriting CSV to {output_path}...")

    fieldnames = ['maze_id', 'path_length', 'validity', 'ground_truth', 'source_image']
    for model_name, _, _, _ in models:
        fieldnames.append(model_name)
    for model_name, _, _, _ in models:
        fieldnames.append(f'{model_name}__output_tail')
        fieldnames.append(f'{model_name}__annotated_image')

    with open(output_path, 'w', newline='') as csvfile:
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
        writer.writeheader()
        writer.writerows(rows)

    print(f"\nSuccessfully wrote {len(rows)} rows to {output_path}")

    # Print summary statistics
    print("\nSummary:")
    print(f"  Total unique mazes: {len(all_maze_ids)}")
    print(f"  Total rows (maze × validity): {len(rows)}")
    print(f"  Models included: {len(models)}")
    print(f"  Columns: {len(fieldnames)}")

    # Check for missing data
    print("\nData completeness check:")
    for model_name, _, _, _ in models:
        invalid_count = sum(1 for row in rows if row['validity'] == 'invalid' and row[model_name] != 'missing')
        valid_count = sum(1 for row in rows if row['validity'] == 'valid' and row[model_name] != 'missing')
        total_invalid = sum(1 for row in rows if row['validity'] == 'invalid')
        total_valid = sum(1 for row in rows if row['validity'] == 'valid')

        print(f"  {model_name}:")
        print(f"    Invalid: {invalid_count}/{total_invalid}")
        print(f"    Valid: {valid_count}/{total_valid}")


if __name__ == '__main__':
    main()