Skip to content
Python
3.6 KB
Raw Blame
#!/usr/bin/env python3
"""
Find all item numbers where the model response is unknown (no answer extracted).
"""

import json
import re
import sys
from pathlib import Path


def extract_answer_from_response(response_text: str) -> str:
    """Extract answer from model response."""
    if not response_text or response_text.strip() == '':
        return 'unknown'

    # Try <final_answer> tags first
    final_answer_match = re.search(r'<final_answer>\s*(.*?)\s*</final_answer>',
                                   response_text, re.IGNORECASE | re.DOTALL)
    if final_answer_match:
        answer_text = final_answer_match.group(1).strip()
        if 'valid' in answer_text.lower():
            if 'invalid' in answer_text.lower():
                return 'invalid'
            else:
                return 'valid'
        return 'unknown'

    # Try <answer> tags (for ViLaSR)
    answer_match = re.search(r'<answer>\s*(.*?)\s*</answer>',
                            response_text, re.IGNORECASE | re.DOTALL)
    if answer_match:
        answer_text = answer_match.group(1).strip()
        if 'valid' in answer_text.lower():
            if 'invalid' in answer_text.lower():
                return 'invalid'
            else:
                return 'valid'
        return 'unknown'

    # Try \boxed{} format (for ViLaSR)
    boxed_match = re.search(r'\$?\\boxed\{(.*?)\}\$?',
                           response_text, re.IGNORECASE | re.DOTALL)
    if boxed_match:
        answer_text = boxed_match.group(1).strip()
        if 'valid' in answer_text.lower():
            if 'invalid' in answer_text.lower():
                return 'invalid'
            else:
                return 'valid'
        return 'unknown'

    # Fallback: check last 30 characters
    last_chars = response_text[-30:].lower()
    if 'invalid' in last_chars:
        return 'invalid'
    elif 'valid' in last_chars:
        return 'valid'

    return 'unknown'


def find_unknown_items(results_dir: Path) -> list:
    """Find all item numbers with unknown responses."""
    unknown_items = []

    if not results_dir.exists():
        print(f"Error: Directory {results_dir} does not exist", file=sys.stderr)
        return unknown_items

    json_files = list(results_dir.glob('item_*.json'))

    for json_file in json_files:
        try:
            with open(json_file, 'r') as f:
                data = json.load(f)

            model_output = data.get('model_output_full', '')
            answer = extract_answer_from_response(model_output)

            if answer == 'unknown':
                # Extract item number from filename
                item_num = json_file.stem.replace('item_', '')
                unknown_items.append(int(item_num))

        except Exception as e:
            print(f"Error processing {json_file}: {e}", file=sys.stderr)

    return sorted(unknown_items)


def main():
    if len(sys.argv) < 2:
        print("Usage: python find_none_responses.py <results_directory>", file=sys.stderr)
        print("\nExample:", file=sys.stderr)
        print("  python find_none_responses.py results/mix_eval/maze_v2/gpt5/gpt5_low_1000_valid", file=sys.stderr)
        sys.exit(1)

    results_dir = Path(sys.argv[1])
    unknown_items = find_unknown_items(results_dir)

    if unknown_items:
        item_list = ','.join(str(n) for n in unknown_items)
        print(f'"{item_list}"')
    else:
        print('""')
        print(f"No unknown responses found in {results_dir}", file=sys.stderr)


if __name__ == '__main__':
    main()