Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

8.8 KB
Raw Blame
"""Model configurations for SimpleQA evaluation.

This module provides model configurations to keep run_naive_simpleqa.py clean.
"""

import os
from typing import Any, Dict

MINIMAX_MODELS = {
    "minimax-m3.1-flash": "MiniMax-M3.1-Flash",
    "minimax-m3": "MiniMax-M3",
}

# Resolve explicit legacy selections without advertising them as current models.
MINIMAX_LEGACY_MODELS = {"minimax-m2.7": "MiniMax-M2.7"}

MINIMAX_MODEL_METADATA = {
    "minimax-m3.1-flash": {
        "context_window": 1_000_000,
        "max_output_tokens": 524_288,
        "pricing_usd_per_million_tokens": {
            "input": 0.15,
            "output": 0.6,
            "cache_read": 0.015,
            "cache_write": None,
        },
        "input_modalities": ["text", "image", "video"],
        "thinking": ["adaptive"],
        "reasoning_effort": ["low", "medium", "high", "xhigh", "max"],
        "default_reasoning_effort": "max",
    },
    "minimax-m3": {
        "context_window": 1_000_000,
        # Standard-tier pricing up to 512k input tokens.
        "pricing_usd_per_million_tokens": {
            "input": 0.3,
            "output": 1.2,
            "cache_read": 0.06,
            "cache_write": None,
        },
        "long_context_pricing_usd_per_million_tokens": {
            "input_range": ">512k",
            "input": 0.6,
            "output": 2.4,
            "cache_read": 0.12,
            "cache_write": None,
        },
        "input_modalities": ["text", "image", "video"],
        "thinking": ["adaptive", "disabled"],
    },
    "minimax-m2.7": {
        "context_window": 204_800,
        "pricing_usd_per_million_tokens": {
            "input": 0.3,
            "output": 1.2,
            "cache_read": 0.06,
            "cache_write": 0.375,
        },
        "input_modalities": ["text"],
        "thinking": ["always_on"],
    },
}

# OrcaRouter is an OpenAI-compatible gateway, the same shape as the OpenRouter
# and Commonstack aggregators wired up in run_bench.py. Defined here so the CLI
# and the model-config branch cannot drift to different endpoints.
ORCAROUTER_API_BASE = "https://api.orcarouter.ai/v1"

MINIMAX_ENDPOINTS = {
    "global_en": {
        "api_base": "https://api.minimax.io/v1",
        "anthropic_api_base": "https://api.minimax.io/anthropic",
    },
    "cn_zh": {
        "api_base": "https://api.minimaxi.com/v1",
        "anthropic_api_base": "https://api.minimaxi.com/anthropic",
    },
}


def get_atlascloud_config(
    model_name: str, api_key: str | None = None
) -> Dict[str, Any]:
    """Resolve an explicitly selected Atlas Cloud reader without native routing."""
    if not api_key or api_key == "dummy":
        api_key = os.getenv("ATLASCLOUD_API_KEY")
    if not api_key or not api_key.strip() or api_key == "dummy":
        raise ValueError("Atlas Cloud requires --api-key or ATLASCLOUD_API_KEY.")
    return {
        "api_base": "https://api.atlascloud.ai/v1",
        "api_key": api_key,
        "model": model_name,
    }


def get_cheaperinference_config(
    model_name: str, api_key: str | None = None
) -> Dict[str, Any]:
    """Resolve an explicitly selected Cheaper Inference reader without native routing."""
    if not api_key or api_key == "dummy":
        api_key = os.getenv("CHEAPER_INFERENCE_API_KEY")
    if not api_key or not api_key.strip() or api_key == "dummy":
        raise ValueError(
            "Cheaper Inference requires --api-key or CHEAPER_INFERENCE_API_KEY."
        )
    return {
        "api_base": "https://api.cheaperinference.com/v1",
        "api_key": api_key,
        "model": model_name,
    }


def get_model_config(model_name: str) -> Dict[str, Any]:
    """
    Get model configuration based on model name.

    Args:
        model_name: Name of the model (e.g., 'Qwen/Qwen3-VL-4B-Instruct', 'gemini-3-pro-preview')

    Returns:
        Dictionary with endpoint, authentication, model, and capability metadata.
    """
    model_lower = model_name.lower()

    # MiniMax models (OpenAI-compatible API)
    minimax_key = model_lower.rsplit("/", 1)[-1]
    minimax_model = MINIMAX_MODELS.get(minimax_key) or MINIMAX_LEGACY_MODELS.get(
        minimax_key
    )
    if minimax_model:
        # The two bases are independent overrides for gateways and proxies, so
        # resolve them first and consult a region only for whichever was left
        # unset. Validating the region up front rejected deployments that had
        # pinned both endpoints and were not using a region at all.
        api_base = os.getenv("MINIMAX_API_BASE")
        anthropic_api_base = os.getenv("MINIMAX_ANTHROPIC_API_BASE")
        if api_base is None or anthropic_api_base is None:
            region = os.getenv("MINIMAX_API_REGION", "global_en").lower()
            if region not in MINIMAX_ENDPOINTS:
                supported = ", ".join(MINIMAX_ENDPOINTS)
                raise ValueError(
                    f"Unsupported MINIMAX_API_REGION {region!r}; "
                    f"expected one of: {supported}"
                )
            endpoints = MINIMAX_ENDPOINTS[region]
            if api_base is None:
                api_base = endpoints["api_base"]
            if anthropic_api_base is None:
                anthropic_api_base = endpoints["anthropic_api_base"]

        return {
            "api_base": api_base,
            "anthropic_api_base": anthropic_api_base,
            "api_key": os.getenv("MINIMAX_API_KEY", os.getenv("API_KEY", "dummy")),
            "model": minimax_model,
            # MINIMAX_MODELS is the registry; this is a side table keyed off it
            # with nothing enforcing that the two agree. A model listed in the
            # registry has to resolve whether or not its capability entry has
            # been filled in — the caller reads api_base, api_key and model.
            **MINIMAX_MODEL_METADATA.get(minimax_key, {}),
        }

    # OrcaRouter models (OpenAI-compatible gateway)
    if "orcarouter" in model_lower:
        return {
            "api_base": os.getenv("ORCAROUTER_API_BASE", ORCAROUTER_API_BASE),
            "api_key": os.getenv("ORCAROUTER_API_KEY", os.getenv("API_KEY", "dummy")),
            "model": model_name,
        }

    # Gemini models
    if "gemini" in model_lower:
        # Check for Vertex AI first
        vertex_api_key = os.getenv("GEMINI_API_KEY")
        use_vertex = os.getenv("GOOGLE_GENAI_USE_VERTEXAI", "").lower() == "true"

        if use_vertex and vertex_api_key:
            # Using Vertex AI - don't pass api_key, use environment variable instead
            api_key = None  # Vertex AI uses environment variable, not api_key parameter
        else:
            # Using standard Gemini API
            api_key = os.getenv("GOOGLE_API_KEY") or os.getenv("GEMINI_API_KEY")
            if not api_key:
                raise ValueError(
                    "GOOGLE_API_KEY or GEMINI_API_KEY environment variable is required for Gemini models. "
                    "Set it with: export GOOGLE_API_KEY='your-api-key' or export GEMINI_API_KEY='your-api-key' and GOOGLE_GENAI_USE_VERTEXAI=true"
                )

        # For Gemini models, we use Google's Generative AI SDK directly
        # The api_base is not used for Gemini (SDK handles it internally)
        # But we set a placeholder for compatibility
        api_base = None  # Not used for Gemini SDK

        return {
            "api_base": api_base,
            "api_key": api_key,
            "model": model_name,  # Use the model name as-is
        }

    # Default: assume OpenAI-compatible API (vLLM, etc.)
    return {
        "api_base": os.getenv("API_BASE", "http://localhost:8000/v1"),
        "api_key": os.getenv("API_KEY", "dummy"),
        "model": model_name,
    }


def get_output_filename(
    output_dir: str,
    model_name: str,
    mode: str = "naive",
    num_examples: int = 1000,
    url_screenshot: bool = False,
    task: str = "simpleqa",
) -> str:
    """
    Generate output filename with model name and task included.

    Args:
        output_dir: Base output directory (e.g., 'eval_output/naive_qa')
        model_name: Model name (e.g., 'Qwen/Qwen3-VL-4B-Instruct')
        mode: Evaluation mode ('naive', 'screenshot', 'retrieval')
        num_examples: Number of examples
        url_screenshot: Whether URL screenshot mode is enabled
        task: Task/benchmark name (e.g., 'simpleqa', 'encyclopedic_vqa', 'worldvqa')

    Returns:
        Full output file path
    """
    # Clean model name for filename (replace special chars)
    model_safe = (
        model_name.replace("/", "_").replace(":", "_").replace("-", "_").lower()
    )

    # Build filename components (task first for easy distinction)
    parts = [task]
    if url_screenshot:
        parts.append("urlscreenshot")
    parts.append(mode)
    parts.append(model_safe)
    parts.append(str(num_examples))

    filename = "_".join(parts) + ".jsonl"
    return os.path.join(output_dir, filename)