https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
feat(eval): add OrcaRouter as a first-class reader provider (#146)
kuswardhanietidims-svgbd4f0d548fb4
diff --git a/eval/lib/model_config.py b/eval/lib/model_config.pyindex d4c226d..24973b5 100644--- a/eval/lib/model_config.py+++ b/eval/lib/model_config.py@@ -36,6 +36,11 @@ MINIMAX_MODEL_METADATA = {},}+# OrcaRouter is an OpenAI-compatible gateway, the same shape as the OpenRouter+# and Commonstack aggregators wired up in run_bench.py. Defined here so the CLI+# and the model-config branch cannot drift to different endpoints.+ORCAROUTER_API_BASE = "https://api.orcarouter.ai/v1"+MINIMAX_ENDPOINTS = {"global_en": {"api_base": "https://api.minimax.io/v1",@@ -96,6 +101,14 @@ def get_model_config(model_name: str) -> Dict[str, Any]:**MINIMAX_MODEL_METADATA.get(minimax_key, {}),}+ # OrcaRouter models (OpenAI-compatible gateway)+ if "orcarouter" in model_lower:+ return {+ "api_base": os.getenv("ORCAROUTER_API_BASE", ORCAROUTER_API_BASE),+ "api_key": os.getenv("ORCAROUTER_API_KEY", os.getenv("API_KEY", "dummy")),+ "model": model_name,+ }+# Gemini modelsif "gemini" in model_lower:# Check for Vertex AI firstdiff --git a/eval/run_bench.py b/eval/run_bench.pyindex e7ca40f..39856dd 100644--- a/eval/run_bench.py+++ b/eval/run_bench.py@@ -14,6 +14,9 @@ Usage:# OpenRouter API (no local vLLM)python run_bench.py --task simpleqa --model openai/gpt-4 --open-router --api-key sk-or-v1-xxx++ # OrcaRouter API (no local vLLM)+ python run_bench.py --task simpleqa --model anthropic/claude-opus-4-8 --orcarouter"""import argparse@@ -68,7 +71,11 @@ from lib.benchmarks import (load_webqa_data,load_worldvqa_data,)-from lib.model_config import get_model_config, get_output_filename+from lib.model_config import (+ ORCAROUTER_API_BASE,+ get_model_config,+ get_output_filename,+)from lib.retrieval import (_get_query_image_path_for_example,_save_task_query_image,@@ -977,6 +984,18 @@ async def run_async(args):)logger.info(f"Using Commonstack API with model: {args.model}")model = args.model+ elif args.orcarouter:+ api_base = ORCAROUTER_API_BASE+ if args.api_key and args.api_key != "dummy":+ api_key = args.api_key+ else:+ api_key = os.getenv("ORCAROUTER_API_KEY")+ if not api_key or api_key == "dummy":+ raise ValueError(+ "OrcaRouter API key required. Set --api-key or ORCAROUTER_API_KEY environment variable."+ )+ logger.info(f"Using OrcaRouter API with model: {args.model}")+ model = args.modelelif args.litellm:# LiteLLM SDK: the provider-prefixed --model selects the backend, and# credentials come from each provider's own env var (or --api-base for a@@ -1114,7 +1133,7 @@ async def run_async(args):max_context_tokens=args.model_context_length,timeout=args.timeout,enable_thinking=(False if args.no_think else None),- force_openai_compat=(args.open_router or args.commonstack),+ force_openai_compat=(args.open_router or args.commonstack or args.orcarouter),use_litellm=args.litellm,)@@ -1332,6 +1351,11 @@ def main():action="store_true",help="Use Commonstack API (https://api.commonstack.ai). Requires --api-key or COMMONSTACK_API_KEY env var.",)+ parser.add_argument(+ "--orcarouter",+ action="store_true",+ help="Use OrcaRouter API (https://api.orcarouter.ai). Requires --api-key or ORCAROUTER_API_KEY env var.",+ )parser.add_argument("--litellm",action="store_true",diff --git a/tests/test_eval_model_config.py b/tests/test_eval_model_config.pyindex d456a45..3ecb6a7 100644--- a/tests/test_eval_model_config.py+++ b/tests/test_eval_model_config.py@@ -183,3 +183,40 @@ def test_minimax_model_registered_without_metadata_still_resolves(monkeypatch):assert config["api_base"] == "https://api.minimax.io/v1"assert config["anthropic_api_base"] == "https://api.minimax.io/anthropic"assert "context_window" not in config+++def test_orcarouter_model_config_uses_the_gateway_default(monkeypatch):+ """The branch referenced a constant that was never defined.++ `os.getenv("ORCAROUTER_API_BASE", ORCAROUTER_API_BASE)` raised NameError for+ every model routed through the gateway, so the provider could not be used at+ all -- and ruff's F821 failed the lint gate for the whole repo.+ """+ monkeypatch.delenv("ORCAROUTER_API_BASE", raising=False)+ monkeypatch.delenv("ORCAROUTER_API_KEY", raising=False)+ monkeypatch.setenv("API_KEY", "fallback-key")++ config = get_model_config("anthropic/claude-opus-4-8-orcarouter")++ assert config["api_base"] == "https://api.orcarouter.ai/v1"+ assert config["api_key"] == "fallback-key"+ assert config["model"] == "anthropic/claude-opus-4-8-orcarouter"+++def test_orcarouter_model_config_supports_endpoint_and_key_overrides(monkeypatch):+ monkeypatch.setenv("ORCAROUTER_API_BASE", "https://gateway.internal/v1")+ monkeypatch.setenv("ORCAROUTER_API_KEY", "sk-orca")++ config = get_model_config("gpt-4o-orcarouter")++ assert config["api_base"] == "https://gateway.internal/v1"+ assert config["api_key"] == "sk-orca"+++def test_orcarouter_branch_does_not_capture_unrelated_models(monkeypatch):+ """`"orcarouter" in model_lower` is a substring match; keep it narrow."""+ monkeypatch.setenv("API_BASE", "http://localhost:9000/v1")++ config = get_model_config("Qwen/Qwen3-VL-4B-Instruct")++ assert config["api_base"] == "http://localhost:9000/v1"