Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

feat(eval): add OrcaRouter as a first-class reader provider (#146)

kuswardhanietidims-svgbd4f0d548fb4
diff --git a/eval/lib/model_config.py b/eval/lib/model_config.py
index d4c226d..24973b5 100644
--- a/eval/lib/model_config.py
+++ b/eval/lib/model_config.py
@@ -36,6 +36,11 @@ MINIMAX_MODEL_METADATA = {
},
}
+# OrcaRouter is an OpenAI-compatible gateway, the same shape as the OpenRouter
+# and Commonstack aggregators wired up in run_bench.py. Defined here so the CLI
+# and the model-config branch cannot drift to different endpoints.
+ORCAROUTER_API_BASE = "https://api.orcarouter.ai/v1"
+
MINIMAX_ENDPOINTS = {
"global_en": {
"api_base": "https://api.minimax.io/v1",
@@ -96,6 +101,14 @@ def get_model_config(model_name: str) -> Dict[str, Any]:
**MINIMAX_MODEL_METADATA.get(minimax_key, {}),
}
+ # OrcaRouter models (OpenAI-compatible gateway)
+ if "orcarouter" in model_lower:
+ return {
+ "api_base": os.getenv("ORCAROUTER_API_BASE", ORCAROUTER_API_BASE),
+ "api_key": os.getenv("ORCAROUTER_API_KEY", os.getenv("API_KEY", "dummy")),
+ "model": model_name,
+ }
+
# Gemini models
if "gemini" in model_lower:
# Check for Vertex AI first
diff --git a/eval/run_bench.py b/eval/run_bench.py
index e7ca40f..39856dd 100644
--- a/eval/run_bench.py
+++ b/eval/run_bench.py
@@ -14,6 +14,9 @@ Usage:
# OpenRouter API (no local vLLM)
python run_bench.py --task simpleqa --model openai/gpt-4 --open-router --api-key sk-or-v1-xxx
+
+ # OrcaRouter API (no local vLLM)
+ python run_bench.py --task simpleqa --model anthropic/claude-opus-4-8 --orcarouter
"""
import argparse
@@ -68,7 +71,11 @@ from lib.benchmarks import (
load_webqa_data,
load_worldvqa_data,
)
-from lib.model_config import get_model_config, get_output_filename
+from lib.model_config import (
+ ORCAROUTER_API_BASE,
+ get_model_config,
+ get_output_filename,
+)
from lib.retrieval import (
_get_query_image_path_for_example,
_save_task_query_image,
@@ -977,6 +984,18 @@ async def run_async(args):
)
logger.info(f"Using Commonstack API with model: {args.model}")
model = args.model
+ elif args.orcarouter:
+ api_base = ORCAROUTER_API_BASE
+ if args.api_key and args.api_key != "dummy":
+ api_key = args.api_key
+ else:
+ api_key = os.getenv("ORCAROUTER_API_KEY")
+ if not api_key or api_key == "dummy":
+ raise ValueError(
+ "OrcaRouter API key required. Set --api-key or ORCAROUTER_API_KEY environment variable."
+ )
+ logger.info(f"Using OrcaRouter API with model: {args.model}")
+ model = args.model
elif args.litellm:
# LiteLLM SDK: the provider-prefixed --model selects the backend, and
# credentials come from each provider's own env var (or --api-base for a
@@ -1114,7 +1133,7 @@ async def run_async(args):
max_context_tokens=args.model_context_length,
timeout=args.timeout,
enable_thinking=(False if args.no_think else None),
- force_openai_compat=(args.open_router or args.commonstack),
+ force_openai_compat=(args.open_router or args.commonstack or args.orcarouter),
use_litellm=args.litellm,
)
@@ -1332,6 +1351,11 @@ def main():
action="store_true",
help="Use Commonstack API (https://api.commonstack.ai). Requires --api-key or COMMONSTACK_API_KEY env var.",
)
+ parser.add_argument(
+ "--orcarouter",
+ action="store_true",
+ help="Use OrcaRouter API (https://api.orcarouter.ai). Requires --api-key or ORCAROUTER_API_KEY env var.",
+ )
parser.add_argument(
"--litellm",
action="store_true",
diff --git a/tests/test_eval_model_config.py b/tests/test_eval_model_config.py
index d456a45..3ecb6a7 100644
--- a/tests/test_eval_model_config.py
+++ b/tests/test_eval_model_config.py
@@ -183,3 +183,40 @@ def test_minimax_model_registered_without_metadata_still_resolves(monkeypatch):
assert config["api_base"] == "https://api.minimax.io/v1"
assert config["anthropic_api_base"] == "https://api.minimax.io/anthropic"
assert "context_window" not in config
+
+
+def test_orcarouter_model_config_uses_the_gateway_default(monkeypatch):
+ """The branch referenced a constant that was never defined.
+
+ `os.getenv("ORCAROUTER_API_BASE", ORCAROUTER_API_BASE)` raised NameError for
+ every model routed through the gateway, so the provider could not be used at
+ all -- and ruff's F821 failed the lint gate for the whole repo.
+ """
+ monkeypatch.delenv("ORCAROUTER_API_BASE", raising=False)
+ monkeypatch.delenv("ORCAROUTER_API_KEY", raising=False)
+ monkeypatch.setenv("API_KEY", "fallback-key")
+
+ config = get_model_config("anthropic/claude-opus-4-8-orcarouter")
+
+ assert config["api_base"] == "https://api.orcarouter.ai/v1"
+ assert config["api_key"] == "fallback-key"
+ assert config["model"] == "anthropic/claude-opus-4-8-orcarouter"
+
+
+def test_orcarouter_model_config_supports_endpoint_and_key_overrides(monkeypatch):
+ monkeypatch.setenv("ORCAROUTER_API_BASE", "https://gateway.internal/v1")
+ monkeypatch.setenv("ORCAROUTER_API_KEY", "sk-orca")
+
+ config = get_model_config("gpt-4o-orcarouter")
+
+ assert config["api_base"] == "https://gateway.internal/v1"
+ assert config["api_key"] == "sk-orca"
+
+
+def test_orcarouter_branch_does_not_capture_unrelated_models(monkeypatch):
+ """`"orcarouter" in model_lower` is a substring match; keep it narrow."""
+ monkeypatch.setenv("API_BASE", "http://localhost:9000/v1")
+
+ config = get_model_config("Qwen/Qwen3-VL-4B-Instruct")
+
+ assert config["api_base"] == "http://localhost:9000/v1"