https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
feat(eval): add optional Cheaper Inference reader (#163)
aiapienthusiast9106cf44e62c
diff --git a/eval/README.md b/eval/README.mdindex bfede20..69ae8ac 100644--- a/eval/README.md+++ b/eval/README.md@@ -194,6 +194,28 @@ with another provider flag or `--api-base`.Atlas requests are not automatically retried, including timeouts, connectionerrors and rate limits, to avoid duplicating billable generation requests.+## Cheaper Inference readers++[Cheaper Inference](https://cheaperinference.com) is an OpenAI-compatible LLM gateway.+Each model costs 15–60% less than the list price of its lab.++Select it explicitly with `--cheaperinference`; other readers are unchanged.+Set `CHEAPER_INFERENCE_API_KEY` and use a bare model ID from the+[model list](https://cheaperinference.com/#models):++```bash+export CHEAPER_INFERENCE_API_KEY=ci_live_...+python run_bench.py --task simpleqa --model gpt-5.4-mini \+ --cheaperinference --num-examples 1 --max-concurrent 1 --max-tokens 256+```++This text-only example does not use retrieval. For screenshot benchmarks, use+`gpt-5.4-mini` or `gpt-5.4`, which accept image input. Model IDs are forwarded+unchanged to `https://api.cheaperinference.com/v1` and never select the native+Google SDK. `--api-key` overrides `CHEAPER_INFERENCE_API_KEY`, and other providers'+environment keys are never used. Do not combine this option with another provider+flag or `--api-base`.+## MiniMax readersSet `MINIMAX_API_KEY` and select either registered model ID. The model context lengthdiff --git a/eval/lib/model_config.py b/eval/lib/model_config.pyindex 6fe6c3d..0b89844 100644--- a/eval/lib/model_config.py+++ b/eval/lib/model_config.py@@ -68,6 +68,23 @@ def get_atlascloud_config(}+def get_cheaperinference_config(+ model_name: str, api_key: str | None = None+) -> Dict[str, Any]:+ """Resolve an explicitly selected Cheaper Inference reader without native routing."""+ if not api_key or api_key == "dummy":+ api_key = os.getenv("CHEAPER_INFERENCE_API_KEY")+ if not api_key or not api_key.strip() or api_key == "dummy":+ raise ValueError(+ "Cheaper Inference requires --api-key or CHEAPER_INFERENCE_API_KEY."+ )+ return {+ "api_base": "https://api.cheaperinference.com/v1",+ "api_key": api_key,+ "model": model_name,+ }++def get_model_config(model_name: str) -> Dict[str, Any]:"""Get model configuration based on model name.diff --git a/eval/run_bench.py b/eval/run_bench.pyindex 05361db..13fbdc1 100644--- a/eval/run_bench.py+++ b/eval/run_bench.py@@ -74,6 +74,7 @@ from lib.benchmarks import (from lib.model_config import (ORCAROUTER_API_BASE,get_atlascloud_config,+ get_cheaperinference_config,get_model_config,get_output_filename,)@@ -958,11 +959,12 @@ async def run_async(args):)# Get model configuration- model_config = (- get_atlascloud_config(args.model, args.api_key)- if args.atlascloud- else get_model_config(args.model)- )+ if args.atlascloud:+ model_config = get_atlascloud_config(args.model, args.api_key)+ elif args.cheaperinference:+ model_config = get_cheaperinference_config(args.model, args.api_key)+ else:+ model_config = get_model_config(args.model)# Handle OpenRouter APIif args.atlascloud:@@ -970,6 +972,11 @@ async def run_async(args):api_key = model_config["api_key"]model = model_config["model"]logger.info(f"Using Atlas Cloud API with model: {model}")+ elif args.cheaperinference:+ api_base = model_config["api_base"]+ api_key = model_config["api_key"]+ model = model_config["model"]+ logger.info(f"Using Cheaper Inference API with model: {model}")elif args.open_router:api_base = "https://openrouter.ai/api/v1"if args.api_key and args.api_key != "dummy":@@ -1144,7 +1151,11 @@ async def run_async(args):timeout=args.timeout,enable_thinking=(False if args.no_think else None),force_openai_compat=(- args.open_router or args.commonstack or args.orcarouter or args.atlascloud+ args.open_router+ or args.commonstack+ or args.orcarouter+ or args.atlascloud+ or args.cheaperinference),use_litellm=args.litellm,retry_requests=not args.atlascloud,@@ -1374,6 +1385,11 @@ def main():action="store_true",help="Use Atlas Cloud with an exact catalog model ID. Requires --api-key or ATLASCLOUD_API_KEY.",)+ parser.add_argument(+ "--cheaperinference",+ action="store_true",+ help="Use Cheaper Inference (https://api.cheaperinference.com/v1) with a bare model ID. Requires --api-key or CHEAPER_INFERENCE_API_KEY.",+ )parser.add_argument("--litellm",action="store_true",@@ -1932,6 +1948,16 @@ def main():parser.error("--atlascloud cannot be combined with another provider flag")if args.atlascloud and args.api_base:parser.error("--atlascloud uses its fixed endpoint; omit --api-base")+ if args.cheaperinference and (+ args.open_router+ or args.commonstack+ or args.orcarouter+ or args.atlascloud+ or args.litellm+ ):+ parser.error("--cheaperinference cannot be combined with another provider flag")+ if args.cheaperinference and args.api_base:+ parser.error("--cheaperinference uses its fixed endpoint; omit --api-base")# Validate mutually exclusive optionsmode_count = sum(diff --git a/tests/test_cheaperinference_client.py b/tests/test_cheaperinference_client.pynew file mode 100644index 0000000..a101ad2--- /dev/null+++ b/tests/test_cheaperinference_client.py@@ -0,0 +1,95 @@+"""Offline coverage of the optional Cheaper Inference evaluation reader."""++import asyncio+import sys+import types+from pathlib import Path+from runpy import run_path++import pytest++_LIB = Path(__file__).parents[1] / "eval" / "lib"+if "lib" not in sys.modules:+ _pkg = types.ModuleType("lib")+ _pkg.__path__ = [str(_LIB)]+ sys.modules["lib"] = _pkg++from lib.llm import LLMClient++get_config = run_path(str(_LIB / "model_config.py"))["get_cheaperinference_config"]++[email protected]("model", ["gpt-5.4-mini", "gemini-3.1-pro"])+def test_cheaperinference_preserves_model_and_isolates_credentials(monkeypatch, model):+ monkeypatch.setenv("CHEAPER_INFERENCE_API_KEY", "ci-test-key")+ monkeypatch.setenv("OPENAI_API_KEY", "other-key")+ monkeypatch.setenv("API_KEY", "generic-key")+ config = get_config(model)+ assert config == {+ "model": model,+ "api_base": "https://api.cheaperinference.com/v1",+ "api_key": "ci-test-key",+ }+ assert get_config(model, "explicit-key")["api_key"] == "explicit-key"++[email protected]("key", [None, "", " ", "dummy"])+def test_cheaperinference_requires_own_key(monkeypatch, key):+ monkeypatch.delenv("CHEAPER_INFERENCE_API_KEY", raising=False)+ if key is not None:+ monkeypatch.setenv("CHEAPER_INFERENCE_API_KEY", key)+ monkeypatch.setenv("API_KEY", "generic-key")+ monkeypatch.setenv("OPENAI_API_KEY", "openai-key")+ with pytest.raises(ValueError, match="CHEAPER_INFERENCE_API_KEY"):+ get_config("gpt-5.4-mini")++[email protected]("model", ["gpt-5.4-mini", "gemini-3.1-pro"])+def test_cheaperinference_client_forwards_image_messages(monkeypatch, model):+ calls = []+ options = {}++ async def create(**kwargs):+ calls.append(kwargs)+ return types.SimpleNamespace(+ choices=[types.SimpleNamespace(message=types.SimpleNamespace(content="4"))],+ usage=types.SimpleNamespace(+ prompt_tokens=3, completion_tokens=1, total_tokens=4+ ),+ )++ def openai_client(**kwargs):+ options.update(kwargs)+ return types.SimpleNamespace(+ chat=types.SimpleNamespace(completions=types.SimpleNamespace(create=create))+ )++ monkeypatch.setitem(+ sys.modules, "openai", types.SimpleNamespace(AsyncOpenAI=openai_client)+ )+ client = LLMClient(+ **get_config(model, "ci-test-key"),+ force_openai_compat=True,+ max_tokens=64,+ )+ messages = [+ {+ "role": "user",+ "content": [+ {+ "type": "image_url",+ "image_url": {"url": "data:image/png;base64,AAAA"},+ },+ {"type": "text", "text": "2+2?"},+ ],+ }+ ]+ text, usage = asyncio.run(client.generate(messages))+ assert text == "4"+ assert usage["total_tokens"] == 4+ assert len(calls) == 1+ assert calls[0]["model"] == model+ assert calls[0]["messages"] == messages+ assert options["base_url"] == "https://api.cheaperinference.com/v1"+ assert options["api_key"] == "ci-test-key"+ assert not client.is_gemini