Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

feat: add LiteLLM SDK backend to eval client (#145)

Prodman Devokadev0ecf227d2f7d
diff --git a/eval/lib/llm.py b/eval/lib/llm.py
index 4fe88c4..0f4879b 100644
--- a/eval/lib/llm.py
+++ b/eval/lib/llm.py
@@ -35,6 +35,11 @@ from .retrieval import RetrievalResult
logger = logging.getLogger(__name__)
+# Some modern reasoning models deprecate `temperature` (Claude Opus 4.7+, some
+# GPT-5 variants). Both backend paths consult this list, so a newly released
+# model is added in one place rather than drifting between them.
+MODELS_DROPPING_TEMPERATURE = ("opus-4-7", "opus-4-8", "gpt-5.4-pro")
+
# System Prompts
SYSTEM_PROMPT_NAIVE = """You are a research assistant who answers questions.
Use <think></think> tags to show your reasoning if needed.
@@ -469,6 +474,7 @@ class LLMClient:
max_context_tokens: int | None = None,
enable_thinking: bool | None = None,
force_openai_compat: bool = False,
+ use_litellm: bool = False,
):
self.model = model
self.temperature = temperature
@@ -476,13 +482,30 @@ class LLMClient:
self.timeout = timeout
self.max_context_tokens = max_context_tokens
self.enable_thinking = enable_thinking
+ # Kept for the LiteLLM path (module-level SDK, no persistent client object).
+ self.api_key = api_key
+ self.api_base = api_base
print(f"context length model: {max_context_tokens}")
+ # LiteLLM routes every provider through one SDK call using a
+ # provider-prefixed model string (e.g. "anthropic/claude-3-5-sonnet",
+ # "gemini/gemini-1.5-pro", "bedrock/..."), so it takes precedence over
+ # the Gemini / OpenAI-compatible branches below.
+ self.use_litellm = use_litellm
+
# Gemini routes to Google GenAI SDK unless forced to OpenAI-compatible
# (aggregators like OpenRouter / Commonstack expose Gemini via OAI-compat).
- self.is_gemini = ("gemini" in model.lower()) and not force_openai_compat
+ self.is_gemini = (
+ (not use_litellm)
+ and ("gemini" in model.lower())
+ and not force_openai_compat
+ )
- if self.is_gemini:
+ if self.use_litellm:
+ # No persistent client: litellm.acompletion is a module-level call.
+ self.client = None
+ self.gemini_client = None
+ elif self.is_gemini:
if not GEMINI_AVAILABLE:
raise ImportError(
"google-genai package is required for Gemini models. Install with: pip install google-genai"
@@ -552,7 +575,9 @@ class LLMClient:
timeout_attempts = 0
while True:
try:
- if self.is_gemini:
+ if self.use_litellm:
+ return await self._generate_litellm(messages)
+ elif self.is_gemini:
return await self._generate_gemini(messages)
else:
return await self._generate_openai(messages)
@@ -713,9 +738,7 @@ class LLMClient:
# Some modern reasoning models deprecate `temperature` (Claude Opus 4.7+, some GPT-5 variants).
# Only send it when we actually want to override the default.
model_lower = self.model.lower()
- drops_temperature = any(
- x in model_lower for x in ("opus-4-7", "opus-4-8", "gpt-5.4-pro")
- )
+ drops_temperature = any(x in model_lower for x in MODELS_DROPPING_TEMPERATURE)
if not drops_temperature:
kwargs["temperature"] = self.temperature
if self.enable_thinking is not None:
@@ -736,6 +759,56 @@ class LLMClient:
return generated_text, usage
+ async def _generate_litellm(self, messages: list[dict]) -> tuple[str, dict]:
+ """Generate via the LiteLLM SDK (one call for 100+ providers).
+
+ The model string is provider-prefixed (e.g. "anthropic/claude-3-5-sonnet",
+ "gemini/gemini-1.5-pro", "bedrock/..."). Vision content blocks in
+ ``messages`` are passed through unchanged — LiteLLM maps them to each
+ provider's native format. ``drop_params=True`` drops any param a given
+ provider does not support, so one config works everywhere.
+ """
+ import litellm
+
+ kwargs: dict = dict(
+ model=self.model,
+ messages=messages,
+ max_tokens=self.max_tokens,
+ timeout=self.timeout,
+ drop_params=True,
+ )
+ # Credentials are optional: when unset, LiteLLM falls back to each
+ # provider's own env var (OPENAI_API_KEY, ANTHROPIC_API_KEY, AWS creds,
+ # ...) or a LiteLLM proxy. Omit blanks rather than sending empty strings.
+ if self.api_key and self.api_key != "dummy":
+ kwargs["api_key"] = self.api_key
+ if self.api_base:
+ kwargs["api_base"] = self.api_base
+
+ # Some reasoning models reject `temperature`; mirror the OpenAI path.
+ model_lower = self.model.lower()
+ drops_temperature = any(x in model_lower for x in MODELS_DROPPING_TEMPERATURE)
+ if not drops_temperature:
+ kwargs["temperature"] = self.temperature
+ if self.enable_thinking is not None:
+ kwargs["extra_body"] = {
+ "chat_template_kwargs": {"enable_thinking": self.enable_thinking}
+ }
+
+ response = await litellm.acompletion(**kwargs)
+
+ generated_text = response.choices[0].message.content or ""
+
+ usage = {}
+ if getattr(response, "usage", None):
+ usage = {
+ "prompt_tokens": response.usage.prompt_tokens,
+ "completion_tokens": response.usage.completion_tokens,
+ "total_tokens": response.usage.total_tokens,
+ }
+
+ return generated_text, usage
+
def _estimate_tokens(self, messages: list[dict]) -> int:
"""Estimate token count from messages (rough: ~4 chars per token)."""
total_chars = 0
diff --git a/eval/pyproject.toml b/eval/pyproject.toml
index 991c066..39fbf88 100644
--- a/eval/pyproject.toml
+++ b/eval/pyproject.toml
@@ -22,7 +22,7 @@ dependencies = [
"lxml>=6.1.1",
"tiktoken>=0.13.0",
"trafilatura>=2.0.0",
- "litellm>=1.86.2",
+ "litellm>=1.89.0,<2.0.0",
"botocore>=1.43.18",
"tenacity>=9.1.4",
"fastmcp>=3.3.1",
diff --git a/eval/run_bench.py b/eval/run_bench.py
index dabf03c..e7ca40f 100644
--- a/eval/run_bench.py
+++ b/eval/run_bench.py
@@ -977,6 +977,14 @@ async def run_async(args):
)
logger.info(f"Using Commonstack API with model: {args.model}")
model = args.model
+ elif args.litellm:
+ # LiteLLM SDK: the provider-prefixed --model selects the backend, and
+ # credentials come from each provider's own env var (or --api-base for a
+ # LiteLLM proxy). Keep both optional so blank values fall through to env.
+ model = args.model
+ api_base = args.api_base or ""
+ api_key = args.api_key or "dummy"
+ logger.info(f"Using LiteLLM SDK with model: {args.model}")
else:
# Override with command-line args if provided
# For Gemini, api_base from config is None, so use command-line arg or default
@@ -1107,6 +1115,7 @@ async def run_async(args):
timeout=args.timeout,
enable_thinking=(False if args.no_think else None),
force_openai_compat=(args.open_router or args.commonstack),
+ use_litellm=args.litellm,
)
# 3b. Create pixel-compressed encoder for generation if requested
@@ -1323,6 +1332,13 @@ def main():
action="store_true",
help="Use Commonstack API (https://api.commonstack.ai). Requires --api-key or COMMONSTACK_API_KEY env var.",
)
+ parser.add_argument(
+ "--litellm",
+ action="store_true",
+ help="Route generation through the LiteLLM SDK (100+ providers via one "
+ "provider-prefixed --model, e.g. anthropic/claude-3-5-sonnet). Optional "
+ "--api-base points at a LiteLLM proxy; keys fall back to each provider's env var.",
+ )
# General args
parser.add_argument(
diff --git a/eval/run_livevqa.py b/eval/run_livevqa.py
index 1403209..671dc24 100644
--- a/eval/run_livevqa.py
+++ b/eval/run_livevqa.py
@@ -822,6 +822,7 @@ async def run_evaluation(args: argparse.Namespace):
max_tokens=args.max_tokens,
timeout=args.reader_timeout,
enable_thinking=False if args.no_think else None,
+ use_litellm=args.litellm,
)
# Smoke test
@@ -1090,6 +1091,12 @@ def main():
parser.add_argument(
"--model", default="Qwen/Qwen3-VL-4B-Instruct", help="Reader model name"
)
+ parser.add_argument(
+ "--litellm",
+ action="store_true",
+ help="Route the reader through the LiteLLM SDK (100+ providers via a "
+ "provider-prefixed --model, e.g. anthropic/claude-3-5-sonnet).",
+ )
parser.add_argument(
"--max-tokens",
type=int,
diff --git a/eval/uv.lock b/eval/uv.lock
index 80cf7f8..7c88f7c 100644
--- a/eval/uv.lock
+++ b/eval/uv.lock
@@ -30,7 +30,7 @@ wheels = [
[[package]]
name = "aiohttp"
-version = "3.13.5"
+version = "3.14.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "aiohappyeyeballs" },
@@ -39,27 +39,29 @@ dependencies = [
{ name = "frozenlist" },
{ name = "multidict" },
{ name = "propcache" },
+ { name = "typing-extensions" },
{ name = "yarl" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/77/9a/152096d4808df8e4268befa55fba462f440f14beab85e8ad9bf990516918/aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1", size = 7858271, upload-time = "2026-03-31T22:01:03.343Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/be/6f/353954c29e7dcce7cf00280a02c75f30e133c00793c7a2ed3776d7b2f426/aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9", size = 748876, upload-time = "2026-03-31T21:57:36.319Z" },
- { url = "https://files.pythonhosted.org/packages/f5/1b/428a7c64687b3b2e9cd293186695affc0e1e54a445d0361743b231f11066/aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416", size = 499557, upload-time = "2026-03-31T21:57:38.236Z" },
- { url = "https://files.pythonhosted.org/packages/29/47/7be41556bfbb6917069d6a6634bb7dd5e163ba445b783a90d40f5ac7e3a7/aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2", size = 500258, upload-time = "2026-03-31T21:57:39.923Z" },
- { url = "https://files.pythonhosted.org/packages/67/84/c9ecc5828cb0b3695856c07c0a6817a99d51e2473400f705275a2b3d9239/aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4", size = 1749199, upload-time = "2026-03-31T21:57:41.938Z" },
- { url = "https://files.pythonhosted.org/packages/f0/d3/3c6d610e66b495657622edb6ae7c7fd31b2e9086b4ec50b47897ad6042a9/aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9", size = 1721013, upload-time = "2026-03-31T21:57:43.904Z" },
- { url = "https://files.pythonhosted.org/packages/49/a0/24409c12217456df0bae7babe3b014e460b0b38a8e60753d6cb339f6556d/aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5", size = 1781501, upload-time = "2026-03-31T21:57:46.285Z" },
- { url = "https://files.pythonhosted.org/packages/98/9d/b65ec649adc5bccc008b0957a9a9c691070aeac4e41cea18559fef49958b/aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e", size = 1878981, upload-time = "2026-03-31T21:57:48.734Z" },
- { url = "https://files.pythonhosted.org/packages/57/d8/8d44036d7eb7b6a8ec4c5494ea0c8c8b94fbc0ed3991c1a7adf230df03bf/aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1", size = 1767934, upload-time = "2026-03-31T21:57:51.171Z" },
- { url = "https://files.pythonhosted.org/packages/31/04/d3f8211f273356f158e3464e9e45484d3fb8c4ce5eb2f6fe9405c3273983/aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286", size = 1566671, upload-time = "2026-03-31T21:57:53.326Z" },
- { url = "https://files.pythonhosted.org/packages/41/db/073e4ebe00b78e2dfcacff734291651729a62953b48933d765dc513bf798/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9", size = 1705219, upload-time = "2026-03-31T21:57:55.385Z" },
- { url = "https://files.pythonhosted.org/packages/48/45/7dfba71a2f9fd97b15c95c06819de7eb38113d2cdb6319669195a7d64270/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88", size = 1743049, upload-time = "2026-03-31T21:57:57.341Z" },
- { url = "https://files.pythonhosted.org/packages/18/71/901db0061e0f717d226386a7f471bb59b19566f2cae5f0d93874b017271f/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3", size = 1749557, upload-time = "2026-03-31T21:57:59.626Z" },
- { url = "https://files.pythonhosted.org/packages/08/d5/41eebd16066e59cd43728fe74bce953d7402f2b4ddfdfef2c0e9f17ca274/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b", size = 1558931, upload-time = "2026-03-31T21:58:01.972Z" },
- { url = "https://files.pythonhosted.org/packages/30/e6/4a799798bf05740e66c3a1161079bda7a3dd8e22ca392481d7a7f9af82a6/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe", size = 1774125, upload-time = "2026-03-31T21:58:04.007Z" },
- { url = "https://files.pythonhosted.org/packages/84/63/7749337c90f92bc2cb18f9560d67aa6258c7060d1397d21529b8004fcf6f/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14", size = 1732427, upload-time = "2026-03-31T21:58:06.337Z" },
- { url = "https://files.pythonhosted.org/packages/98/de/cf2f44ff98d307e72fb97d5f5bbae3bfcb442f0ea9790c0bf5c5c2331404/aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3", size = 433534, upload-time = "2026-03-31T21:58:08.712Z" },
- { url = "https://files.pythonhosted.org/packages/aa/ca/eadf6f9c8fa5e31d40993e3db153fb5ed0b11008ad5d9de98a95045bed84/aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1", size = 460446, upload-time = "2026-03-31T21:58:10.945Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/58/d9/22ce5786ac0c1653ae8b6c23bded02c1686d11f0dbb45b31ce128e0df985/aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc", size = 7971213, upload-time = "2026-07-23T01:57:27.037Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/18/d4/eb96299230e20acf2efae207cb8d69051f1f68e357e5ea5e479bf6fb097a/aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5", size = 754690, upload-time = "2026-07-23T01:53:47.332Z" },
+ { url = "https://files.pythonhosted.org/packages/88/11/e7a70a209eb9a067c0d3212b518a0134e3484f5178c7533878b6b514d469/aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228", size = 509484, upload-time = "2026-07-23T01:53:51.159Z" },
+ { url = "https://files.pythonhosted.org/packages/30/07/4bbc222cc8dbe31d4c3e8a5baad2286e4d42026ac0c570027b89afce6344/aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee", size = 511949, upload-time = "2026-07-23T01:53:55.083Z" },
+ { url = "https://files.pythonhosted.org/packages/54/b9/42e74c46b7b7c794b995bbc1f573fb48950c38b19d8600c62a6804ee2d67/aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a", size = 1765282, upload-time = "2026-07-23T01:53:59.662Z" },
+ { url = "https://files.pythonhosted.org/packages/6b/ed/62bc4d74363ad346d518e0720363a949f63e2e23439a79eb5813d4d29bb3/aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b", size = 1741511, upload-time = "2026-07-23T01:54:04.063Z" },
+ { url = "https://files.pythonhosted.org/packages/d0/9f/181e8a8bc79e47d13c7fc4540bd7a3b729d9505609c61f392a8dd2fbfe55/aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529", size = 1810680, upload-time = "2026-07-23T01:54:09.882Z" },
+ { url = "https://files.pythonhosted.org/packages/5c/9a/dec94d6ad694552fe3424e3f1928d7a606a5d9d9433a04e7ecdd9d38ae7f/aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787", size = 1905646, upload-time = "2026-07-23T01:54:13.475Z" },
+ { url = "https://files.pythonhosted.org/packages/52/b7/7cd31f29d6055bd711ae6e669367fba6f5ae9de463910a793e30556a8db7/aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42", size = 1792122, upload-time = "2026-07-23T01:54:15.752Z" },
+ { url = "https://files.pythonhosted.org/packages/66/73/10b1ef93afa61f4963c746257b70ced619cf31a4798671de5fdb2608501d/aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b", size = 1591127, upload-time = "2026-07-23T01:54:19.489Z" },
+ { url = "https://files.pythonhosted.org/packages/49/ed/3b203fa6de1b338c14acdc06bf6ca9b043b7944f005966958c2ced932cde/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043", size = 1725210, upload-time = "2026-07-23T01:54:24.129Z" },
+ { url = "https://files.pythonhosted.org/packages/28/b7/1c2aab8c706436dcc28598452488ac9cd7c409da815237c28c27d58993e6/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427", size = 1764848, upload-time = "2026-07-23T01:54:27.973Z" },
+ { url = "https://files.pythonhosted.org/packages/54/50/94c28f08b131c4bf10984ea2c7a536c9920608bb2d6e7f95642c30cc87b7/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d", size = 1777102, upload-time = "2026-07-23T01:54:31.775Z" },
+ { url = "https://files.pythonhosted.org/packages/13/d4/e7d09ba7d345fb2d74440fd2fa033c5e079fac05552927705986f41a364f/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0", size = 1580205, upload-time = "2026-07-23T01:54:34.518Z" },
+ { url = "https://files.pythonhosted.org/packages/a3/84/072a91d68e1e1eb587985b54baab94221277f877e8ef274fc213a0ceae28/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d", size = 1797219, upload-time = "2026-07-23T01:54:36.995Z" },
+ { url = "https://files.pythonhosted.org/packages/e0/eb/aad34e897e668424d6e995da5dff8a4a09af93363d3392488772957a63aa/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19", size = 1768629, upload-time = "2026-07-23T01:54:40.103Z" },
+ { url = "https://files.pythonhosted.org/packages/b6/2b/6bb88ddba0fecd9122aa3ebcad25996cf6c083a4a7040dbb3a4f97972af6/aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559", size = 451481, upload-time = "2026-07-23T01:54:42.547Z" },
+ { url = "https://files.pythonhosted.org/packages/76/9b/f2f8f108da17ecef2cc3efc424e8b7ad3782b1a8360f7b8eae8ced84f6ea/aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a", size = 476845, upload-time = "2026-07-23T01:54:44.853Z" },
+ { url = "https://files.pythonhosted.org/packages/3e/44/28dac80a8941b604f4da10ce21097614ca1bf905ce93dca28d8d7de9c1e7/aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c", size = 448050, upload-time = "2026-07-23T01:54:47.087Z" },
]
[[package]]
@@ -231,18 +233,32 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/12/91/7db93e4689f0f145bcb954dc62936e5f5090548a9fa20c6bbebfaeaa648a/blake3-1.0.9-cp312-cp312-win_amd64.whl", hash = "sha256:15566065ff90ab3da46ec0be1417406f00507af902b6fb0fbc6563e77f02fc42", size = 218220, upload-time = "2026-06-22T18:00:50.659Z" },
]
+[[package]]
+name = "boto3"
+version = "1.43.78"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "botocore" },
+ { name = "jmespath" },
+ { name = "s3transfer" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/be/55/e026c943f7f1ed6d2f5e6035713f21233bbe9ee975008662dc64ca0d4ced/boto3-1.43.78.tar.gz", hash = "sha256:2fa59116e298171ef59e7600a8be6c01177faef8af4b9a4314b7a57a04009ada", size = 112679, upload-time = "2026-08-21T19:37:36.84Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/e3/55/288b1f0d987e15db0417f5ea43da564bfa042063f508c259eec893134628/boto3-1.43.78-py3-none-any.whl", hash = "sha256:893f06a171469618e17de78dc927aca6e74fcf45a70d2c5918e9ac9919e96cc9", size = 140028, upload-time = "2026-08-21T19:37:35.358Z" },
+]
+
[[package]]
name = "botocore"
-version = "1.43.18"
+version = "1.43.78"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "jmespath" },
{ name = "python-dateutil" },
{ name = "urllib3" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/01/6d/436d69ec484ffc43635b38d7fb7d717d38824671f10e12e77924019ca929/botocore-1.43.18.tar.gz", hash = "sha256:dc8c105351b49688c667065cd5a45fc5b9db982657cefc9e3fbfb9417a55c7df", size = 15424886, upload-time = "2026-05-29T19:33:16.251Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/67/71/490aaa384855bf3b69405ded52ae77a0e5f4eeb2165044fa65466c4d3a73/botocore-1.43.78.tar.gz", hash = "sha256:e8238d22c1e1342025d75d2e33d154a375e7caad0fc67f77d77faa2d82668b94", size = 15982895, upload-time = "2026-08-21T19:37:32.402Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/21/5a/35c92c0af1514581031fe66c398b622176b3c928a6d5cf8133c7207e3bd7/botocore-1.43.18-py3-none-any.whl", hash = "sha256:e2610fce16df9f89deab5f3c163430a814e6804034eb95bef8957c8db60b7dbc", size = 15106258, upload-time = "2026-05-29T19:33:11.18Z" },
+ { url = "https://files.pythonhosted.org/packages/c5/5b/b7c5c22767c0e8eb4bb2bd0a972d77bd0be3dd9908e15d4487094a31fa4b/botocore-1.43.78-py3-none-any.whl", hash = "sha256:ddd020493235e264b3bd12606f239a3d3b2dd7cfb1d25a0691061183c290c228", size = 15677214, upload-time = "2026-08-21T19:37:28.894Z" },
]
[[package]]
@@ -1346,10 +1362,11 @@ wheels = [
[[package]]
name = "litellm"
-version = "1.86.2"
+version = "1.98.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "aiohttp" },
+ { name = "boto3" },
{ name = "click" },
{ name = "fastuuid" },
{ name = "httpx" },
@@ -1358,13 +1375,20 @@ dependencies = [
{ name = "jsonschema" },
{ name = "openai" },
{ name = "pydantic" },
+ { name = "pydantic-settings" },
{ name = "python-dotenv" },
{ name = "tiktoken" },
{ name = "tokenizers" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/01/ee/1f6f3f5d01e0910629b6af3757e82c47b8b87dca497a661a9c63280b4bd8/litellm-1.86.2.tar.gz", hash = "sha256:7d559ad48b97d796ff325af88fd7eebbdc66e58773fb5312130ab1cac968f8f3", size = 15380548, upload-time = "2026-05-27T16:19:58.45Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/9c/97/c9da198af273d700bf44d7d82eb21c5b8078c82574b31856b71b1298234b/litellm-1.98.0.tar.gz", hash = "sha256:0e6ba5d645a73ca6d0ffb4e8ec539d94b6e8fad691f2a54c6819011e6d0de8bf", size = 17577139, upload-time = "2026-08-22T22:19:21.931Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/4d/37/4da1dd67157aaa11477d6c63e4725216bfc46b4661e581b490d17e5b2831/litellm-1.86.2-py3-none-any.whl", hash = "sha256:27096463be7add661513ada3d9039e8f1a6195859e604d30fde96c939efe0a03", size = 17013061, upload-time = "2026-05-27T16:19:53.219Z" },
+ { url = "https://files.pythonhosted.org/packages/c8/90/2ef5e33b0a67b309be124a77e5098a261beffe28439221dbbc28e5f02e2e/litellm-1.98.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9fda3497f1ec4686c943ce2aeab5767f8fb4a5989305d4b28d6d5d7e488b850c", size = 24026097, upload-time = "2026-08-22T22:19:03.567Z" },
+ { url = "https://files.pythonhosted.org/packages/b0/56/b4569b4ef3640732d5770e0d3f46a395fd20f35594db1f65629595daed00/litellm-1.98.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:b89b6a0fc179d881191579f5309e622173dca802ee991b92e382acb918eea437", size = 23683944, upload-time = "2026-08-22T22:19:06.952Z" },
+ { url = "https://files.pythonhosted.org/packages/d9/a7/9f03de0e8d767ff27964ea99bbf07e4c37a12f9d3c168c09f40e068535d4/litellm-1.98.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3a95260f087a4cf763da85bbeeb2efa82caec243ad2394c9e6362ff747963738", size = 23824066, upload-time = "2026-08-22T22:19:09.714Z" },
+ { url = "https://files.pythonhosted.org/packages/69/de/ab46b521e2a6e6a94a5cb91ba3debd6c4e922feaeb47158a389400b0a0fe/litellm-1.98.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:150993180bf049feafa3e20cf46ca0978c69cc66e2ef1639a47e852c682a4721", size = 24190393, upload-time = "2026-08-22T22:19:12.156Z" },
+ { url = "https://files.pythonhosted.org/packages/c5/0a/d2f549d906b9b267b38b406dde721eb93db21b46ec907ae0a7a2a89a50f6/litellm-1.98.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:54d0bc2aba84644de5e84a265f24e5d436d91592ca5b7cc61cee478db297b0c3", size = 23901211, upload-time = "2026-08-22T22:19:14.708Z" },
+ { url = "https://files.pythonhosted.org/packages/85/08/1bd1653297d9c92eaf04425f8a21da817fcde12e1d9469394c543df19d72/litellm-1.98.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5e546d4af197c257d320299af11f4619f8e5a29f9bb7ce2d9974dd4b1e045499", size = 24290140, upload-time = "2026-08-22T22:19:17.145Z" },
+ { url = "https://files.pythonhosted.org/packages/de/91/14d11ad7e290137400e5b30bcf23de86f811085f4a46756f60684ed0f064/litellm-1.98.0-cp310-abi3-win_amd64.whl", hash = "sha256:1daac9a9a9d052fdbe58ee711c9924dc81d349d4621286cbd96d77baa12158c4", size = 24079638, upload-time = "2026-08-22T22:19:19.558Z" },
]
[[package]]
@@ -2239,7 +2263,7 @@ requires-dist = [
{ name = "botocore", specifier = ">=1.43.18" },
{ name = "datasets", specifier = ">=4.8.5" },
{ name = "fastmcp", specifier = ">=3.3.1" },
- { name = "litellm", specifier = ">=1.86.2" },
+ { name = "litellm", specifier = ">=1.89.0,<2.0.0" },
{ name = "lxml", specifier = ">=6.1.1" },
{ name = "numpy", specifier = ">=1.26.0,<2.3" },
{ name = "omegaconf", specifier = ">=2.3.0" },
@@ -2850,6 +2874,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/cb/53/6c3419d85eb2ec5938a37627c585b42d76a63bb731d6e42ed4b079ebf486/rpds_py-2026.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:1841d067089e117142d79b98aa0df2f08b52f2ecc1819dd2700636c0db74a473", size = 223967, upload-time = "2026-05-28T11:59:32.318Z" },
]
+[[package]]
+name = "s3transfer"
+version = "0.19.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "botocore" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/76/43/35e4d8aa320bffe8287fe8f65f578fa2d2db0a64212f0e710dce58267854/s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993", size = 165592, upload-time = "2026-07-22T19:30:44.432Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/bc/e7/5c595c75e9f41a44f30e526eda465ea0b4eec93470e074e4a111b253f13a/s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:d8168eccca828cbb2cd573675333f3bddd254313a9c42494b84c76b539e8ba25", size = 90216, upload-time = "2026-07-22T19:30:43.251Z" },
+]
+
[[package]]
name = "safetensors"
version = "0.8.0"
diff --git a/tests/test_litellm_client.py b/tests/test_litellm_client.py
new file mode 100644
index 0000000..a366393
--- /dev/null
+++ b/tests/test_litellm_client.py
@@ -0,0 +1,140 @@
+"""Unit tests for the LiteLLM path in eval/lib/llm.py.
+
+Lives here rather than under ``eval/`` because CI runs ``pytest tests/``: a
+test outside this directory is never collected, so it gates nothing.
+
+``eval/lib/__init__.py`` eagerly imports the retrieval and data modules, which
+pull trafilatura, pandas and friends -- none of which CI installs, since it
+syncs the root project's dev extra. So bind a stub ``lib`` package to the eval
+directory and import ``lib.llm`` through it: that resolves the module's
+``from .retrieval import ...`` without ever executing the package __init__.
+``llm.py`` and ``retrieval.py`` are stdlib-only at module scope, so this needs
+nothing installed beyond pytest.
+
+litellm itself is stubbed in sys.modules -- the SDK is imported inside
+``_generate_litellm``, so these run with no network, no keys, and no litellm.
+"""
+
+import asyncio
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+_LIB = Path(__file__).parents[1] / "eval" / "lib"
+if "lib" not in sys.modules:
+ _pkg = types.ModuleType("lib")
+ _pkg.__path__ = [str(_LIB)]
+ sys.modules["lib"] = _pkg
+
+from lib.llm import LLMClient
+
+
[email protected](autouse=True)
+def _clear_litellm_stub():
+ """Keep the fake out of sys.modules for anything that runs after these."""
+ yield
+ sys.modules.pop("litellm", None)
+
+
+def _stub_litellm(content="4"):
+ fake = types.ModuleType("litellm")
+
+ async def acompletion(**kwargs):
+ acompletion.calls.append(kwargs)
+ message = types.SimpleNamespace(content=content)
+ choice = types.SimpleNamespace(message=message)
+ usage = types.SimpleNamespace(
+ prompt_tokens=3, completion_tokens=1, total_tokens=4
+ )
+ return types.SimpleNamespace(choices=[choice], usage=usage)
+
+ acompletion.calls = []
+ fake.acompletion = acompletion
+ sys.modules["litellm"] = fake
+ return fake
+
+
+def test_litellm_dispatch_forwards_model_creds_and_drop_params():
+ fake = _stub_litellm("4")
+
+ client = LLMClient(
+ model="anthropic/claude-3-5-sonnet",
+ api_key="sk-x",
+ api_base="",
+ use_litellm=True,
+ max_tokens=64,
+ )
+ text, usage = asyncio.run(client.generate([{"role": "user", "content": "2+2?"}]))
+
+ assert text == "4"
+ assert usage["total_tokens"] == 4
+ call = fake.acompletion.calls[0]
+ assert call["model"] == "anthropic/claude-3-5-sonnet"
+ assert call["drop_params"] is True
+ assert call["api_key"] == "sk-x"
+ assert "api_base" not in call # blank base omitted
+ assert call["temperature"] == 0.0
+
+
+def test_credentials_omitted_when_dummy_or_blank():
+ fake = _stub_litellm("ok")
+
+ client = LLMClient(
+ model="gemini/gemini-1.5-pro",
+ api_key="dummy",
+ api_base="",
+ use_litellm=True,
+ )
+ asyncio.run(client.generate([{"role": "user", "content": "hi"}]))
+
+ call = fake.acompletion.calls[0]
+ assert "api_key" not in call # "dummy" treated as unset -> provider env fallback
+ assert "api_base" not in call
+
+
+def test_api_base_forwarded_for_proxy():
+ fake = _stub_litellm("ok")
+
+ client = LLMClient(
+ model="openai/gpt-4o-mini",
+ api_key="sk-proxy",
+ api_base="http://localhost:4000",
+ use_litellm=True,
+ )
+ asyncio.run(client.generate([{"role": "user", "content": "hi"}]))
+
+ assert fake.acompletion.calls[0]["api_base"] == "http://localhost:4000"
+
+
+def test_use_litellm_takes_precedence_over_gemini_routing():
+ _stub_litellm("x")
+
+ client = LLMClient(model="gemini/gemini-1.5-pro", use_litellm=True)
+
+ # LiteLLM handles gemini itself, so the native GenAI branch must be disabled.
+ assert client.is_gemini is False
+ assert client.gemini_client is None
+
+
+ ("model", "sends_temperature"),
+ [
+ ("anthropic/claude-opus-4-7", False),
+ ("openai/gpt-5.4-pro", False),
+ ("anthropic/claude-3-5-sonnet", True),
+ ],
+)
+def test_reasoning_models_drop_temperature(model, sends_temperature):
+ """The rule lives in one constant now; both backends must apply it.
+
+ It was duplicated as a literal in the OpenAI and LiteLLM paths, so a newly
+ released model could be taught to one and not the other.
+ """
+ fake = _stub_litellm("ok")
+
+ client = LLMClient(model=model, use_litellm=True)
+ asyncio.run(client.generate([{"role": "user", "content": "hi"}]))
+
+ assert ("temperature" in fake.acompletion.calls[0]) is sends_temperature