Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(anyllm): forward openai api_base/api_key to the any-llm backend (#942) (#954)

Aniruddh Jhaa7ee8a60a7ac
diff --git a/headroom/backends/anyllm.py b/headroom/backends/anyllm.py
index 95d355c62..e5d3d8ee7 100644
--- a/headroom/backends/anyllm.py
+++ b/headroom/backends/anyllm.py
@@ -41,13 +41,25 @@ class AnyLLMBackend(Backend):
)
self.provider = provider.lower()
- self.api_key = api_key
- self.api_base = api_base
-
- # Create the AnyLLM instance once and reuse
- self.llm = AnyLLM.create(self.provider)
-
- logger.info(f"any-llm backend initialized (provider={provider})")
+ # Normalize empty-string overrides (e.g. an env var set to "") to None
+ # so provider defaults stay active instead of forwarding a blank value.
+ self.api_key = api_key or None
+ self.api_base = api_base or None
+
+ # Create the AnyLLM instance once and reuse. api_key/api_base are only
+ # forwarded when set so providers keep their own env-var defaults
+ # (e.g. OPENAI_API_KEY / OPENAI_BASE_URL) otherwise.
+ create_kwargs: dict[str, Any] = {}
+ if self.api_key is not None:
+ create_kwargs["api_key"] = self.api_key
+ if self.api_base is not None:
+ create_kwargs["api_base"] = self.api_base
+ self.llm = AnyLLM.create(self.provider, **create_kwargs)
+
+ logger.info(
+ f"any-llm backend initialized (provider={provider}, "
+ f"api_base={self.api_base or 'default'})"
+ )
@property
def name(self) -> str:
diff --git a/headroom/providers/registry.py b/headroom/providers/registry.py
index 2300739db..5ecf73b13 100644
--- a/headroom/providers/registry.py
+++ b/headroom/providers/registry.py
@@ -149,6 +149,7 @@ def create_proxy_backend(
anyllm_provider: str,
bedrock_region: str | None,
logger: logging.Logger,
+ openai_api_url: str | None = None,
anyllm_backend_cls: Any | None = None,
litellm_backend_cls: Any | None = None,
) -> Backend | None:
@@ -160,7 +161,7 @@ def create_proxy_backend(
provider = anyllm_provider
try:
backend_cls = anyllm_backend_cls or _load_anyllm_backend()
- instance = cast("Backend", backend_cls(provider=provider))
+ instance = cast("Backend", backend_cls(provider=provider, api_base=openai_api_url))
logger.info("any-llm backend enabled (provider=%s)", provider)
return instance
except ImportError as exc:
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index a12244648..728ab9a7a 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -793,6 +793,7 @@ class HeadroomProxy(
anyllm_provider=config.anyllm_provider,
bedrock_region=config.bedrock_region,
logger=logger,
+ openai_api_url=config.openai_api_url,
anyllm_backend_cls=AnyLLMBackend,
litellm_backend_cls=LiteLLMBackend,
)
diff --git a/tests/test_backend_anyllm.py b/tests/test_backend_anyllm.py
index 584702d41..ec152f0d4 100644
--- a/tests/test_backend_anyllm.py
+++ b/tests/test_backend_anyllm.py
@@ -43,7 +43,7 @@ def make_backend(
class FakeAnyLLM:
@staticmethod
- def create(requested_provider: str):
+ def create(requested_provider: str, **kwargs): # noqa: ANN003
assert requested_provider == provider
return fake_instance
@@ -52,6 +52,76 @@ def make_backend(
return anyllm.AnyLLMBackend(provider=provider.upper()), fake_instance
+def test_init_forwards_api_base_and_api_key(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Regression for #942: custom api_base/api_key must reach AnyLLM.create."""
+ fake_instance = FakeAnyLLMInstance()
+ create_calls: list[dict[str, object]] = []
+
+ class FakeAnyLLM:
+ @staticmethod
+ def create(requested_provider: str, **kwargs): # noqa: ANN003
+ create_calls.append({"provider": requested_provider, **kwargs})
+ return fake_instance
+
+ monkeypatch.setattr(anyllm, "ANYLLM_AVAILABLE", True)
+ monkeypatch.setattr(anyllm, "AnyLLM", FakeAnyLLM)
+
+ backend = anyllm.AnyLLMBackend(
+ provider="openai",
+ api_key="sk-custom",
+ api_base="https://custom-provider.example/v1",
+ )
+
+ assert backend.api_base == "https://custom-provider.example/v1"
+ assert create_calls == [
+ {
+ "provider": "openai",
+ "api_key": "sk-custom",
+ "api_base": "https://custom-provider.example/v1",
+ }
+ ]
+
+
+def test_init_omits_unset_api_base_and_api_key(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Unset overrides must not be forwarded, preserving provider env defaults."""
+ fake_instance = FakeAnyLLMInstance()
+ create_calls: list[dict[str, object]] = []
+
+ class FakeAnyLLM:
+ @staticmethod
+ def create(requested_provider: str, **kwargs): # noqa: ANN003
+ create_calls.append({"provider": requested_provider, **kwargs})
+ return fake_instance
+
+ monkeypatch.setattr(anyllm, "ANYLLM_AVAILABLE", True)
+ monkeypatch.setattr(anyllm, "AnyLLM", FakeAnyLLM)
+
+ anyllm.AnyLLMBackend(provider="openai")
+
+ assert create_calls == [{"provider": "openai"}]
+
+
+def test_init_treats_empty_overrides_as_unset(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Empty-string api_base/api_key must not be forwarded (env var set to "")."""
+ fake_instance = FakeAnyLLMInstance()
+ create_calls: list[dict[str, object]] = []
+
+ class FakeAnyLLM:
+ @staticmethod
+ def create(requested_provider: str, **kwargs): # noqa: ANN003
+ create_calls.append({"provider": requested_provider, **kwargs})
+ return fake_instance
+
+ monkeypatch.setattr(anyllm, "ANYLLM_AVAILABLE", True)
+ monkeypatch.setattr(anyllm, "AnyLLM", FakeAnyLLM)
+
+ backend = anyllm.AnyLLMBackend(provider="openai", api_key="", api_base="")
+
+ assert backend.api_base is None
+ assert backend.api_key is None
+ assert create_calls == [{"provider": "openai"}]
+
+
def make_choice(
content: str = "hello", finish_reason: str = "stop", tool_calls=None, index: int = 0
):
diff --git a/tests/test_provider_registry_extended.py b/tests/test_provider_registry_extended.py
index 97f50ab28..9c8050adf 100644
--- a/tests/test_provider_registry_extended.py
+++ b/tests/test_provider_registry_extended.py
@@ -102,7 +102,11 @@ def test_create_proxy_backend_uses_injected_backend_types() -> None:
anyllm_provider="groq",
bedrock_region=None,
logger=logger,
- anyllm_backend_cls=lambda provider: {"kind": "anyllm", "provider": provider},
+ anyllm_backend_cls=lambda provider, api_base: {
+ "kind": "anyllm",
+ "provider": provider,
+ "api_base": api_base,
+ },
)
litellm = create_proxy_backend(
backend="bedrock",
@@ -116,10 +120,32 @@ def test_create_proxy_backend_uses_injected_backend_types() -> None:
},
)
- assert anyllm == {"kind": "anyllm", "provider": "groq"}
+ assert anyllm == {"kind": "anyllm", "provider": "groq", "api_base": None}
assert litellm == {"kind": "litellm", "provider": "bedrock", "region": "us-east-1"}
+def test_create_proxy_backend_passes_openai_api_url_to_anyllm() -> None:
+ """Regression for #942: --openai-api-url must reach the any-llm backend."""
+ logger = logging.getLogger("test")
+
+ anyllm = create_proxy_backend(
+ backend="anyllm",
+ anyllm_provider="openai",
+ bedrock_region=None,
+ logger=logger,
+ openai_api_url="https://custom-provider.example/v1",
+ anyllm_backend_cls=lambda provider, api_base: {
+ "provider": provider,
+ "api_base": api_base,
+ },
+ )
+
+ assert anyllm == {
+ "provider": "openai",
+ "api_base": "https://custom-provider.example/v1",
+ }
+
+
def test_create_proxy_backend_handles_missing_or_direct_backends(
caplog: pytest.LogCaptureFixture,
) -> None:
@@ -138,7 +164,9 @@ def test_create_proxy_backend_handles_missing_or_direct_backends(
anyllm_provider="groq",
bedrock_region=None,
logger=logger,
- anyllm_backend_cls=lambda provider: (_ for _ in ()).throw(ImportError("missing")),
+ anyllm_backend_cls=lambda provider, api_base: (_ for _ in ()).throw(
+ ImportError("missing")
+ ),
)
assert direct is None