Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

chore(providers): log the LiteLLM probe fallbacks at debug (#4086)

Tejas Chopradb72e6717f24
diff --git a/headroom/pricing/counterfactual.py b/headroom/pricing/counterfactual.py
index b14b9e3c0..2763120fb 100644
--- a/headroom/pricing/counterfactual.py
+++ b/headroom/pricing/counterfactual.py
@@ -62,6 +62,7 @@ from functools import lru_cache
from typing import Any
from headroom.cache_economics import CACHE_ECONOMICS
+from headroom.log_safety import describe_exception, safe_id
from headroom.pricing.cache_ttl import CACHE_WRITE_MULTIPLIERS
logger = logging.getLogger(__name__)
@@ -421,7 +422,12 @@ def resolve_rates(
resolved_model = resolve_litellm_model(model)
info = litellm.model_cost.get(resolved_model, {}) or {}
- except Exception:
+ except Exception as exc:
+ logger.debug(
+ "counterfactual: LiteLLM catalog lookup failed for %s: %s",
+ safe_id(model),
+ describe_exception(exc),
+ )
return None
base = info.get("input_cost_per_token")
diff --git a/headroom/providers/cohere.py b/headroom/providers/cohere.py
index 49e0ec571..421f01961 100644
--- a/headroom/providers/cohere.py
+++ b/headroom/providers/cohere.py
@@ -21,6 +21,7 @@ import warnings
from datetime import date
from typing import Any
+from headroom.log_safety import describe_exception, safe_id
from headroom.pricing.litellm_pricing import estimate_cost_from_tokens
from headroom.tokenizers import EstimatingTokenCounter
@@ -284,8 +285,11 @@ class CohereProvider(Provider):
result = info["max_tokens"]
if result is not None:
return int(result)
- except Exception:
- pass
+ except Exception as e:
+ logger.debug(
+ f"LiteLLM get_model_info failed for {safe_id(model_variant)}: "
+ f"{describe_exception(e)}"
+ )
# Fallback to built-in limits
model_lower = model.lower()
diff --git a/headroom/providers/openai.py b/headroom/providers/openai.py
index 4466d07ec..06c2e3802 100644
--- a/headroom/providers/openai.py
+++ b/headroom/providers/openai.py
@@ -16,6 +16,7 @@ from functools import lru_cache
from typing import Any, cast
from headroom import paths as _paths
+from headroom.log_safety import describe_exception, safe_id
from headroom.pricing.litellm_pricing import estimate_cost_from_tokens
from headroom.tokenizers.base import (
TokenCountCache,
@@ -603,8 +604,10 @@ class OpenAIProvider(Provider):
max_tokens = info["max_input_tokens"]
if max_tokens is not None:
return int(max_tokens)
- except Exception:
- pass
+ except Exception as e:
+ logger.debug(
+ f"LiteLLM get_model_info failed for {safe_id(model)}: {describe_exception(e)}"
+ )
# Fall back to hardcoded
return self._get_context_limit_manual(model)
diff --git a/tests/test_litellm_probe_fallback_logs.py b/tests/test_litellm_probe_fallback_logs.py
new file mode 100644
index 000000000..4adcc20ca
--- /dev/null
+++ b/tests/test_litellm_probe_fallback_logs.py
@@ -0,0 +1,120 @@
+"""A failed LiteLLM probe falls back as before and logs the model and failure type only.
+
+The probe's exception text comes from LiteLLM and can carry backend configuration or
+credentials, so no log record may contain it, at any level, with or without a traceback.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Iterator
+from types import SimpleNamespace
+
+import pytest
+
+import headroom.pricing.counterfactual as counterfactual
+import headroom.providers.cohere as cohere
+import headroom.providers.openai as openai_provider
+
+CREDENTIAL_CANARY = "sk-canary-0123456789abcdef"
+CONTENT_CANARY = "user-prompt-canary-text"
+
+# A dated variant is not in the built-in table, so get_context_limit reaches the LiteLLM probe.
+_DATED_GPT4O = "gpt-4o-2099-01-01"
+
+
+class _Records(logging.Handler):
+ def __init__(self) -> None:
+ super().__init__(logging.DEBUG)
+ self.records: list[logging.LogRecord] = []
+
+ def emit(self, record: logging.LogRecord) -> None:
+ self.records.append(record)
+
+ def formatted(self) -> list[str]:
+ # A real Formatter renders exc_info and stack_info, which getMessage() leaves out.
+ formatter = logging.Formatter("%(levelname)s %(name)s %(message)s")
+ return [formatter.format(record) for record in self.records]
+
+
+def records(monkeypatch: pytest.MonkeyPatch) -> Iterator[_Records]:
+ # The content opt-in would put exception text back on purpose; test the default.
+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
+ # Attach directly: headroom.* loggers may not propagate to caplog's root handler.
+ handler = _Records()
+ loggers = [logging.getLogger(m.__name__) for m in (openai_provider, cohere, counterfactual)]
+ saved = [lg.level for lg in loggers]
+ for lg in loggers:
+ lg.addHandler(handler)
+ lg.setLevel(logging.DEBUG)
+ yield handler
+ for lg, level in zip(loggers, saved):
+ lg.removeHandler(handler)
+ lg.setLevel(level)
+
+
+def _failing(*_args: object, **_kwargs: object) -> dict[str, object]:
+ try:
+ raise KeyError(f"api_key={CREDENTIAL_CANARY}")
+ except KeyError as cause:
+ raise RuntimeError(
+ f"catalog error for {CONTENT_CANARY} at https://u:{CREDENTIAL_CANARY}@x.test"
+ ) from cause
+
+
+def _assert_safe(records: _Records, model: str) -> None:
+ lines = records.formatted()
+ for line in lines:
+ assert CREDENTIAL_CANARY not in line
+ assert CONTENT_CANARY not in line
+ assert "Traceback" not in line
+ failure_lines = [line for line in lines if "failed for" in line and repr(model) in line]
+ assert failure_lines, lines
+ assert all("RuntimeError" in line for line in failure_lines)
+
+
+def test_openai_context_limit_probe_failure_is_logged_safely(
+ monkeypatch: pytest.MonkeyPatch, records: _Records
+) -> None:
+ provider = openai_provider.OpenAIProvider()
+ expected = provider._get_context_limit_manual(_DATED_GPT4O)
+
+ monkeypatch.setattr(
+ openai_provider, "_get_litellm_module", lambda: SimpleNamespace(get_model_info=_failing)
+ )
+ assert provider.get_context_limit(_DATED_GPT4O) == expected
+ _assert_safe(records, _DATED_GPT4O)
+
+ records.records.clear()
+ monkeypatch.setattr(
+ openai_provider,
+ "_get_litellm_module",
+ lambda: SimpleNamespace(get_model_info=lambda m: {"max_input_tokens": 1234}),
+ )
+ assert provider.get_context_limit(_DATED_GPT4O) == 1234
+ assert not [line for line in records.formatted() if "get_model_info failed" in line]
+
+
+def test_cohere_context_limit_probe_failure_is_logged_safely(
+ monkeypatch: pytest.MonkeyPatch, records: _Records
+) -> None:
+ monkeypatch.setattr(cohere, "LITELLM_AVAILABLE", True)
+ monkeypatch.setattr(cohere, "litellm", SimpleNamespace(get_model_info=_failing), raising=False)
+
+ assert cohere.CohereProvider().get_context_limit("command-r") == 128000
+ _assert_safe(records, "cohere/command-r")
+ _assert_safe(records, "command-r")
+
+
+def test_counterfactual_catalog_failure_is_logged_safely(
+ monkeypatch: pytest.MonkeyPatch, records: _Records
+) -> None:
+ broken_catalog = SimpleNamespace(model_cost=SimpleNamespace(get=_failing))
+ monkeypatch.setattr(counterfactual, "_litellm", lambda: broken_catalog)
+ counterfactual.resolve_rates.cache_clear()
+ try:
+ assert counterfactual.resolve_rates("probe-fail-model") is None
+ finally:
+ counterfactual.resolve_rates.cache_clear()
+ _assert_safe(records, "probe-fail-model")