Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(offline): guard the learn and memory-eval LLM calls under HEADROOM_OFFLINE (#4344)

Abel Yagubyandd361bb7a3a1
diff --git a/headroom/cli/evals.py b/headroom/cli/evals.py
index 8a6c848de..3515fef0d 100644
--- a/headroom/cli/evals.py
+++ b/headroom/cli/evals.py
@@ -481,6 +481,9 @@ def _run_memory_eval(
# Create answer function based on provider
answer_fn = None
if answer_model:
+ from headroom.offline import guard_egress
+
+ guard_egress(f"{answer_model} API for the memory eval answers")
try:
pin_litellm_cost_map_local()
import litellm
diff --git a/headroom/cli/learn.py b/headroom/cli/learn.py
index e38995dc3..15889b060 100644
--- a/headroom/cli/learn.py
+++ b/headroom/cli/learn.py
@@ -387,6 +387,9 @@ def _make_llm_judge(model: str) -> Any:
best-effort: any failure (no key, parse error) returns None so the caller
falls back to the heuristic.
"""
+ from headroom.offline import guard_egress
+
+ guard_egress(f"{model} API for the headroom learn --llm-judge verbosity judge")
def judge(signals: dict) -> tuple[int, str] | None:
try:
diff --git a/headroom/evals/memory/judge.py b/headroom/evals/memory/judge.py
index 49d761c52..2833830e0 100644
--- a/headroom/evals/memory/judge.py
+++ b/headroom/evals/memory/judge.py
@@ -176,6 +176,8 @@ def create_litellm_judge(
"LiteLLM package required for LLM judge. Install with: pip install litellm"
) from e
+ guard_egress(f"{model} API for the memory eval LLM judge")
+
def judge(question: str, ground_truth: str, prediction: str) -> tuple[float, str]:
prompt = JUDGE_PROMPT.format(
question=question,
diff --git a/headroom/evals/memory/locomo.py b/headroom/evals/memory/locomo.py
index ecee5b384..cc9922d7b 100644
--- a/headroom/evals/memory/locomo.py
+++ b/headroom/evals/memory/locomo.py
@@ -29,6 +29,8 @@ from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
+from headroom.offline import guard_egress
+
logger = logging.getLogger(__name__)
# LoCoMo dataset URL
@@ -231,6 +233,7 @@ def download_locomo(cache_dir: Path | None = None) -> Path:
logger.info(f"Using cached LoCoMo dataset: {cache_path}")
return cache_path
+ guard_egress("LoCoMo eval-dataset download", LOCOMO_URL)
logger.info(f"Downloading LoCoMo dataset from {LOCOMO_URL}...")
urllib.request.urlretrieve(LOCOMO_URL, cache_path) # nosec B310
logger.info(f"Downloaded to {cache_path}")
diff --git a/headroom/evals/memory/runner.py b/headroom/evals/memory/runner.py
index 5775ff267..357a7a073 100644
--- a/headroom/evals/memory/runner.py
+++ b/headroom/evals/memory/runner.py
@@ -323,6 +323,9 @@ OUTPUT FORMAT (JSON only):
IMPORTANT: Every event MUST have a specific date. If you cannot determine the date, state it as "around {session_datetime}"."""
+ from headroom.offline import guard_egress
+
+ guard_egress(f"{self.config.extraction_model} API for memory eval extraction")
try:
response = litellm.completion(
model=self.config.extraction_model,
diff --git a/headroom/evals/memory/runner_v2.py b/headroom/evals/memory/runner_v2.py
index 5864bebf0..286fbd6e1 100644
--- a/headroom/evals/memory/runner_v2.py
+++ b/headroom/evals/memory/runner_v2.py
@@ -467,6 +467,9 @@ Categories to use:
# Get just the save tool
save_tool = next(t for t in MEMORY_TOOLS if t["function"]["name"] == "memory_save")
+ from headroom.offline import guard_egress
+
+ guard_egress(f"{self._config.save_model} API for the memory eval save phase")
for session in conversation.sessions:
# Format session as dialogue
session_text = f"\n[Session Date: {session.datetime}]\n"
@@ -689,6 +692,9 @@ The answer should contain ONLY the specific information requested, nothing more.
max_rounds = 3
final_answer = "Information not found"
+ from headroom.offline import guard_egress
+
+ guard_egress(f"{self._config.answer_model} API for the memory eval answers")
for round_num in range(max_rounds):
try:
response = await asyncio.to_thread(
diff --git a/headroom/learn/analyzer.py b/headroom/learn/analyzer.py
index b5a6d28e5..363f8697e 100644
--- a/headroom/learn/analyzer.py
+++ b/headroom/learn/analyzer.py
@@ -1093,7 +1093,12 @@ def _call_llm(
the provider: "claude-*" → Anthropic, "gpt-*" → OpenAI, "gemini/*" → Google, etc.
For CLI-based models (ending in "-cli"), delegates to ``_call_cli_llm``.
"""
+ from headroom.offline import guard_egress
+
if model in _CLI_MODEL_IDS:
+ # The provider CLI sends the digest to its own API, so refuse before
+ # spawning it rather than letting the subprocess reach the network.
+ guard_egress(f"{model} provider CLI for headroom learn session analysis")
return _call_cli_llm(digest, model, on_progress=on_progress)
pin_litellm_cost_map_local()
@@ -1108,6 +1113,7 @@ def _call_llm(
if model.startswith("claude"):
api_base = "https://api.anthropic.com"
+ guard_egress(f"{model} API for headroom learn session analysis", api_base)
response = litellm.completion(
model=model,
messages=[
diff --git a/tests/test_offline_egress_chokepoint.py b/tests/test_offline_egress_chokepoint.py
index 22bf0c9e3..8c85dbf7f 100644
--- a/tests/test_offline_egress_chokepoint.py
+++ b/tests/test_offline_egress_chokepoint.py
@@ -2147,6 +2147,167 @@ class TestEvalRunnerLocalProvider:
runner._init_llm_client()
+class TestLearnAndMemoryEvalOffline:
+ """`headroom learn` and the memory evals reach their LLM through litellm,
+ and the LoCoMo loader downloads with urlretrieve. None of them consulted
+ the switch (#4338). litellm is replaced by a recorder so a refused path
+ is seen to stop before the request, not just to raise somewhere."""
+
+ @pytest.fixture
+ def litellm_calls(self, monkeypatch: pytest.MonkeyPatch) -> list[dict]:
+ import sys
+
+ calls: list[dict] = []
+ fake = SimpleNamespace(
+ completion=lambda **kwargs: calls.append(kwargs),
+ suppress_debug_info=False,
+ )
+ monkeypatch.setitem(sys.modules, "litellm", fake)
+ return calls
+
+ def test_learn_analysis_is_refused(
+ self, offline: None, no_sockets: None, litellm_calls: list[dict]
+ ) -> None:
+ from headroom.learn import analyzer
+
+ with pytest.raises(OfflineEgressBlocked) as excinfo:
+ analyzer._call_llm("digest", "claude-sonnet-4-6")
+ assert _refused_hostname(excinfo.value) == "api.anthropic.com"
+ assert litellm_calls == []
+
+ @pytest.fixture
+ def cli_calls(self, monkeypatch: pytest.MonkeyPatch) -> list[str]:
+ """Record the provider-CLI delegate and fail on any spawned process."""
+ from headroom.learn import analyzer
+
+ calls: list[str] = []
+
+ def no_process(*args: object, **kwargs: object) -> None:
+ raise AssertionError(f"learn spawned a provider CLI: {args!r}")
+
+ def fake_cli(digest: str, model: str, on_progress: object = None) -> dict:
+ calls.append(model)
+ return {"context_file_rules": []}
+
+ monkeypatch.setattr(analyzer, "_call_cli_llm", fake_cli)
+ monkeypatch.setattr(analyzer, "Popen", no_process)
+ monkeypatch.setattr(analyzer, "run", no_process)
+ monkeypatch.setattr(analyzer.subprocess, "Popen", no_process)
+ monkeypatch.setattr(analyzer.subprocess, "run", no_process)
+ return calls
+
+ @pytest.mark.parametrize("model", ["claude-cli", "gemini-cli", "codex-cli", "agy-cli"])
+ def test_learn_cli_analysis_is_refused_before_the_cli_runs(
+ self, offline: None, no_sockets: None, cli_calls: list[str], model: str
+ ) -> None:
+ from headroom.learn import analyzer
+
+ assert model in analyzer._CLI_MODEL_IDS
+ with pytest.raises(OfflineEgressBlocked, match=model):
+ analyzer._call_llm("digest", model)
+ assert cli_calls == []
+
+ def test_learn_cli_analysis_still_runs_online(
+ self, monkeypatch: pytest.MonkeyPatch, cli_calls: list[str]
+ ) -> None:
+ from headroom.learn import analyzer
+
+ monkeypatch.delenv("HEADROOM_OFFLINE", raising=False)
+ assert analyzer._call_llm("digest", "claude-cli") == {"context_file_rules": []}
+ assert cli_calls == ["claude-cli"]
+
+ def test_learn_verbosity_llm_judge_is_refused(
+ self, offline: None, no_sockets: None, litellm_calls: list[dict]
+ ) -> None:
+ from headroom.cli import learn
+
+ with pytest.raises(OfflineEgressBlocked):
+ learn._make_llm_judge("gpt-4o")
+ assert litellm_calls == []
+
+ def test_memory_eval_answer_model_is_refused(
+ self, offline: None, no_sockets: None, litellm_calls: list[dict]
+ ) -> None:
+ from headroom.cli import evals
+
+ with pytest.raises(OfflineEgressBlocked):
+ evals._run_memory_eval(
+ n_conversations=1,
+ categories=None,
+ include_adversarial=False,
+ top_k=10,
+ f1_threshold=0.5,
+ answer_model="gpt-4o",
+ llm_judge=False,
+ judge_provider="simple",
+ judge_model="gpt-4o",
+ output=None,
+ no_extract=True,
+ extraction_model="gpt-4o-mini",
+ pass_all=False,
+ parallel=1,
+ debug=False,
+ )
+ assert litellm_calls == []
+
+ def test_memory_eval_litellm_judge_is_refused(
+ self, offline: None, no_sockets: None, litellm_calls: list[dict]
+ ) -> None:
+ from headroom.evals.memory.judge import create_litellm_judge
+
+ with pytest.raises(OfflineEgressBlocked):
+ create_litellm_judge(model="gpt-4o")
+ assert litellm_calls == []
+
+ def test_memory_eval_extraction_is_refused(
+ self, offline: None, no_sockets: None, litellm_calls: list[dict]
+ ) -> None:
+ from headroom.evals.memory.runner import LoCoMoEvaluator
+
+ evaluator = LoCoMoEvaluator()
+ with pytest.raises(OfflineEgressBlocked):
+ evaluator._extract_memories_from_session("A: hi", "1 May 2023", "A", "B")
+ assert litellm_calls == []
+
+ def test_memory_eval_v2_save_and_answer_are_refused(
+ self, offline: None, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ import asyncio
+
+ from headroom.evals.memory import runner_v2
+
+ calls: list[dict] = []
+ monkeypatch.setattr(
+ runner_v2, "litellm", SimpleNamespace(completion=lambda **kw: calls.append(kw))
+ )
+ evaluator = runner_v2.LoCoMoEvaluatorV2()
+ conversation = SimpleNamespace(speaker_a="A", speaker_b="B", sessions=[])
+ with pytest.raises(OfflineEgressBlocked):
+ asyncio.run(evaluator._replay_conversation(conversation))
+ with pytest.raises(OfflineEgressBlocked):
+ asyncio.run(evaluator._answer_question(SimpleNamespace(question="Who?"), conversation))
+ assert calls == []
+
+ def test_locomo_download_opens_no_socket(
+ self, offline: None, no_sockets: None, tmp_path: Path
+ ) -> None:
+ from headroom.evals.memory.locomo import download_locomo
+
+ with pytest.raises(OfflineEgressBlocked) as excinfo:
+ download_locomo(cache_dir=tmp_path)
+ assert _refused_hostname(excinfo.value) == "raw.githubusercontent.com"
+ assert not (tmp_path / "locomo10.json").exists()
+
+ def test_a_cached_locomo_dataset_still_loads_offline(
+ self, offline: None, no_sockets: None, tmp_path: Path
+ ) -> None:
+ from headroom.evals.memory.locomo import download_locomo
+
+ cached = tmp_path / "locomo10.json"
+ cached.write_text("[]", encoding="utf-8")
+ assert download_locomo(cache_dir=tmp_path) == cached
+
+
class TestHeadroomCloudCompressionOffline:
"""Both Headroom Cloud integrations.
@@ -2407,6 +2568,7 @@ _EGRESS_CALLEES = re.compile(
| httpx\.(?:get|post|put|patch|delete|head|request|stream) # module-level verbs
| requests\.(?:get|post|put|patch|delete|head|request|Session)
| (?:urllib\.request\.)?urlopen | _urlopen # urllib, bare or wrapped
+ | (?:urllib\.request\.)?urlretrieve # urllib file download
| aiohttp\.ClientSession
| urllib3\.PoolManager
| (?:huggingface_hub\.)?hf_hub_download # the Python half of the HF fetch
@@ -2418,6 +2580,8 @@ _EGRESS_CALLEES = re.compile(
| OTLP(?:Metric|Span|Log)Exporter # OTEL export, incl. its background timer
| (?:openai\.)?(?:Async)?(?:OpenAI|AzureOpenAI) # provider SDKs build their own
| (?:anthropic\.)?(?:Async)?Anthropic(?:Bedrock|Vertex)?
+ | litellm\.a?completion # litellm-routed provider calls
+ | _call_cli_llm # learn's provider CLIs call their APIs
)$""",
re.VERBOSE,
)
@@ -2668,6 +2832,13 @@ _EGRESS_ALLOWLIST: dict[str, tuple[int, str]] = {
"deployment that points Headroom at an on-prem model endpoint, which "
"is the main reason such a deployment exists.",
),
+ "backends/litellm.py": (
+ 4,
+ "user-traffic: LiteLLMBackend's send/stream paths (Anthropic and "
+ "OpenAI shaped) forward the caller's request to the provider the "
+ "operator configured as the proxy backend (Bedrock, Vertex, "
+ "OpenRouter, ...). Same role as the proxy's own forwarding above.",
+ ),
"telemetry/session.py": (
1,
"gated: the beacon upload is reached only via is_telemetry_enabled(), "