Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(offline): guard the learn and memory-eval LLM calls under HEADROOM_OFFLINE (#4344)
Abel Yagubyandd361bb7a3a1
diff --git a/headroom/cli/evals.py b/headroom/cli/evals.pyindex 8a6c848de..3515fef0d 100644--- a/headroom/cli/evals.py+++ b/headroom/cli/evals.py@@ -481,6 +481,9 @@ def _run_memory_eval(# Create answer function based on provideranswer_fn = Noneif answer_model:+ from headroom.offline import guard_egress++ guard_egress(f"{answer_model} API for the memory eval answers")try:pin_litellm_cost_map_local()import litellmdiff --git a/headroom/cli/learn.py b/headroom/cli/learn.pyindex e38995dc3..15889b060 100644--- a/headroom/cli/learn.py+++ b/headroom/cli/learn.py@@ -387,6 +387,9 @@ def _make_llm_judge(model: str) -> Any:best-effort: any failure (no key, parse error) returns None so the callerfalls back to the heuristic."""+ from headroom.offline import guard_egress++ guard_egress(f"{model} API for the headroom learn --llm-judge verbosity judge")def judge(signals: dict) -> tuple[int, str] | None:try:diff --git a/headroom/evals/memory/judge.py b/headroom/evals/memory/judge.pyindex 49d761c52..2833830e0 100644--- a/headroom/evals/memory/judge.py+++ b/headroom/evals/memory/judge.py@@ -176,6 +176,8 @@ def create_litellm_judge("LiteLLM package required for LLM judge. Install with: pip install litellm") from e+ guard_egress(f"{model} API for the memory eval LLM judge")+def judge(question: str, ground_truth: str, prediction: str) -> tuple[float, str]:prompt = JUDGE_PROMPT.format(question=question,diff --git a/headroom/evals/memory/locomo.py b/headroom/evals/memory/locomo.pyindex ecee5b384..cc9922d7b 100644--- a/headroom/evals/memory/locomo.py+++ b/headroom/evals/memory/locomo.py@@ -29,6 +29,8 @@ from dataclasses import dataclass, fieldfrom pathlib import Pathfrom typing import Any+from headroom.offline import guard_egress+logger = logging.getLogger(__name__)# LoCoMo dataset URL@@ -231,6 +233,7 @@ def download_locomo(cache_dir: Path | None = None) -> Path:logger.info(f"Using cached LoCoMo dataset: {cache_path}")return cache_path+ guard_egress("LoCoMo eval-dataset download", LOCOMO_URL)logger.info(f"Downloading LoCoMo dataset from {LOCOMO_URL}...")urllib.request.urlretrieve(LOCOMO_URL, cache_path) # nosec B310logger.info(f"Downloaded to {cache_path}")diff --git a/headroom/evals/memory/runner.py b/headroom/evals/memory/runner.pyindex 5775ff267..357a7a073 100644--- a/headroom/evals/memory/runner.py+++ b/headroom/evals/memory/runner.py@@ -323,6 +323,9 @@ OUTPUT FORMAT (JSON only):IMPORTANT: Every event MUST have a specific date. If you cannot determine the date, state it as "around {session_datetime}"."""+ from headroom.offline import guard_egress++ guard_egress(f"{self.config.extraction_model} API for memory eval extraction")try:response = litellm.completion(model=self.config.extraction_model,diff --git a/headroom/evals/memory/runner_v2.py b/headroom/evals/memory/runner_v2.pyindex 5864bebf0..286fbd6e1 100644--- a/headroom/evals/memory/runner_v2.py+++ b/headroom/evals/memory/runner_v2.py@@ -467,6 +467,9 @@ Categories to use:# Get just the save toolsave_tool = next(t for t in MEMORY_TOOLS if t["function"]["name"] == "memory_save")+ from headroom.offline import guard_egress++ guard_egress(f"{self._config.save_model} API for the memory eval save phase")for session in conversation.sessions:# Format session as dialoguesession_text = f"\n[Session Date: {session.datetime}]\n"@@ -689,6 +692,9 @@ The answer should contain ONLY the specific information requested, nothing more.max_rounds = 3final_answer = "Information not found"+ from headroom.offline import guard_egress++ guard_egress(f"{self._config.answer_model} API for the memory eval answers")for round_num in range(max_rounds):try:response = await asyncio.to_thread(diff --git a/headroom/learn/analyzer.py b/headroom/learn/analyzer.pyindex b5a6d28e5..363f8697e 100644--- a/headroom/learn/analyzer.py+++ b/headroom/learn/analyzer.py@@ -1093,7 +1093,12 @@ def _call_llm(the provider: "claude-*" → Anthropic, "gpt-*" → OpenAI, "gemini/*" → Google, etc.For CLI-based models (ending in "-cli"), delegates to ``_call_cli_llm``."""+ from headroom.offline import guard_egress+if model in _CLI_MODEL_IDS:+ # The provider CLI sends the digest to its own API, so refuse before+ # spawning it rather than letting the subprocess reach the network.+ guard_egress(f"{model} provider CLI for headroom learn session analysis")return _call_cli_llm(digest, model, on_progress=on_progress)pin_litellm_cost_map_local()@@ -1108,6 +1113,7 @@ def _call_llm(if model.startswith("claude"):api_base = "https://api.anthropic.com"+ guard_egress(f"{model} API for headroom learn session analysis", api_base)response = litellm.completion(model=model,messages=[diff --git a/tests/test_offline_egress_chokepoint.py b/tests/test_offline_egress_chokepoint.pyindex 22bf0c9e3..8c85dbf7f 100644--- a/tests/test_offline_egress_chokepoint.py+++ b/tests/test_offline_egress_chokepoint.py@@ -2147,6 +2147,167 @@ class TestEvalRunnerLocalProvider:runner._init_llm_client()+class TestLearnAndMemoryEvalOffline:+ """`headroom learn` and the memory evals reach their LLM through litellm,+ and the LoCoMo loader downloads with urlretrieve. None of them consulted+ the switch (#4338). litellm is replaced by a recorder so a refused path+ is seen to stop before the request, not just to raise somewhere."""++ @pytest.fixture+ def litellm_calls(self, monkeypatch: pytest.MonkeyPatch) -> list[dict]:+ import sys++ calls: list[dict] = []+ fake = SimpleNamespace(+ completion=lambda **kwargs: calls.append(kwargs),+ suppress_debug_info=False,+ )+ monkeypatch.setitem(sys.modules, "litellm", fake)+ return calls++ def test_learn_analysis_is_refused(+ self, offline: None, no_sockets: None, litellm_calls: list[dict]+ ) -> None:+ from headroom.learn import analyzer++ with pytest.raises(OfflineEgressBlocked) as excinfo:+ analyzer._call_llm("digest", "claude-sonnet-4-6")+ assert _refused_hostname(excinfo.value) == "api.anthropic.com"+ assert litellm_calls == []++ @pytest.fixture+ def cli_calls(self, monkeypatch: pytest.MonkeyPatch) -> list[str]:+ """Record the provider-CLI delegate and fail on any spawned process."""+ from headroom.learn import analyzer++ calls: list[str] = []++ def no_process(*args: object, **kwargs: object) -> None:+ raise AssertionError(f"learn spawned a provider CLI: {args!r}")++ def fake_cli(digest: str, model: str, on_progress: object = None) -> dict:+ calls.append(model)+ return {"context_file_rules": []}++ monkeypatch.setattr(analyzer, "_call_cli_llm", fake_cli)+ monkeypatch.setattr(analyzer, "Popen", no_process)+ monkeypatch.setattr(analyzer, "run", no_process)+ monkeypatch.setattr(analyzer.subprocess, "Popen", no_process)+ monkeypatch.setattr(analyzer.subprocess, "run", no_process)+ return calls++ @pytest.mark.parametrize("model", ["claude-cli", "gemini-cli", "codex-cli", "agy-cli"])+ def test_learn_cli_analysis_is_refused_before_the_cli_runs(+ self, offline: None, no_sockets: None, cli_calls: list[str], model: str+ ) -> None:+ from headroom.learn import analyzer++ assert model in analyzer._CLI_MODEL_IDS+ with pytest.raises(OfflineEgressBlocked, match=model):+ analyzer._call_llm("digest", model)+ assert cli_calls == []++ def test_learn_cli_analysis_still_runs_online(+ self, monkeypatch: pytest.MonkeyPatch, cli_calls: list[str]+ ) -> None:+ from headroom.learn import analyzer++ monkeypatch.delenv("HEADROOM_OFFLINE", raising=False)+ assert analyzer._call_llm("digest", "claude-cli") == {"context_file_rules": []}+ assert cli_calls == ["claude-cli"]++ def test_learn_verbosity_llm_judge_is_refused(+ self, offline: None, no_sockets: None, litellm_calls: list[dict]+ ) -> None:+ from headroom.cli import learn++ with pytest.raises(OfflineEgressBlocked):+ learn._make_llm_judge("gpt-4o")+ assert litellm_calls == []++ def test_memory_eval_answer_model_is_refused(+ self, offline: None, no_sockets: None, litellm_calls: list[dict]+ ) -> None:+ from headroom.cli import evals++ with pytest.raises(OfflineEgressBlocked):+ evals._run_memory_eval(+ n_conversations=1,+ categories=None,+ include_adversarial=False,+ top_k=10,+ f1_threshold=0.5,+ answer_model="gpt-4o",+ llm_judge=False,+ judge_provider="simple",+ judge_model="gpt-4o",+ output=None,+ no_extract=True,+ extraction_model="gpt-4o-mini",+ pass_all=False,+ parallel=1,+ debug=False,+ )+ assert litellm_calls == []++ def test_memory_eval_litellm_judge_is_refused(+ self, offline: None, no_sockets: None, litellm_calls: list[dict]+ ) -> None:+ from headroom.evals.memory.judge import create_litellm_judge++ with pytest.raises(OfflineEgressBlocked):+ create_litellm_judge(model="gpt-4o")+ assert litellm_calls == []++ def test_memory_eval_extraction_is_refused(+ self, offline: None, no_sockets: None, litellm_calls: list[dict]+ ) -> None:+ from headroom.evals.memory.runner import LoCoMoEvaluator++ evaluator = LoCoMoEvaluator()+ with pytest.raises(OfflineEgressBlocked):+ evaluator._extract_memories_from_session("A: hi", "1 May 2023", "A", "B")+ assert litellm_calls == []++ def test_memory_eval_v2_save_and_answer_are_refused(+ self, offline: None, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ import asyncio++ from headroom.evals.memory import runner_v2++ calls: list[dict] = []+ monkeypatch.setattr(+ runner_v2, "litellm", SimpleNamespace(completion=lambda **kw: calls.append(kw))+ )+ evaluator = runner_v2.LoCoMoEvaluatorV2()+ conversation = SimpleNamespace(speaker_a="A", speaker_b="B", sessions=[])+ with pytest.raises(OfflineEgressBlocked):+ asyncio.run(evaluator._replay_conversation(conversation))+ with pytest.raises(OfflineEgressBlocked):+ asyncio.run(evaluator._answer_question(SimpleNamespace(question="Who?"), conversation))+ assert calls == []++ def test_locomo_download_opens_no_socket(+ self, offline: None, no_sockets: None, tmp_path: Path+ ) -> None:+ from headroom.evals.memory.locomo import download_locomo++ with pytest.raises(OfflineEgressBlocked) as excinfo:+ download_locomo(cache_dir=tmp_path)+ assert _refused_hostname(excinfo.value) == "raw.githubusercontent.com"+ assert not (tmp_path / "locomo10.json").exists()++ def test_a_cached_locomo_dataset_still_loads_offline(+ self, offline: None, no_sockets: None, tmp_path: Path+ ) -> None:+ from headroom.evals.memory.locomo import download_locomo++ cached = tmp_path / "locomo10.json"+ cached.write_text("[]", encoding="utf-8")+ assert download_locomo(cache_dir=tmp_path) == cached++class TestHeadroomCloudCompressionOffline:"""Both Headroom Cloud integrations.@@ -2407,6 +2568,7 @@ _EGRESS_CALLEES = re.compile(| httpx\.(?:get|post|put|patch|delete|head|request|stream) # module-level verbs| requests\.(?:get|post|put|patch|delete|head|request|Session)| (?:urllib\.request\.)?urlopen | _urlopen # urllib, bare or wrapped+ | (?:urllib\.request\.)?urlretrieve # urllib file download| aiohttp\.ClientSession| urllib3\.PoolManager| (?:huggingface_hub\.)?hf_hub_download # the Python half of the HF fetch@@ -2418,6 +2580,8 @@ _EGRESS_CALLEES = re.compile(| OTLP(?:Metric|Span|Log)Exporter # OTEL export, incl. its background timer| (?:openai\.)?(?:Async)?(?:OpenAI|AzureOpenAI) # provider SDKs build their own| (?:anthropic\.)?(?:Async)?Anthropic(?:Bedrock|Vertex)?+ | litellm\.a?completion # litellm-routed provider calls+ | _call_cli_llm # learn's provider CLIs call their APIs)$""",re.VERBOSE,)@@ -2668,6 +2832,13 @@ _EGRESS_ALLOWLIST: dict[str, tuple[int, str]] = {"deployment that points Headroom at an on-prem model endpoint, which ""is the main reason such a deployment exists.",),+ "backends/litellm.py": (+ 4,+ "user-traffic: LiteLLMBackend's send/stream paths (Anthropic and "+ "OpenAI shaped) forward the caller's request to the provider the "+ "operator configured as the proxy backend (Bedrock, Vertex, "+ "OpenRouter, ...). Same role as the proxy's own forwarding above.",+ ),"telemetry/session.py": (1,"gated: the beacon upload is reached only via is_telemetry_enabled(), "