Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(learn): add CLI-based LLM backends for keyless headroom learn
Gyeonghun Parke98c9751538f
diff --git a/docs/learn.md b/docs/learn.mdindex dce072a81..45e762aeb 100644--- a/docs/learn.md+++ b/docs/learn.md@@ -145,9 +145,39 @@ Options:--project PATH Project directory to analyze (default: current directory)--all Analyze all discovered projects--apply Write recommendations (default: dry-run)- --claude-dir PATH Path to .claude directory (default: ~/.claude)+ --model TEXT LLM model for analysis (default: auto-detected)+ --agent TEXT Which coding agent to analyze: auto, claude, codex, gemini```+## LLM Backend Selection++`headroom learn` needs an LLM to analyze your sessions. It picks one automatically using this priority:++| Priority | Source | Example |+|----------|--------|---------|+| 1 | `--model` flag | `headroom learn --model gpt-4o` |+| 2 | API key env var | `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY` |+| 3 | `HEADROOM_LEARN_CLI` env var | `export HEADROOM_LEARN_CLI=gemini` |+| 4 | Auto-detect installed CLIs | Checks PATH for `claude`, `gemini`, `codex` |++### Using without an API key++If you use Claude Code, Gemini CLI, or Codex via subscription (no raw API key), `headroom learn` can call them directly:++```bash+# Auto-detects claude in PATH — no API key needed+headroom learn++# Explicitly select a CLI backend+headroom learn --model gemini-cli++# Pin a CLI via environment variable+export HEADROOM_LEARN_CLI=codex+headroom learn+```++Valid values for `HEADROOM_LEARN_CLI`: `claude`, `gemini`, `codex`.+## Real-World ResultsTested on 67,583 tool calls across 23 projects:diff --git a/headroom/learn/analyzer.py b/headroom/learn/analyzer.pyindex 621e2377a..03761fdb8 100644--- a/headroom/learn/analyzer.py+++ b/headroom/learn/analyzer.py@@ -8,6 +8,8 @@ structured recommendations for CLAUDE.md / MEMORY.md.Supports any LLM provider via LiteLLM: Anthropic, OpenAI, Google, Bedrock,Ollama, and 100+ others. Auto-detects the best available model from env vars.+Also supports CLI-based backends (claude, gemini, codex) for subscription+users without raw API keys."""from __future__ import annotations@@ -15,6 +17,8 @@ from __future__ import annotationsimport jsonimport loggingimport os+import shutil+import subprocessfrom .models import (AnalysisResult,@@ -37,17 +41,61 @@ _MODEL_DEFAULTS: list[tuple[str, str]] = [_MAX_DIGEST_TOKENS = 80_000 # Budget for the digest (leave room for prompt + output)+# CLI tools to try when no API key is set (checked in order).+# Each entry: (binary_name, model_identifier, command_prefix)+_CLI_BACKENDS: list[tuple[str, str, list[str]]] = [+ ("claude", "claude-cli", ["claude", "-p"]),+ ("gemini", "gemini-cli", ["gemini", "-p"]),+ ("codex", "codex-cli", ["codex", "exec"]),+]++# Set of valid CLI model identifiers, derived from _CLI_BACKENDS.+_CLI_MODEL_IDS: set[str] = {model for _, model, _ in _CLI_BACKENDS}++_USER_PROMPT_PREFIX = "Analyze these coding agent sessions and return JSON recommendations:\n\n" # Shared by _call_cli_llm and _call_llm+_MAX_SNIPPET_LEN = 2000 # Max chars of CLI output (stdout/stderr) in error messages+_CLI_TIMEOUT = 120 # Subprocess timeout for CLI backends, in seconds+def _detect_default_model() -> str:- """Pick the best available model based on which API keys are set."""+ """Pick the best available model based on API keys, env config, or CLI tools.++ Priority order:+ 1. API key present → use corresponding LiteLLM model+ 2. HEADROOM_LEARN_CLI env var → use specified CLI backend+ 3. Auto-detect installed CLI tools (claude > gemini > codex)+ 4. Raise RuntimeError with setup instructions+ """+ # 1. API key detection (existing behavior)for env_var, model in _MODEL_DEFAULTS:if os.environ.get(env_var):return model++ # 2. Explicit CLI selection via environment variable+ cli_override = os.environ.get("HEADROOM_LEARN_CLI")+ if cli_override:+ for cli_name, model, _cmd in _CLI_BACKENDS:+ if cli_name == cli_override:+ logger.info("HEADROOM_LEARN_CLI=%s — using %s CLI backend", cli_override, cli_name)+ return model+ valid = ", ".join(name for name, _, _ in _CLI_BACKENDS)+ raise ValueError(+ f"HEADROOM_LEARN_CLI={cli_override!r} is not a supported CLI. Valid values: {valid}"+ )++ # 3. Auto-detect installed CLI tools+ for cli_name, model, _cmd in _CLI_BACKENDS:+ if shutil.which(cli_name):+ logger.info("No API key found — auto-detected %s CLI as LLM backend", cli_name)+ return model+raise RuntimeError("No LLM API key found. headroom learn needs one of:\n"" export ANTHROPIC_API_KEY=sk-ant-... → uses claude-sonnet-4-6\n"" export OPENAI_API_KEY=sk-... → uses gpt-4o\n"" export GEMINI_API_KEY=... → uses gemini-2.0-flash\n"+ "Or set HEADROOM_LEARN_CLI to a coding agent CLI (claude, gemini, codex).\n"+ "Or install one of those CLIs for auto-detection.\n""Or specify a model directly: headroom learn --model <litellm-model-name>")@@ -263,12 +311,113 @@ Return ONLY valid JSON matching this schema — no other text:"""+def _strip_fenced_json(raw: str) -> dict:+ """Strip optional markdown fences and parse JSON.++ Handles both raw JSON and fenced code blocks (e.g. ```json ... ```).+ Only the first opening fence and last closing fence are removed, preserving+ any triple-backtick content that may appear inside the JSON payload.++ Args:+ raw: Raw text output from an LLM, possibly wrapped in markdown fences.++ Returns:+ Parsed JSON as a dictionary.++ Raises:+ json.JSONDecodeError: If the text is not valid JSON after stripping.+ """+ text = raw.strip()+ if text.startswith("```"):+ lines = text.split("\n")+ # Remove the first line (opening fence, e.g. ```json)+ lines = lines[1:]+ # Remove the last line if it is a closing fence+ if lines and lines[-1].strip().startswith("```"):+ lines = lines[:-1]+ text = "\n".join(lines)+ result: dict = json.loads(text)+ return result+++def _call_cli_llm(digest: str, model: str) -> dict:+ """Call a locally installed CLI tool as the LLM backend.++ Enables keyless usage for subscription-based CLI tools that handle+ their own OAuth authentication. The prompt is passed via stdin to avoid+ OS ``ARG_MAX`` limits and argument-injection risks.++ CLI invocations:+ claude-cli → echo <prompt> | claude -p+ gemini-cli → echo <prompt> | gemini -p+ codex-cli → echo <prompt> | codex exec++ Args:+ digest: Token-efficient session digest to analyze.+ model: CLI model identifier (e.g. ``claude-cli``).++ Returns:+ Parsed JSON recommendations from the CLI tool.++ Raises:+ ValueError: If *model* is not a known CLI backend.+ RuntimeError: If the CLI exits with a non-zero code or times out.+ """+ cmd: list[str] | None = None+ for _name, model_name, cmd_parts in _CLI_BACKENDS:+ if model_name == model:+ cmd = cmd_parts+ break+ if cmd is None:+ raise ValueError(f"Unknown CLI model: {model}")++ prompt = _SYSTEM_PROMPT + "\n\n" + _USER_PROMPT_PREFIX + digest++ try:+ result = subprocess.run(+ cmd,+ input=prompt,+ capture_output=True,+ text=True,+ timeout=_CLI_TIMEOUT,+ )+ except subprocess.TimeoutExpired:+ raise RuntimeError(+ f"`{' '.join(cmd)}` did not respond within {_CLI_TIMEOUT}s. "+ "Check network connectivity or try a different backend with "+ "--model <litellm-model-name>."+ ) from None++ if result.returncode != 0:+ stderr_snippet = (result.stderr or "")[:_MAX_SNIPPET_LEN]+ raise RuntimeError(+ f"`{' '.join(cmd)}` failed (exit {result.returncode}):\n{stderr_snippet}"+ )++ # Log stderr warnings even on success (auth refreshes, deprecation notices).+ if result.stderr and result.stderr.strip():+ logger.debug("CLI stderr (exit 0): %s", result.stderr[:_MAX_SNIPPET_LEN])++ try:+ return _strip_fenced_json(result.stdout)+ except json.JSONDecodeError as exc:+ stdout_snippet = (result.stdout or "")[:_MAX_SNIPPET_LEN]+ raise RuntimeError(+ f"`{' '.join(cmd)}` returned unparseable output. "+ f"First {_MAX_SNIPPET_LEN} chars:\n{stdout_snippet}"+ ) from exc++def _call_llm(digest: str, model: str) -> dict:"""Call LLM with the session digest and return parsed JSON.Uses LiteLLM for provider-agnostic access. The model string determinesthe provider: "claude-*" → Anthropic, "gpt-*" → OpenAI, "gemini/*" → Google, etc.+ For CLI-based models (ending in "-cli"), delegates to ``_call_cli_llm``."""+ if model in _CLI_MODEL_IDS:+ return _call_cli_llm(digest, model)+import litellm# Suppress LiteLLM's verbose logging@@ -286,10 +435,7 @@ def _call_llm(digest: str, model: str) -> dict:{"role": "system", "content": _SYSTEM_PROMPT},{"role": "user",- "content": (- "Analyze these coding agent sessions and return JSON recommendations:\n\n"- + digest- ),+ "content": _USER_PROMPT_PREFIX + digest,},],max_tokens=4096,@@ -298,16 +444,7 @@ def _call_llm(digest: str, model: str) -> dict:# Extract text from responsetext = response.choices[0].message.content or ""-- # Parse JSON — handle both raw JSON and ```json fenced blocks- text = text.strip()- if text.startswith("```"):- lines = text.split("\n")- lines = [ln for ln in lines[1:] if not ln.strip().startswith("```")]- text = "\n".join(lines)-- result: dict = json.loads(text)- return result+ return _strip_fenced_json(text)# =============================================================================diff --git a/tests/test_learn/test_analyzer.py b/tests/test_learn/test_analyzer.pyindex 30b344d76..4c1225d0d 100644--- a/tests/test_learn/test_analyzer.py+++ b/tests/test_learn/test_analyzer.py@@ -1,13 +1,20 @@"""Tests for session analyzer — digest builder and LLM-based analysis."""+import json+import subprocessfrom pathlib import Pathfrom unittest.mock import MagicMock, patch+import pytest+from headroom.learn.analyzer import (SessionAnalyzer,_build_digest,+ _call_cli_llm,+ _call_llm,_detect_default_model,_parse_llm_response,+ _strip_fenced_json,)from headroom.learn.models import (AnalysisResult,@@ -347,15 +354,230 @@ class TestDetectDefaultModel:monkeypatch.setenv("OPENAI_API_KEY", "sk-test")assert _detect_default_model() == "claude-sonnet-4-6"- def test_no_keys_raises(self, monkeypatch):+ def test_no_keys_no_cli_raises(self, monkeypatch):monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)monkeypatch.delenv("OPENAI_API_KEY", raising=False)monkeypatch.delenv("GEMINI_API_KEY", raising=False)- import pytest+ monkeypatch.setattr("headroom.learn.analyzer.shutil.which", lambda _name: None)with pytest.raises(RuntimeError, match="No LLM API key found"):_detect_default_model()+ def test_cli_fallback_claude(self, monkeypatch):+ monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)+ monkeypatch.delenv("OPENAI_API_KEY", raising=False)+ monkeypatch.delenv("GEMINI_API_KEY", raising=False)+ monkeypatch.setattr(+ "headroom.learn.analyzer.shutil.which",+ lambda name: f"/usr/bin/{name}" if name == "claude" else None,+ )+ assert _detect_default_model() == "claude-cli"++ def test_cli_fallback_gemini(self, monkeypatch):+ monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)+ monkeypatch.delenv("OPENAI_API_KEY", raising=False)+ monkeypatch.delenv("GEMINI_API_KEY", raising=False)+ monkeypatch.setattr(+ "headroom.learn.analyzer.shutil.which",+ lambda name: f"/usr/bin/{name}" if name == "gemini" else None,+ )+ assert _detect_default_model() == "gemini-cli"++ def test_cli_fallback_codex(self, monkeypatch):+ monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)+ monkeypatch.delenv("OPENAI_API_KEY", raising=False)+ monkeypatch.delenv("GEMINI_API_KEY", raising=False)+ monkeypatch.setattr(+ "headroom.learn.analyzer.shutil.which",+ lambda name: f"/usr/bin/{name}" if name == "codex" else None,+ )+ assert _detect_default_model() == "codex-cli"++ def test_api_key_preferred_over_cli(self, monkeypatch):+ monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-test")+ monkeypatch.setattr(+ "headroom.learn.analyzer.shutil.which",+ lambda name: f"/usr/bin/{name}" if name == "claude" else None,+ )+ assert _detect_default_model() == "claude-sonnet-4-6"++ def test_env_var_selects_gemini(self, monkeypatch):+ monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)+ monkeypatch.delenv("OPENAI_API_KEY", raising=False)+ monkeypatch.delenv("GEMINI_API_KEY", raising=False)+ monkeypatch.setenv("HEADROOM_LEARN_CLI", "gemini")+ assert _detect_default_model() == "gemini-cli"++ def test_env_var_selects_codex(self, monkeypatch):+ monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)+ monkeypatch.delenv("OPENAI_API_KEY", raising=False)+ monkeypatch.delenv("GEMINI_API_KEY", raising=False)+ monkeypatch.setenv("HEADROOM_LEARN_CLI", "codex")+ assert _detect_default_model() == "codex-cli"++ def test_env_var_invalid_raises(self, monkeypatch):+ monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)+ monkeypatch.delenv("OPENAI_API_KEY", raising=False)+ monkeypatch.delenv("GEMINI_API_KEY", raising=False)+ monkeypatch.setenv("HEADROOM_LEARN_CLI", "unknown-tool")+ with pytest.raises(ValueError, match="not a supported CLI"):+ _detect_default_model()++ def test_api_key_preferred_over_env_var(self, monkeypatch):+ monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-test")+ monkeypatch.setenv("HEADROOM_LEARN_CLI", "gemini")+ assert _detect_default_model() == "claude-sonnet-4-6"++ def test_env_var_preferred_over_auto_detect(self, monkeypatch):+ monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)+ monkeypatch.delenv("OPENAI_API_KEY", raising=False)+ monkeypatch.delenv("GEMINI_API_KEY", raising=False)+ monkeypatch.setenv("HEADROOM_LEARN_CLI", "codex")+ monkeypatch.setattr(+ "headroom.learn.analyzer.shutil.which",+ lambda name: f"/usr/bin/{name}" if name == "claude" else None,+ )+ # codex selected via env var, even though claude is in PATH+ assert _detect_default_model() == "codex-cli"+++# =============================================================================+# CLI LLM Backend+# =============================================================================+++class TestStripFencedJson:+ def test_raw_json(self):+ result = _strip_fenced_json('{"key": "value"}')+ assert result == {"key": "value"}++ def test_fenced_json(self):+ raw = '```json\n{"key": "value"}\n```'+ result = _strip_fenced_json(raw)+ assert result == {"key": "value"}++ def test_fenced_no_language_tag(self):+ raw = '```\n{"key": "value"}\n```'+ result = _strip_fenced_json(raw)+ assert result == {"key": "value"}++ def test_whitespace_padding(self):+ raw = ' \n```json\n{"key": "value"}\n```\n '+ result = _strip_fenced_json(raw)+ assert result == {"key": "value"}++ def test_invalid_json_raises(self):+ with pytest.raises(json.JSONDecodeError):+ _strip_fenced_json("not json at all")+++class TestCallCliLlm:+ @patch("headroom.learn.analyzer.subprocess.run")+ def test_claude_cli_success(self, mock_run: MagicMock):+ mock_run.return_value = MagicMock(+ returncode=0,+ stdout='{"context_file_rules": [], "memory_file_rules": []}',+ stderr="",+ )+ result = _call_cli_llm("test digest", "claude-cli")+ assert result == {"context_file_rules": [], "memory_file_rules": []}+ mock_run.assert_called_once()+ cmd = mock_run.call_args[0][0]+ assert cmd == ["claude", "-p"]+ # Prompt passed via stdin, not as an argument+ assert mock_run.call_args.kwargs.get("input") is not None++ @patch("headroom.learn.analyzer.subprocess.run")+ def test_codex_cli_uses_exec(self, mock_run: MagicMock):+ mock_run.return_value = MagicMock(+ returncode=0,+ stdout='{"context_file_rules": [], "memory_file_rules": []}',+ stderr="",+ )+ result = _call_cli_llm("test digest", "codex-cli")+ assert result == {"context_file_rules": [], "memory_file_rules": []}+ cmd = mock_run.call_args[0][0]+ assert cmd == ["codex", "exec"]++ @patch("headroom.learn.analyzer.subprocess.run")+ def test_gemini_cli_uses_p_flag(self, mock_run: MagicMock):+ mock_run.return_value = MagicMock(+ returncode=0,+ stdout='{"context_file_rules": [], "memory_file_rules": []}',+ stderr="",+ )+ _call_cli_llm("test digest", "gemini-cli")+ cmd = mock_run.call_args[0][0]+ assert cmd == ["gemini", "-p"]++ @patch("headroom.learn.analyzer.subprocess.run")+ def test_cli_nonzero_exit_raises(self, mock_run: MagicMock):+ mock_run.return_value = MagicMock(+ returncode=1,+ stdout="",+ stderr="Error: auth required",+ )+ with pytest.raises(RuntimeError, match="failed.*exit 1"):+ _call_cli_llm("test digest", "claude-cli")++ @patch("headroom.learn.analyzer.subprocess.run")+ def test_cli_stderr_truncated_in_error(self, mock_run: MagicMock):+ long_stderr = "x" * 5000+ mock_run.return_value = MagicMock(+ returncode=1,+ stdout="",+ stderr=long_stderr,+ )+ with pytest.raises(RuntimeError) as exc_info:+ _call_cli_llm("test digest", "claude-cli")+ # Full 5000-char stderr should not appear in the error message+ assert long_stderr not in str(exc_info.value)++ def test_unknown_cli_model_raises(self):+ with pytest.raises(ValueError, match="Unknown CLI model"):+ _call_cli_llm("test digest", "unknown-cli")++ @patch("headroom.learn.analyzer.subprocess.run")+ def test_fenced_output_parsed(self, mock_run: MagicMock):+ mock_run.return_value = MagicMock(+ returncode=0,+ stdout='```json\n{"context_file_rules": [], "memory_file_rules": []}\n```',+ stderr="",+ )+ result = _call_cli_llm("test digest", "claude-cli")+ assert result == {"context_file_rules": [], "memory_file_rules": []}++ @patch("headroom.learn.analyzer.subprocess.run")+ def test_timeout_raises_runtime_error(self, mock_run: MagicMock):+ mock_run.side_effect = subprocess.TimeoutExpired(cmd=["claude", "-p"], timeout=120)+ with pytest.raises(RuntimeError, match="did not respond within"):+ _call_cli_llm("test digest", "claude-cli")++ @patch("headroom.learn.analyzer.subprocess.run")+ def test_unparseable_output_raises_with_context(self, mock_run: MagicMock):+ mock_run.return_value = MagicMock(+ returncode=0,+ stdout="This is not JSON at all",+ stderr="",+ )+ with pytest.raises(RuntimeError, match="unparseable output"):+ _call_cli_llm("test digest", "claude-cli")+++class TestCallLlmRouting:+ @patch("headroom.learn.analyzer._call_cli_llm")+ def test_routes_cli_model_to_cli_backend(self, mock_cli: MagicMock):+ mock_cli.return_value = {"context_file_rules": [], "memory_file_rules": []}+ result = _call_llm("test digest", "claude-cli")+ mock_cli.assert_called_once_with("test digest", "claude-cli")+ assert result == {"context_file_rules": [], "memory_file_rules": []}++ @patch("headroom.learn.analyzer._call_cli_llm")+ def test_routes_codex_cli(self, mock_cli: MagicMock):+ mock_cli.return_value = {}+ _call_llm("digest", "codex-cli")+ mock_cli.assert_called_once_with("digest", "codex-cli")+# =============================================================================# Legacy Compatibility