Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(learn): add CLI-based LLM backends for keyless headroom learn

Gyeonghun Parke98c9751538f
diff --git a/docs/learn.md b/docs/learn.md
index dce072a81..45e762aeb 100644
--- a/docs/learn.md
+++ b/docs/learn.md
@@ -145,9 +145,39 @@ Options:
--project PATH Project directory to analyze (default: current directory)
--all Analyze all discovered projects
--apply Write recommendations (default: dry-run)
- --claude-dir PATH Path to .claude directory (default: ~/.claude)
+ --model TEXT LLM model for analysis (default: auto-detected)
+ --agent TEXT Which coding agent to analyze: auto, claude, codex, gemini
```
+## LLM Backend Selection
+
+`headroom learn` needs an LLM to analyze your sessions. It picks one automatically using this priority:
+
+| Priority | Source | Example |
+|----------|--------|---------|
+| 1 | `--model` flag | `headroom learn --model gpt-4o` |
+| 2 | API key env var | `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY` |
+| 3 | `HEADROOM_LEARN_CLI` env var | `export HEADROOM_LEARN_CLI=gemini` |
+| 4 | Auto-detect installed CLIs | Checks PATH for `claude`, `gemini`, `codex` |
+
+### Using without an API key
+
+If you use Claude Code, Gemini CLI, or Codex via subscription (no raw API key), `headroom learn` can call them directly:
+
+```bash
+# Auto-detects claude in PATH — no API key needed
+headroom learn
+
+# Explicitly select a CLI backend
+headroom learn --model gemini-cli
+
+# Pin a CLI via environment variable
+export HEADROOM_LEARN_CLI=codex
+headroom learn
+```
+
+Valid values for `HEADROOM_LEARN_CLI`: `claude`, `gemini`, `codex`.
+
## Real-World Results
Tested on 67,583 tool calls across 23 projects:
diff --git a/headroom/learn/analyzer.py b/headroom/learn/analyzer.py
index 621e2377a..03761fdb8 100644
--- a/headroom/learn/analyzer.py
+++ b/headroom/learn/analyzer.py
@@ -8,6 +8,8 @@ structured recommendations for CLAUDE.md / MEMORY.md.
Supports any LLM provider via LiteLLM: Anthropic, OpenAI, Google, Bedrock,
Ollama, and 100+ others. Auto-detects the best available model from env vars.
+Also supports CLI-based backends (claude, gemini, codex) for subscription
+users without raw API keys.
"""
from __future__ import annotations
@@ -15,6 +17,8 @@ from __future__ import annotations
import json
import logging
import os
+import shutil
+import subprocess
from .models import (
AnalysisResult,
@@ -37,17 +41,61 @@ _MODEL_DEFAULTS: list[tuple[str, str]] = [
_MAX_DIGEST_TOKENS = 80_000 # Budget for the digest (leave room for prompt + output)
+# CLI tools to try when no API key is set (checked in order).
+# Each entry: (binary_name, model_identifier, command_prefix)
+_CLI_BACKENDS: list[tuple[str, str, list[str]]] = [
+ ("claude", "claude-cli", ["claude", "-p"]),
+ ("gemini", "gemini-cli", ["gemini", "-p"]),
+ ("codex", "codex-cli", ["codex", "exec"]),
+]
+
+# Set of valid CLI model identifiers, derived from _CLI_BACKENDS.
+_CLI_MODEL_IDS: set[str] = {model for _, model, _ in _CLI_BACKENDS}
+
+_USER_PROMPT_PREFIX = "Analyze these coding agent sessions and return JSON recommendations:\n\n" # Shared by _call_cli_llm and _call_llm
+_MAX_SNIPPET_LEN = 2000 # Max chars of CLI output (stdout/stderr) in error messages
+_CLI_TIMEOUT = 120 # Subprocess timeout for CLI backends, in seconds
+
def _detect_default_model() -> str:
- """Pick the best available model based on which API keys are set."""
+ """Pick the best available model based on API keys, env config, or CLI tools.
+
+ Priority order:
+ 1. API key present → use corresponding LiteLLM model
+ 2. HEADROOM_LEARN_CLI env var → use specified CLI backend
+ 3. Auto-detect installed CLI tools (claude > gemini > codex)
+ 4. Raise RuntimeError with setup instructions
+ """
+ # 1. API key detection (existing behavior)
for env_var, model in _MODEL_DEFAULTS:
if os.environ.get(env_var):
return model
+
+ # 2. Explicit CLI selection via environment variable
+ cli_override = os.environ.get("HEADROOM_LEARN_CLI")
+ if cli_override:
+ for cli_name, model, _cmd in _CLI_BACKENDS:
+ if cli_name == cli_override:
+ logger.info("HEADROOM_LEARN_CLI=%s — using %s CLI backend", cli_override, cli_name)
+ return model
+ valid = ", ".join(name for name, _, _ in _CLI_BACKENDS)
+ raise ValueError(
+ f"HEADROOM_LEARN_CLI={cli_override!r} is not a supported CLI. Valid values: {valid}"
+ )
+
+ # 3. Auto-detect installed CLI tools
+ for cli_name, model, _cmd in _CLI_BACKENDS:
+ if shutil.which(cli_name):
+ logger.info("No API key found — auto-detected %s CLI as LLM backend", cli_name)
+ return model
+
raise RuntimeError(
"No LLM API key found. headroom learn needs one of:\n"
" export ANTHROPIC_API_KEY=sk-ant-... → uses claude-sonnet-4-6\n"
" export OPENAI_API_KEY=sk-... → uses gpt-4o\n"
" export GEMINI_API_KEY=... → uses gemini-2.0-flash\n"
+ "Or set HEADROOM_LEARN_CLI to a coding agent CLI (claude, gemini, codex).\n"
+ "Or install one of those CLIs for auto-detection.\n"
"Or specify a model directly: headroom learn --model <litellm-model-name>"
)
@@ -263,12 +311,113 @@ Return ONLY valid JSON matching this schema — no other text:
"""
+def _strip_fenced_json(raw: str) -> dict:
+ """Strip optional markdown fences and parse JSON.
+
+ Handles both raw JSON and fenced code blocks (e.g. ``​`json ... ``​`).
+ Only the first opening fence and last closing fence are removed, preserving
+ any triple-backtick content that may appear inside the JSON payload.
+
+ Args:
+ raw: Raw text output from an LLM, possibly wrapped in markdown fences.
+
+ Returns:
+ Parsed JSON as a dictionary.
+
+ Raises:
+ json.JSONDecodeError: If the text is not valid JSON after stripping.
+ """
+ text = raw.strip()
+ if text.startswith("```"):
+ lines = text.split("\n")
+ # Remove the first line (opening fence, e.g. ```json)
+ lines = lines[1:]
+ # Remove the last line if it is a closing fence
+ if lines and lines[-1].strip().startswith("```"):
+ lines = lines[:-1]
+ text = "\n".join(lines)
+ result: dict = json.loads(text)
+ return result
+
+
+def _call_cli_llm(digest: str, model: str) -> dict:
+ """Call a locally installed CLI tool as the LLM backend.
+
+ Enables keyless usage for subscription-based CLI tools that handle
+ their own OAuth authentication. The prompt is passed via stdin to avoid
+ OS ``ARG_MAX`` limits and argument-injection risks.
+
+ CLI invocations:
+ claude-cli → echo <prompt> | claude -p
+ gemini-cli → echo <prompt> | gemini -p
+ codex-cli → echo <prompt> | codex exec
+
+ Args:
+ digest: Token-efficient session digest to analyze.
+ model: CLI model identifier (e.g. ``claude-cli``).
+
+ Returns:
+ Parsed JSON recommendations from the CLI tool.
+
+ Raises:
+ ValueError: If *model* is not a known CLI backend.
+ RuntimeError: If the CLI exits with a non-zero code or times out.
+ """
+ cmd: list[str] | None = None
+ for _name, model_name, cmd_parts in _CLI_BACKENDS:
+ if model_name == model:
+ cmd = cmd_parts
+ break
+ if cmd is None:
+ raise ValueError(f"Unknown CLI model: {model}")
+
+ prompt = _SYSTEM_PROMPT + "\n\n" + _USER_PROMPT_PREFIX + digest
+
+ try:
+ result = subprocess.run(
+ cmd,
+ input=prompt,
+ capture_output=True,
+ text=True,
+ timeout=_CLI_TIMEOUT,
+ )
+ except subprocess.TimeoutExpired:
+ raise RuntimeError(
+ f"`{' '.join(cmd)}` did not respond within {_CLI_TIMEOUT}s. "
+ "Check network connectivity or try a different backend with "
+ "--model <litellm-model-name>."
+ ) from None
+
+ if result.returncode != 0:
+ stderr_snippet = (result.stderr or "")[:_MAX_SNIPPET_LEN]
+ raise RuntimeError(
+ f"`{' '.join(cmd)}` failed (exit {result.returncode}):\n{stderr_snippet}"
+ )
+
+ # Log stderr warnings even on success (auth refreshes, deprecation notices).
+ if result.stderr and result.stderr.strip():
+ logger.debug("CLI stderr (exit 0): %s", result.stderr[:_MAX_SNIPPET_LEN])
+
+ try:
+ return _strip_fenced_json(result.stdout)
+ except json.JSONDecodeError as exc:
+ stdout_snippet = (result.stdout or "")[:_MAX_SNIPPET_LEN]
+ raise RuntimeError(
+ f"`{' '.join(cmd)}` returned unparseable output. "
+ f"First {_MAX_SNIPPET_LEN} chars:\n{stdout_snippet}"
+ ) from exc
+
+
def _call_llm(digest: str, model: str) -> dict:
"""Call LLM with the session digest and return parsed JSON.
Uses LiteLLM for provider-agnostic access. The model string determines
the provider: "claude-*" → Anthropic, "gpt-*" → OpenAI, "gemini/*" → Google, etc.
+ For CLI-based models (ending in "-cli"), delegates to ``_call_cli_llm``.
"""
+ if model in _CLI_MODEL_IDS:
+ return _call_cli_llm(digest, model)
+
import litellm
# Suppress LiteLLM's verbose logging
@@ -286,10 +435,7 @@ def _call_llm(digest: str, model: str) -> dict:
{"role": "system", "content": _SYSTEM_PROMPT},
{
"role": "user",
- "content": (
- "Analyze these coding agent sessions and return JSON recommendations:\n\n"
- + digest
- ),
+ "content": _USER_PROMPT_PREFIX + digest,
},
],
max_tokens=4096,
@@ -298,16 +444,7 @@ def _call_llm(digest: str, model: str) -> dict:
# Extract text from response
text = response.choices[0].message.content or ""
-
- # Parse JSON — handle both raw JSON and ```json fenced blocks
- text = text.strip()
- if text.startswith("```"):
- lines = text.split("\n")
- lines = [ln for ln in lines[1:] if not ln.strip().startswith("```")]
- text = "\n".join(lines)
-
- result: dict = json.loads(text)
- return result
+ return _strip_fenced_json(text)
# =============================================================================
diff --git a/tests/test_learn/test_analyzer.py b/tests/test_learn/test_analyzer.py
index 30b344d76..4c1225d0d 100644
--- a/tests/test_learn/test_analyzer.py
+++ b/tests/test_learn/test_analyzer.py
@@ -1,13 +1,20 @@
"""Tests for session analyzer — digest builder and LLM-based analysis."""
+import json
+import subprocess
from pathlib import Path
from unittest.mock import MagicMock, patch
+import pytest
+
from headroom.learn.analyzer import (
SessionAnalyzer,
_build_digest,
+ _call_cli_llm,
+ _call_llm,
_detect_default_model,
_parse_llm_response,
+ _strip_fenced_json,
)
from headroom.learn.models import (
AnalysisResult,
@@ -347,15 +354,230 @@ class TestDetectDefaultModel:
monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
assert _detect_default_model() == "claude-sonnet-4-6"
- def test_no_keys_raises(self, monkeypatch):
+ def test_no_keys_no_cli_raises(self, monkeypatch):
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
monkeypatch.delenv("GEMINI_API_KEY", raising=False)
- import pytest
+ monkeypatch.setattr("headroom.learn.analyzer.shutil.which", lambda _name: None)
with pytest.raises(RuntimeError, match="No LLM API key found"):
_detect_default_model()
+ def test_cli_fallback_claude(self, monkeypatch):
+ monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
+ monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+ monkeypatch.delenv("GEMINI_API_KEY", raising=False)
+ monkeypatch.setattr(
+ "headroom.learn.analyzer.shutil.which",
+ lambda name: f"/usr/bin/{name}" if name == "claude" else None,
+ )
+ assert _detect_default_model() == "claude-cli"
+
+ def test_cli_fallback_gemini(self, monkeypatch):
+ monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
+ monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+ monkeypatch.delenv("GEMINI_API_KEY", raising=False)
+ monkeypatch.setattr(
+ "headroom.learn.analyzer.shutil.which",
+ lambda name: f"/usr/bin/{name}" if name == "gemini" else None,
+ )
+ assert _detect_default_model() == "gemini-cli"
+
+ def test_cli_fallback_codex(self, monkeypatch):
+ monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
+ monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+ monkeypatch.delenv("GEMINI_API_KEY", raising=False)
+ monkeypatch.setattr(
+ "headroom.learn.analyzer.shutil.which",
+ lambda name: f"/usr/bin/{name}" if name == "codex" else None,
+ )
+ assert _detect_default_model() == "codex-cli"
+
+ def test_api_key_preferred_over_cli(self, monkeypatch):
+ monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-test")
+ monkeypatch.setattr(
+ "headroom.learn.analyzer.shutil.which",
+ lambda name: f"/usr/bin/{name}" if name == "claude" else None,
+ )
+ assert _detect_default_model() == "claude-sonnet-4-6"
+
+ def test_env_var_selects_gemini(self, monkeypatch):
+ monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
+ monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+ monkeypatch.delenv("GEMINI_API_KEY", raising=False)
+ monkeypatch.setenv("HEADROOM_LEARN_CLI", "gemini")
+ assert _detect_default_model() == "gemini-cli"
+
+ def test_env_var_selects_codex(self, monkeypatch):
+ monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
+ monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+ monkeypatch.delenv("GEMINI_API_KEY", raising=False)
+ monkeypatch.setenv("HEADROOM_LEARN_CLI", "codex")
+ assert _detect_default_model() == "codex-cli"
+
+ def test_env_var_invalid_raises(self, monkeypatch):
+ monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
+ monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+ monkeypatch.delenv("GEMINI_API_KEY", raising=False)
+ monkeypatch.setenv("HEADROOM_LEARN_CLI", "unknown-tool")
+ with pytest.raises(ValueError, match="not a supported CLI"):
+ _detect_default_model()
+
+ def test_api_key_preferred_over_env_var(self, monkeypatch):
+ monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-test")
+ monkeypatch.setenv("HEADROOM_LEARN_CLI", "gemini")
+ assert _detect_default_model() == "claude-sonnet-4-6"
+
+ def test_env_var_preferred_over_auto_detect(self, monkeypatch):
+ monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
+ monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+ monkeypatch.delenv("GEMINI_API_KEY", raising=False)
+ monkeypatch.setenv("HEADROOM_LEARN_CLI", "codex")
+ monkeypatch.setattr(
+ "headroom.learn.analyzer.shutil.which",
+ lambda name: f"/usr/bin/{name}" if name == "claude" else None,
+ )
+ # codex selected via env var, even though claude is in PATH
+ assert _detect_default_model() == "codex-cli"
+
+
+# =============================================================================
+# CLI LLM Backend
+# =============================================================================
+
+
+class TestStripFencedJson:
+ def test_raw_json(self):
+ result = _strip_fenced_json('{"key": "value"}')
+ assert result == {"key": "value"}
+
+ def test_fenced_json(self):
+ raw = '```json\n{"key": "value"}\n```'
+ result = _strip_fenced_json(raw)
+ assert result == {"key": "value"}
+
+ def test_fenced_no_language_tag(self):
+ raw = '```\n{"key": "value"}\n```'
+ result = _strip_fenced_json(raw)
+ assert result == {"key": "value"}
+
+ def test_whitespace_padding(self):
+ raw = ' \n```json\n{"key": "value"}\n```\n '
+ result = _strip_fenced_json(raw)
+ assert result == {"key": "value"}
+
+ def test_invalid_json_raises(self):
+ with pytest.raises(json.JSONDecodeError):
+ _strip_fenced_json("not json at all")
+
+
+class TestCallCliLlm:
+ @patch("headroom.learn.analyzer.subprocess.run")
+ def test_claude_cli_success(self, mock_run: MagicMock):
+ mock_run.return_value = MagicMock(
+ returncode=0,
+ stdout='{"context_file_rules": [], "memory_file_rules": []}',
+ stderr="",
+ )
+ result = _call_cli_llm("test digest", "claude-cli")
+ assert result == {"context_file_rules": [], "memory_file_rules": []}
+ mock_run.assert_called_once()
+ cmd = mock_run.call_args[0][0]
+ assert cmd == ["claude", "-p"]
+ # Prompt passed via stdin, not as an argument
+ assert mock_run.call_args.kwargs.get("input") is not None
+
+ @patch("headroom.learn.analyzer.subprocess.run")
+ def test_codex_cli_uses_exec(self, mock_run: MagicMock):
+ mock_run.return_value = MagicMock(
+ returncode=0,
+ stdout='{"context_file_rules": [], "memory_file_rules": []}',
+ stderr="",
+ )
+ result = _call_cli_llm("test digest", "codex-cli")
+ assert result == {"context_file_rules": [], "memory_file_rules": []}
+ cmd = mock_run.call_args[0][0]
+ assert cmd == ["codex", "exec"]
+
+ @patch("headroom.learn.analyzer.subprocess.run")
+ def test_gemini_cli_uses_p_flag(self, mock_run: MagicMock):
+ mock_run.return_value = MagicMock(
+ returncode=0,
+ stdout='{"context_file_rules": [], "memory_file_rules": []}',
+ stderr="",
+ )
+ _call_cli_llm("test digest", "gemini-cli")
+ cmd = mock_run.call_args[0][0]
+ assert cmd == ["gemini", "-p"]
+
+ @patch("headroom.learn.analyzer.subprocess.run")
+ def test_cli_nonzero_exit_raises(self, mock_run: MagicMock):
+ mock_run.return_value = MagicMock(
+ returncode=1,
+ stdout="",
+ stderr="Error: auth required",
+ )
+ with pytest.raises(RuntimeError, match="failed.*exit 1"):
+ _call_cli_llm("test digest", "claude-cli")
+
+ @patch("headroom.learn.analyzer.subprocess.run")
+ def test_cli_stderr_truncated_in_error(self, mock_run: MagicMock):
+ long_stderr = "x" * 5000
+ mock_run.return_value = MagicMock(
+ returncode=1,
+ stdout="",
+ stderr=long_stderr,
+ )
+ with pytest.raises(RuntimeError) as exc_info:
+ _call_cli_llm("test digest", "claude-cli")
+ # Full 5000-char stderr should not appear in the error message
+ assert long_stderr not in str(exc_info.value)
+
+ def test_unknown_cli_model_raises(self):
+ with pytest.raises(ValueError, match="Unknown CLI model"):
+ _call_cli_llm("test digest", "unknown-cli")
+
+ @patch("headroom.learn.analyzer.subprocess.run")
+ def test_fenced_output_parsed(self, mock_run: MagicMock):
+ mock_run.return_value = MagicMock(
+ returncode=0,
+ stdout='```json\n{"context_file_rules": [], "memory_file_rules": []}\n```',
+ stderr="",
+ )
+ result = _call_cli_llm("test digest", "claude-cli")
+ assert result == {"context_file_rules": [], "memory_file_rules": []}
+
+ @patch("headroom.learn.analyzer.subprocess.run")
+ def test_timeout_raises_runtime_error(self, mock_run: MagicMock):
+ mock_run.side_effect = subprocess.TimeoutExpired(cmd=["claude", "-p"], timeout=120)
+ with pytest.raises(RuntimeError, match="did not respond within"):
+ _call_cli_llm("test digest", "claude-cli")
+
+ @patch("headroom.learn.analyzer.subprocess.run")
+ def test_unparseable_output_raises_with_context(self, mock_run: MagicMock):
+ mock_run.return_value = MagicMock(
+ returncode=0,
+ stdout="This is not JSON at all",
+ stderr="",
+ )
+ with pytest.raises(RuntimeError, match="unparseable output"):
+ _call_cli_llm("test digest", "claude-cli")
+
+
+class TestCallLlmRouting:
+ @patch("headroom.learn.analyzer._call_cli_llm")
+ def test_routes_cli_model_to_cli_backend(self, mock_cli: MagicMock):
+ mock_cli.return_value = {"context_file_rules": [], "memory_file_rules": []}
+ result = _call_llm("test digest", "claude-cli")
+ mock_cli.assert_called_once_with("test digest", "claude-cli")
+ assert result == {"context_file_rules": [], "memory_file_rules": []}
+
+ @patch("headroom.learn.analyzer._call_cli_llm")
+ def test_routes_codex_cli(self, mock_cli: MagicMock):
+ mock_cli.return_value = {}
+ _call_llm("digest", "codex-cli")
+ mock_cli.assert_called_once_with("digest", "codex-cli")
+
# =============================================================================
# Legacy Compatibility