Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

Add multi-agent support, quality gates, and integration tests for headroom learn

chopratejas7cf086c2e89f
diff --git a/headroom/cli/learn.py b/headroom/cli/learn.py
index 8671f9fbc..a1d8f0902 100644
--- a/headroom/cli/learn.py
+++ b/headroom/cli/learn.py
@@ -3,11 +3,65 @@
from __future__ import annotations
from pathlib import Path
+from typing import TYPE_CHECKING
import click
+if TYPE_CHECKING:
+ from ..learn.scanner import ConversationScanner
+ from ..learn.writer import ContextWriter
+
from .main import main
+_AGENT_HELP = """Which coding agent to analyze. Auto-detects by default.
+
+\b
+Supported agents:
+ claude Claude Code (~/.claude/)
+ codex OpenAI Codex CLI (~/.codex/)
+ gemini Google Gemini CLI (~/.gemini/)
+ auto Auto-detect (check all, default)
+"""
+
+
+def _get_scanner_writer(agent: str) -> tuple[ConversationScanner, ContextWriter]:
+ """Get the appropriate scanner and writer for an agent type."""
+ from ..learn.scanner import ClaudeCodeScanner, CodexScanner
+ from ..learn.writer import ClaudeCodeWriter, CodexWriter
+
+ scanners = {
+ "claude": (ClaudeCodeScanner, ClaudeCodeWriter),
+ "codex": (CodexScanner, CodexWriter),
+ # Gemini scanner not yet implemented (protobuf sessions)
+ # Cursor scanner not yet implemented (SQLite blobs)
+ }
+
+ if agent in scanners:
+ scanner_cls, writer_cls = scanners[agent]
+ return scanner_cls(), writer_cls()
+
+ raise click.BadParameter(f"Unknown agent: {agent}. Supported: claude, codex")
+
+
+def _auto_detect_agents() -> list[tuple[str, ConversationScanner, ContextWriter]]:
+ """Auto-detect which agents have data on this machine."""
+ from ..learn.scanner import ClaudeCodeScanner, CodexScanner
+ from ..learn.writer import ClaudeCodeWriter, CodexWriter
+
+ agents: list[tuple[str, ConversationScanner, ContextWriter]] = []
+
+ # Claude Code
+ claude_dir = Path.home() / ".claude" / "projects"
+ if claude_dir.exists() and any(claude_dir.iterdir()):
+ agents.append(("claude", ClaudeCodeScanner(), ClaudeCodeWriter()))
+
+ # Codex
+ codex_dir = Path.home() / ".codex" / "sessions"
+ if codex_dir.exists() and any(codex_dir.glob("*.json")):
+ agents.append(("codex", CodexScanner(), CodexWriter()))
+
+ return agents
+
@main.command()
@click.option(
@@ -27,19 +81,19 @@ from .main import main
"--apply",
is_flag=True,
default=False,
- help="Write recommendations to CLAUDE.md / MEMORY.md (default: dry-run).",
+ help="Write recommendations to context/memory files (default: dry-run).",
)
@click.option(
- "--claude-dir",
- type=click.Path(path_type=Path),
- default=None,
- help="Path to .claude directory. Defaults to ~/.claude.",
+ "--agent",
+ type=click.Choice(["auto", "claude", "codex", "gemini"], case_sensitive=False),
+ default="auto",
+ help=_AGENT_HELP,
)
def learn(
project: Path | None,
analyze_all: bool,
apply: bool,
- claude_dir: Path | None,
+ agent: str,
) -> None:
"""Learn from past tool call failures to prevent future ones.
@@ -47,102 +101,120 @@ def learn(
missing modules, stubborn retries) and generates context that prevents
them from recurring.
+ Supports multiple coding agents: Claude Code, Codex, Gemini CLI.
+ Auto-detects which agents have data on your machine by default.
+
\b
Examples:
- headroom learn # Analyze current project (dry-run)
- headroom learn --apply # Write recommendations
- headroom learn --all # Analyze all projects
- headroom learn --project ~/myapp # Analyze specific project
+ headroom learn # Auto-detect agent, analyze current project
+ headroom learn --apply # Write recommendations
+ headroom learn --all # Analyze all projects
+ headroom learn --agent codex --all # Analyze all Codex sessions
+ headroom learn --agent claude --project ~/myapp
"""
from ..learn.analyzer import FailureAnalyzer
- from ..learn.scanner import ClaudeCodeScanner
- from ..learn.writer import ClaudeCodeWriter, Recommender
- scanner = ClaudeCodeScanner(claude_dir=claude_dir)
analyzer = FailureAnalyzer()
- recommender = Recommender()
- writer = ClaudeCodeWriter()
-
- # Discover projects
- all_projects = scanner.discover_projects()
-
- if not all_projects:
- click.echo("No projects found in ~/.claude/projects/")
- return
-
- # Filter to target project(s)
- if analyze_all:
- targets = all_projects
- elif project:
- resolved = project.resolve()
- targets = [p for p in all_projects if p.project_path == resolved]
- if not targets:
- click.echo(f"Project not found: {resolved}")
- click.echo(f"Available projects: {', '.join(p.name for p in all_projects)}")
+
+ # Determine which agents to scan
+ if agent == "auto":
+ agent_configs = _auto_detect_agents()
+ if not agent_configs:
+ click.echo("No coding agent data found. Checked: ~/.claude/, ~/.codex/")
return
+ click.echo(f"Detected agents: {', '.join(name for name, _, _ in agent_configs)}")
else:
- # Auto-detect from cwd
- cwd = Path.cwd().resolve()
- targets = [p for p in all_projects if p.project_path == cwd]
- if not targets:
- # Try parent directories
- for parent in cwd.parents:
- targets = [p for p in all_projects if p.project_path == parent]
- if targets:
- break
- if not targets:
- click.echo(f"No project data found for {cwd}")
- click.echo("Try: headroom learn --project <path> or headroom learn --all")
- click.echo("\nAvailable projects:")
- for p in all_projects[:10]:
- click.echo(f" {p.name:30s} {p.project_path}")
- return
-
- # Analyze each target
- for proj in targets:
- click.echo(f"\n{'=' * 60}")
- click.echo(f"Project: {proj.name}")
- click.echo(f"Path: {proj.project_path}")
- click.echo(f"{'=' * 60}")
-
- sessions = scanner.scan_project(proj)
- if not sessions:
- click.echo(" No conversation data found.")
- continue
-
- report = analyzer.analyze(proj, sessions)
+ scanner, writer = _get_scanner_writer(agent)
+ agent_configs = [(agent, scanner, writer)]
- # Print summary
- click.echo(f"\n Sessions analyzed: {report.total_sessions}")
- click.echo(f" Total tool calls: {report.total_calls}")
- click.echo(f" Failed calls: {report.total_failures} ({report.failure_rate:.1%})")
- click.echo(f" Waste bytes: {report.waste_bytes / 1024:.0f} KB")
+ total_projects = 0
+ total_failures = 0
+ total_recommendations = 0
- if report.failure_rate == 0:
- click.echo("\n No failures found. Nothing to learn.")
+ for agent_name, scanner, writer in agent_configs:
+ all_projects = scanner.discover_projects()
+ if not all_projects:
continue
- # Generate recommendations
- recommendations = recommender.recommend(report)
-
- if not recommendations:
- click.echo("\n No actionable patterns found.")
- continue
-
- click.echo(f"\n Recommendations: {len(recommendations)}")
-
- # Write (or dry-run)
- result = writer.write(recommendations, proj, dry_run=not apply)
-
- for file_path, content in result.content_by_file.items():
- click.echo(f"\n {'[WOULD WRITE]' if result.dry_run else '[WROTE]'} {file_path}")
- click.echo(f" {'─' * 50}")
- # Show content preview (indented)
- for line in content.split("\n"):
- if line.startswith("<!-- headroom"):
- continue # Skip markers in display
- click.echo(f" {line}")
- click.echo(f" {'─' * 50}")
-
- if result.dry_run:
- click.echo("\n Dry run — no files modified. Use --apply to write.")
+ # Filter to target project(s)
+ if analyze_all:
+ targets = all_projects
+ elif project:
+ resolved = project.resolve()
+ targets = [p for p in all_projects if p.project_path == resolved]
+ if not targets:
+ continue
+ else:
+ cwd = Path.cwd().resolve()
+ targets = [p for p in all_projects if p.project_path == cwd]
+ if not targets:
+ for parent in cwd.parents:
+ targets = [p for p in all_projects if p.project_path == parent]
+ if targets:
+ break
+ if not targets and len(agent_configs) == 1:
+ click.echo(f"No {agent_name} project data found for {cwd}")
+ click.echo("Try: headroom learn --all or headroom learn --project <path>")
+ click.echo(f"\nAvailable {agent_name} projects:")
+ for p in all_projects[:10]:
+ click.echo(f" {p.name:30s} {p.project_path}")
+ return
+
+ for proj in targets:
+ click.echo(f"\n{'=' * 60}")
+ click.echo(f"[{agent_name}] {proj.name}")
+ click.echo(f"Path: {proj.project_path}")
+ click.echo(f"{'=' * 60}")
+
+ sessions = scanner.scan_project(proj)
+ if not sessions:
+ click.echo(" No conversation data found.")
+ continue
+
+ from ..learn.writer import Recommender
+
+ recommender = Recommender()
+ report = analyzer.analyze(proj, sessions)
+ total_projects += 1
+ total_failures += report.total_failures
+
+ click.echo(
+ f"\n Sessions: {report.total_sessions} | "
+ f"Calls: {report.total_calls} | "
+ f"Failures: {report.total_failures} ({report.failure_rate:.1%}) | "
+ f"Corrections: {len(report.corrections)}"
+ )
+
+ if report.failure_rate == 0:
+ click.echo(" No failures found.")
+ continue
+
+ recommendations = recommender.recommend(report)
+ if not recommendations:
+ click.echo(" No actionable patterns found.")
+ continue
+
+ total_recommendations += len(recommendations)
+ click.echo(f" Recommendations: {len(recommendations)}")
+
+ result = writer.write(recommendations, proj, dry_run=not apply)
+
+ for file_path, content in result.content_by_file.items():
+ click.echo(f"\n {'[WOULD WRITE]' if result.dry_run else '[WROTE]'} {file_path}")
+ click.echo(f" {'─' * 50}")
+ for line in content.split("\n"):
+ if line.startswith("<!-- headroom"):
+ continue
+ click.echo(f" {line}")
+ click.echo(f" {'─' * 50}")
+
+ if result.dry_run:
+ click.echo("\n Dry run — use --apply to write.")
+
+ # Summary
+ if total_projects > 1:
+ click.echo(f"\n{'=' * 60}")
+ click.echo(
+ f"Total: {total_projects} projects, {total_failures} failures, "
+ f"{total_recommendations} recommendations"
+ )
diff --git a/headroom/learn/analyzer.py b/headroom/learn/analyzer.py
index f793b6bb3..b8f47d644 100644
--- a/headroom/learn/analyzer.py
+++ b/headroom/learn/analyzer.py
@@ -171,12 +171,17 @@ def _analyze_structure(
# 1. Path corrections: wrong path → correct path (from success correlation)
path_corrections: dict[str, Counter[str]] = defaultdict(Counter)
for c in corrections:
- if c.tool_name not in ("Read", "read"):
- continue
if c.error_category != ErrorCategory.FILE_NOT_FOUND:
continue
- failed_path = c.failed_input.get("file_path", "")
- success_path = c.success_input.get("file_path", "")
+ # Extract file paths from Read tool or from Bash commands (Codex uses shell for everything)
+ if c.tool_name in ("Read", "read"):
+ failed_path = c.failed_input.get("file_path", "")
+ success_path = c.success_input.get("file_path", "")
+ elif c.tool_name in ("Bash", "bash"):
+ failed_path = _extract_path_from_command(c.failed_input.get("command", ""))
+ success_path = _extract_path_from_command(c.success_input.get("command", ""))
+ else:
+ continue
if failed_path and success_path and failed_path != success_path:
path_corrections[failed_path][success_path] += 1
@@ -585,6 +590,7 @@ def _analyze_cross_session(sessions: list[SessionData]) -> list[str]:
_PYTHON_CMD_RE = re.compile(
r"^((?:source\s+\S+\s*&&\s*)?(?:\.venv/bin/)?(?:python3?|uv run python|uv run|/opt/nflx/python))"
+ r"(?:\s|$|-)" # Must be followed by space, end, or dash (python3 -c, python -)
)
@@ -610,6 +616,22 @@ def _extract_command_signature(cmd: str) -> str:
return cmd[:60]
+def _extract_path_from_command(cmd: str) -> str:
+ """Extract a file path from a shell command (cat, sed, head, etc.)."""
+ cmd = cmd.strip()
+ # Look for path-like tokens (containing / and ending in a file extension)
+ path_re = re.compile(r"""(?:^|[\s'"])(/[^\s'"]+\.\w+)""")
+ match = path_re.search(cmd)
+ if match:
+ return match.group(1)
+ # Also try: last token that looks like a path
+ tokens = cmd.split()
+ for token in reversed(tokens):
+ if "/" in token and not token.startswith("-"):
+ return token.strip("'\"")
+ return ""
+
+
def _shorten_path(path: str) -> str:
"""Make a path relative to home for readability."""
home = os.path.expanduser("~")
diff --git a/headroom/learn/scanner.py b/headroom/learn/scanner.py
index 6ab8e9acc..90ed7508c 100644
--- a/headroom/learn/scanner.py
+++ b/headroom/learn/scanner.py
@@ -356,3 +356,152 @@ def _greedy_path_decode(base: Path, parts: list[str]) -> Path | None:
# If we've exhausted parts, check if current path exists
return base if base.exists() else None
+
+
+# =============================================================================
+# Codex Scanner (OpenAI Codex CLI)
+# =============================================================================
+
+
+class CodexScanner(ConversationScanner):
+ """Reads OpenAI Codex CLI session logs from ~/.codex/sessions/.
+
+ Codex stores sessions as JSON files with:
+ - session.id, session.timestamp, session.instructions
+ - items[]: array of message/function_call/function_call_output/reasoning objects
+
+ function_call items have: name, call_id, arguments (JSON string)
+ function_call_output items have: call_id, output (string or JSON string)
+ """
+
+ def __init__(self, codex_dir: Path | None = None):
+ self.codex_dir = codex_dir or Path.home() / ".codex"
+ self.sessions_dir = self.codex_dir / "sessions"
+
+ def discover_projects(self) -> list[ProjectInfo]:
+ """Codex doesn't organize by project — return a single 'codex' project.
+
+ Codex sessions aren't scoped to projects. We treat all sessions as one
+ project and use the cwd from session data (if available) to group later.
+ """
+ if not self.sessions_dir.exists():
+ return []
+
+ session_files = list(self.sessions_dir.glob("*.json"))
+ if not session_files:
+ return []
+
+ # Check for global AGENTS.md
+ agents_md = self.codex_dir / "AGENTS.md"
+ instructions_md = self.codex_dir / "instructions.md"
+
+ return [
+ ProjectInfo(
+ name="codex",
+ project_path=Path.cwd(), # Codex doesn't track project paths
+ data_path=self.sessions_dir,
+ context_file=agents_md if agents_md.exists() else None,
+ memory_file=instructions_md if instructions_md.exists() else None,
+ )
+ ]
+
+ def scan_project(self, project: ProjectInfo) -> list[SessionData]:
+ """Scan all Codex session JSON files."""
+ sessions = []
+ for json_path in sorted(project.data_path.glob("*.json")):
+ session = self._scan_session(json_path)
+ if session and session.tool_calls:
+ sessions.append(session)
+ return sessions
+
+ def _scan_session(self, json_path: Path) -> SessionData | None:
+ """Parse a single Codex session file."""
+ try:
+ with open(json_path) as f:
+ data = json.load(f)
+ except (OSError, json.JSONDecodeError) as e:
+ logger.debug("Failed to read Codex session %s: %s", json_path, e)
+ return None
+
+ session_info = data.get("session", {})
+ session_id = session_info.get("id", json_path.stem)
+ items = data.get("items", [])
+
+ if not items:
+ return None
+
+ # Build call_id → (name, input) map from function_call items
+ func_calls: dict[str, tuple[str, dict]] = {}
+ tool_calls: list[ToolCall] = []
+ msg_index = 0
+
+ for item in items:
+ msg_index += 1
+ item_type = item.get("type", "")
+
+ if item_type == "function_call":
+ call_id = item.get("call_id", "")
+ name = item.get("name", "")
+ # Parse arguments (JSON string or list)
+ raw_args = item.get("arguments", "")
+ if isinstance(raw_args, str):
+ try:
+ parsed = json.loads(raw_args)
+ except (json.JSONDecodeError, TypeError):
+ parsed = {"raw": raw_args}
+ elif isinstance(raw_args, dict):
+ parsed = raw_args
+ else:
+ parsed = {"raw": str(raw_args)}
+
+ # Codex uses "shell" as the tool name with command in args
+ # Normalize: extract command for consistency with other scanners
+ if name == "shell" and "command" in parsed:
+ cmd = parsed["command"]
+ if isinstance(cmd, list):
+ # Codex passes commands as ["bash", "-lc", "actual command"]
+ parsed["command"] = cmd[-1] if cmd else ""
+ name = "Bash" # Normalize to match Claude Code tool names
+
+ if call_id:
+ func_calls[call_id] = (name, parsed)
+
+ elif item_type == "function_call_output":
+ call_id = item.get("call_id", "")
+ output_raw = item.get("output", "")
+
+ if call_id not in func_calls:
+ continue
+
+ name, inp = func_calls[call_id]
+
+ # Output may be JSON string with "output" field
+ if isinstance(output_raw, str):
+ try:
+ parsed_out = json.loads(output_raw)
+ if isinstance(parsed_out, dict) and "output" in parsed_out:
+ result_content = str(parsed_out["output"])
+ else:
+ result_content = output_raw
+ except (json.JSONDecodeError, TypeError):
+ result_content = output_raw
+ else:
+ result_content = str(output_raw)
+
+ is_err = is_error_content(result_content)
+ error_cat = classify_error(result_content) if is_err else ErrorCategory.UNKNOWN
+
+ tool_calls.append(
+ ToolCall(
+ name=name,
+ tool_call_id=call_id,
+ input_data=inp,
+ output=result_content,
+ is_error=is_err,
+ error_category=error_cat,
+ msg_index=msg_index,
+ output_bytes=len(result_content.encode("utf-8")),
+ )
+ )
+
+ return SessionData(session_id=session_id, tool_calls=tool_calls)
diff --git a/headroom/learn/writer.py b/headroom/learn/writer.py
index 7c4fa4ee8..414369acf 100644
--- a/headroom/learn/writer.py
+++ b/headroom/learn/writer.py
@@ -37,7 +37,23 @@ _MARKER_PATTERN = re.compile(
class Recommender:
- """Converts an AnalysisReport into concrete markdown recommendations."""
+ """Converts an AnalysisReport into concrete markdown recommendations.
+
+ Quality gates:
+ - Each recommendation needs min_evidence failures to be included
+ - Total recommendations need min_total_evidence to trigger any file writes
+ - Low-confidence recommendations are filtered out
+ """
+
+ def __init__(
+ self,
+ min_evidence: int = 2,
+ min_confidence: float = 0.3,
+ min_total_evidence: int = 3,
+ ):
+ self.min_evidence = min_evidence
+ self.min_confidence = min_confidence
+ self.min_total_evidence = min_total_evidence
def recommend(self, report: AnalysisReport) -> list[Recommendation]:
recommendations: list[Recommendation] = []
@@ -152,6 +168,18 @@ class Recommender:
)
)
+ # Quality gate: filter out weak recommendations
+ recommendations = [
+ r
+ for r in recommendations
+ if r.evidence_count >= self.min_evidence and r.confidence >= self.min_confidence
+ ]
+
+ # Quality gate: if total evidence is too low, don't recommend anything
+ total_evidence = sum(r.evidence_count for r in recommendations)
+ if total_evidence < self.min_total_evidence:
+ return []
+
return recommendations
def _format_environment(self, facts: list[EnvironmentFact]) -> str:
@@ -331,3 +359,131 @@ class ClaudeCodeWriter(ContextWriter):
return existing.rstrip() + "\n\n" + section + "\n"
else:
return section + "\n"
+
+
+# =============================================================================
+# Codex Writer (OpenAI Codex CLI)
+# =============================================================================
+
+
+class CodexWriter(ContextWriter):
+ """Writes learned patterns to AGENTS.md and instructions.md for Codex CLI.
+
+ Codex reads AGENTS.md from project root and ~/.codex/ for instructions.
+ instructions.md in ~/.codex/ acts as persistent cross-session memory.
+ """
+
+ def write(
+ self,
+ recommendations: list[Recommendation],
+ project: ProjectInfo,
+ dry_run: bool = True,
+ ) -> WriteResult:
+ result = WriteResult()
+ result.dry_run = dry_run
+
+ context_recs = [r for r in recommendations if r.target == RecommendationTarget.CONTEXT_FILE]
+ memory_recs = [r for r in recommendations if r.target == RecommendationTarget.MEMORY_FILE]
+
+ if context_recs:
+ agents_md = project.context_file or (project.project_path / "AGENTS.md")
+ section_content = self._build_section(context_recs)
+ full_content = self._merge_into_file(agents_md, section_content)
+ result.add(agents_md, full_content)
+ if not dry_run:
+ agents_md.parent.mkdir(parents=True, exist_ok=True)
+ agents_md.write_text(full_content)
+
+ if memory_recs:
+ instructions_md = project.memory_file or (project.data_path.parent / "instructions.md")
+ section_content = self._build_section(memory_recs)
+ full_content = self._merge_into_file(instructions_md, section_content)
+ result.add(instructions_md, full_content)
+ if not dry_run:
+ instructions_md.parent.mkdir(parents=True, exist_ok=True)
+ instructions_md.write_text(full_content)
+
+ return result
+
+ def _build_section(self, recommendations: list[Recommendation]) -> str:
+ now = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+ lines = [
+ _MARKER_START,
+ "## Headroom Learned Patterns",
+ f"*Auto-generated by `headroom learn` on {now} — do not edit manually*",
+ "",
+ ]
+ for rec in recommendations:
+ lines.append(f"### {rec.section}")
+ lines.append(rec.content)
+ lines.append("")
+ lines.append(_MARKER_END)
+ return "\n".join(lines)
+
+ def _merge_into_file(self, file_path: Path, section: str) -> str:
+ if file_path.exists():
+ existing = file_path.read_text()
+ if _MARKER_START in existing:
+ return _MARKER_PATTERN.sub(section, existing)
+ return existing.rstrip() + "\n\n" + section + "\n"
+ return section + "\n"
+
+
+# =============================================================================
+# Gemini Writer (Google Gemini CLI)
+# =============================================================================
+
+
+class GeminiWriter(ContextWriter):
+ """Writes learned patterns to GEMINI.md for Gemini CLI.
+
+ Gemini reads GEMINI.md from project root and ~/.gemini/ for persistent context.
+ No separate memory file — everything goes into GEMINI.md.
+ """
+
+ def write(
+ self,
+ recommendations: list[Recommendation],
+ project: ProjectInfo,
+ dry_run: bool = True,
+ ) -> WriteResult:
+ result = WriteResult()
+ result.dry_run = dry_run
+
+ # Gemini has no memory file — all recommendations go to GEMINI.md
+ all_recs = recommendations
+ if not all_recs:
+ return result
+
+ gemini_md = project.context_file or (project.project_path / "GEMINI.md")
+ section_content = self._build_section(all_recs)
+ full_content = self._merge_into_file(gemini_md, section_content)
+ result.add(gemini_md, full_content)
+ if not dry_run:
+ gemini_md.parent.mkdir(parents=True, exist_ok=True)
+ gemini_md.write_text(full_content)
+
+ return result
+
+ def _build_section(self, recommendations: list[Recommendation]) -> str:
+ now = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+ lines = [
+ _MARKER_START,
+ "## Headroom Learned Patterns",
+ f"*Auto-generated by `headroom learn` on {now} — do not edit manually*",
+ "",
+ ]
+ for rec in recommendations:
+ lines.append(f"### {rec.section}")
+ lines.append(rec.content)
+ lines.append("")
+ lines.append(_MARKER_END)
+ return "\n".join(lines)
+
+ def _merge_into_file(self, file_path: Path, section: str) -> str:
+ if file_path.exists():
+ existing = file_path.read_text()
+ if _MARKER_START in existing:
+ return _MARKER_PATTERN.sub(section, existing)
+ return existing.rstrip() + "\n\n" + section + "\n"
+ return section + "\n"
diff --git a/tests/test_learn/test_integration.py b/tests/test_learn/test_integration.py
new file mode 100644
index 000000000..541c2d757
--- /dev/null
+++ b/tests/test_learn/test_integration.py
@@ -0,0 +1,336 @@
+"""Integration tests for headroom learn — using real session data.
+
+These tests run against actual conversation data on the machine.
+They verify the full pipeline: scan → analyze → recommend → write.
+Tests are skipped if the required data directories don't exist.
+
+Key behaviors tested:
+- Quality gates prevent writing when evidence is weak
+- False positives don't generate recommendations
+- Real Codex/Claude Code sessions produce sensible output
+- Skip logic: no file writes when nothing meaningful is found
+- Idempotency: running twice produces same output
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from headroom.learn.analyzer import FailureAnalyzer
+from headroom.learn.models import (
+ AnalysisReport,
+ ProjectInfo,
+ Recommendation,
+ RecommendationTarget,
+)
+from headroom.learn.writer import ClaudeCodeWriter, CodexWriter, Recommender
+
+# =============================================================================
+# Quality Gate Tests (synthetic data, always run)
+# =============================================================================
+
+
+class TestQualityGates:
+ """Verify that weak signals don't generate recommendations."""
+
+ def test_single_failure_not_enough(self):
+ """One failure shouldn't generate a recommendation (min_evidence=2)."""
+ recommender = Recommender(min_evidence=2)
+ report = AnalysisReport(
+ project=ProjectInfo(name="test", project_path=Path("/tmp"), data_path=Path("/tmp")),
+ total_calls=100,
+ total_failures=1,
+ total_sessions=1,
+ )
+ # One missing path — below threshold
+ from headroom.learn.models import StructureNote
+
+ report.structure_notes = [
+ StructureNote(
+ category="missing_path",
+ path="/src/foo.py",
+ note="not found",
+ evidence_count=1,
+ )
+ ]
+ recs = recommender.recommend(report)
+ assert recs == []
+
+ def test_low_total_evidence_skipped(self):
+ """Even if individual recs pass, if total evidence is too low, skip all."""
+ recommender = Recommender(min_evidence=1, min_total_evidence=10)
+ report = AnalysisReport(
+ project=ProjectInfo(name="test", project_path=Path("/tmp"), data_path=Path("/tmp")),
+ total_calls=100,
+ total_failures=2,
+ total_sessions=1,
+ )
+ from headroom.learn.models import StructureNote
+
+ report.structure_notes = [
+ StructureNote(category="missing_path", path="/a.py", note="x", evidence_count=2)
+ ]
+ recs = recommender.recommend(report)
+ assert recs == [] # Total evidence=2 < min_total_evidence=10
+
+ def test_sufficient_evidence_passes(self):
+ """Enough evidence generates recommendations."""
+ recommender = Recommender(min_evidence=2, min_total_evidence=3)
+ report = AnalysisReport(
+ project=ProjectInfo(name="test", project_path=Path("/tmp"), data_path=Path("/tmp")),
+ total_calls=100,
+ total_failures=10,
+ total_sessions=5,
+ )
+ from headroom.learn.models import StructureNote
+
+ report.structure_notes = [
+ StructureNote(
+ category="large_file",
+ path="/big.py",
+ note="too big",
+ evidence_count=5,
+ sessions_seen=3,
+ ),
+ ]
+ recs = recommender.recommend(report)
+ assert len(recs) >= 1
+
+ def test_no_failures_no_recommendations(self):
+ """Zero failures = zero recommendations, no files touched."""
+ recommender = Recommender()
+ report = AnalysisReport(
+ project=ProjectInfo(name="test", project_path=Path("/tmp"), data_path=Path("/tmp")),
+ total_calls=500,
+ total_failures=0,
+ total_sessions=10,
+ )
+ recs = recommender.recommend(report)
+ assert recs == []
+
+
+class TestSkipWriteLogic:
+ """Verify files are NOT written when there's nothing to write."""
+
+ def test_empty_recommendations_no_write(self, tmp_path):
+ """Zero recommendations = zero files written."""
+ proj = ProjectInfo(
+ name="clean-project",
+ project_path=tmp_path / "proj",
+ data_path=tmp_path / "data",
+ )
+ (tmp_path / "proj").mkdir()
+ (tmp_path / "data" / "memory").mkdir(parents=True)
+
+ writer = ClaudeCodeWriter()
+ result = writer.write([], proj, dry_run=False)
+
+ assert result.files_written == []
+ assert not (tmp_path / "proj" / "CLAUDE.md").exists()
+ assert not (tmp_path / "data" / "memory" / "MEMORY.md").exists()
+
+ def test_codex_empty_no_write(self, tmp_path):
+ """Codex writer also skips on empty recommendations."""
+ proj = ProjectInfo(name="clean", project_path=tmp_path, data_path=tmp_path)
+ writer = CodexWriter()
+ result = writer.write([], proj, dry_run=False)
+ assert result.files_written == []
+ assert not (tmp_path / "AGENTS.md").exists()
+
+
+class TestIdempotency:
+ """Running learn twice should produce the same output, not duplicate."""
+
+ def test_double_write_replaces_not_appends(self, tmp_path):
+ proj = ProjectInfo(name="test", project_path=tmp_path, data_path=tmp_path / "data")
+ (tmp_path / "data" / "memory").mkdir(parents=True)
+
+ recs = [
+ Recommendation(
+ target=RecommendationTarget.CONTEXT_FILE,
+ section="Environment",
+ content="- Use `uv run python`",
+ confidence=0.9,
+ evidence_count=10,
+ )
+ ]
+
+ writer = ClaudeCodeWriter()
+ # First write
+ writer.write(recs, proj, dry_run=False)
+ first_content = (tmp_path / "CLAUDE.md").read_text()
+
+ # Second write (same recs)
+ writer.write(recs, proj, dry_run=False)
+ second_content = (tmp_path / "CLAUDE.md").read_text()
+
+ # Content should be identical (replaced, not appended)
+ assert first_content == second_content
+ assert second_content.count("uv run python") == 1
+
+
+class TestFalsePositiveFiltering:
+ """Verify that common false positives don't generate recommendations."""
+
+ def test_sed_output_not_error(self):
+ """sed printing file content with 'Error:' in it isn't a real error."""
+ from headroom.learn.scanner import is_error_content
+
+ # Normal code output that happens to contain "error" in identifiers
+ assert not is_error_content("def handle_error(e):\n print('ok')")
+ # Normal sed output with error handling code in the file content
+ assert not is_error_content(
+ ' return fmt.Errorf("connection failed")\n log.Print("ok")'
+ )
+
+ def test_real_error_detected(self):
+ """Actual errors should be detected."""
+ from headroom.learn.scanner import is_error_content
+
+ assert is_error_content("ModuleNotFoundError: No module named 'flask'")
+ assert is_error_content(
+ "FileNotFoundError: [Errno 2] No such file or directory: '/bad/path'"
+ )
+ assert is_error_content("bash: unknown_cmd: command not found")
+
+
+# =============================================================================
+# Real-World Integration Tests (skipped if data not present)
+# =============================================================================
+
+
+CLAUDE_DIR = Path.home() / ".claude" / "projects"
+CODEX_DIR = Path.home() / ".codex" / "sessions"
+
+
[email protected](not CLAUDE_DIR.exists(), reason="No Claude Code data")
+class TestClaudeCodeIntegration:
+ """Integration tests against real Claude Code session data."""
+
+ def test_scanner_discovers_projects(self):
+ from headroom.learn.scanner import ClaudeCodeScanner
+
+ scanner = ClaudeCodeScanner()
+ projects = scanner.discover_projects()
+ assert len(projects) > 0
+ for p in projects:
+ assert p.name
+ assert p.data_path.exists()
+
+ def test_full_pipeline_produces_output(self):
+ """Scan → analyze → recommend on real data produces valid output."""
+ from headroom.learn.scanner import ClaudeCodeScanner
+
+ scanner = ClaudeCodeScanner()
+ projects = scanner.discover_projects()
+
+ # Find a project with the most sessions
+ best = max(projects, key=lambda p: len(list(p.data_path.glob("*.jsonl"))))
+
+ sessions = scanner.scan_project(best)
+ assert len(sessions) > 0
+
+ analyzer = FailureAnalyzer()
+ report = analyzer.analyze(best, sessions)
+
+ assert report.total_calls > 0
+ assert report.total_sessions > 0
+ # Failure rate should be realistic (0-30%)
+ assert 0 <= report.failure_rate <= 0.5
+
+ # Corrections should be found if there are failures
+ if report.total_failures > 10:
+ assert len(report.corrections) > 0
+
+ def test_dry_run_writes_nothing(self):
+ """Dry run should never create files."""
+ from headroom.learn.scanner import ClaudeCodeScanner
+
+ scanner = ClaudeCodeScanner()
+ projects = scanner.discover_projects()
+ best = max(projects, key=lambda p: len(list(p.data_path.glob("*.jsonl"))))
+
+ sessions = scanner.scan_project(best)
+ report = FailureAnalyzer().analyze(best, sessions)
+ recs = Recommender().recommend(report)
+
+ writer = ClaudeCodeWriter()
+ result = writer.write(recs, best, dry_run=True)
+
+ assert result.dry_run is True
+ # Verify no new files were created (check a temp path won't exist)
+ for fp in result.files_written:
+ # Files should only be in expected locations
+ assert "CLAUDE.md" in fp.name or "MEMORY.md" in fp.name
+
+
[email protected](not CODEX_DIR.exists(), reason="No Codex data")
+class TestCodexIntegration:
+ """Integration tests against real Codex session data."""
+
+ def test_scanner_discovers_sessions(self):
+ from headroom.learn.scanner import CodexScanner
+
+ scanner = CodexScanner()
+ projects = scanner.discover_projects()
+ assert len(projects) == 1 # Codex returns one "project"
+
+ def test_full_pipeline(self):
+ """Full pipeline on real Codex data."""
+ from headroom.learn.scanner import CodexScanner
+
+ scanner = CodexScanner()
+ projects = scanner.discover_projects()
+ sessions = scanner.scan_project(projects[0])
+
+ assert len(sessions) > 0
+
+ analyzer = FailureAnalyzer()
+ report = analyzer.analyze(projects[0], sessions)
+
+ assert report.total_calls > 0
+ # Codex has only Bash tool (shell)
+ all_tools = {tc.name for s in sessions for tc in s.tool_calls}
+ assert "Bash" in all_tools
+
+ # Failure rate should be realistic
+ assert 0 <= report.failure_rate <= 0.5
+
+ def test_quality_gate_filters_noise(self):
+ """Codex has false positive 'runtime_error' from sed output.
+ Quality gate should prevent these from generating weak recommendations."""
+ from headroom.learn.scanner import CodexScanner
+
+ scanner = CodexScanner()
+ projects = scanner.discover_projects()
+ sessions = scanner.scan_project(projects[0])
+
+ report = FailureAnalyzer().analyze(projects[0], sessions)
+ # Use strict quality gates
+ recommender = Recommender(min_evidence=5, min_total_evidence=10)
+ recs = recommender.recommend(report)
+
+ # Every recommendation should have meaningful evidence
+ for rec in recs:
+ assert rec.evidence_count >= 5
+ assert rec.confidence >= 0.3
+
+ def test_codex_writer_targets_agents_md(self):
+ """Codex writer should target AGENTS.md, not CLAUDE.md."""
+ from headroom.learn.scanner import CodexScanner
+
+ scanner = CodexScanner()
+ projects = scanner.discover_projects()
+ sessions = scanner.scan_project(projects[0])
+
+ report = FailureAnalyzer().analyze(projects[0], sessions)
+ recs = Recommender().recommend(report)
+
+ if recs:
+ writer = CodexWriter()
+ result = writer.write(recs, projects[0], dry_run=True)
+ for fp in result.files_written:
+ assert fp.name in ("AGENTS.md", "instructions.md")
+ assert "CLAUDE.md" not in fp.name