Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(learn): show prior patterns block to LLM to prevent dangling refs
Garm6d2aba8741ad
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 0cb71f636..1dc8a103e 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -14,6 +14,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0new run win; sections not re-surfaced are carried forward so learningsaccumulate across runs instead of disappearing. To fully rebuild theblock, delete it manually and re-run. (#231)+- **`headroom learn` no longer emits dangling cross-references when a+ section is re-surfaced** — the analyzer now includes the project's+ current `<!-- headroom:learn -->` block (from `CLAUDE.md` and+ `MEMORY.md`) in the LLM digest as a "Prior Learned Patterns" section,+ and the system prompt instructs the LLM that re-emitting a section+ replaces the prior one wholesale. Prevents bullets like "`X` is *also*+ large — same rule as `Y`, `Z`" from appearing after `Y` and `Z` got+ dropped during per-section replacement. The writer's section-level+ carry-forward from #231 remains in place as a safety net for sections+ the LLM omits entirely. New helper `extract_marker_block` added to+ `headroom.learn.writer`.### Added- **`turn_id` linking agent-loop API calls to a single user prompt** — a newdiff --git a/headroom/learn/analyzer.py b/headroom/learn/analyzer.pyindex 15b27e51e..7f9053969 100644--- a/headroom/learn/analyzer.py+++ b/headroom/learn/analyzer.py@@ -29,6 +29,7 @@ from .models import (SessionEvent,ToolCall,)+from .writer import extract_marker_blocklogger = logging.getLogger(__name__)@@ -147,11 +148,51 @@ class SessionAnalyzer:# =============================================================================+def _build_prior_patterns_section(project: ProjectInfo) -> str:+ """Format the current marker blocks from CLAUDE.md / MEMORY.md for the LLM.++ Returns "" when neither file exists nor contains a marker block. When at+ least one file has a block, returns a header + labeled raw blocks so the+ LLM can treat them as the starting baseline. See the "Prior Learned+ Patterns" rule in _SYSTEM_PROMPT for the contract with the model.+ """+ parts: list[tuple[str, str]] = [] # (label, block)+ candidates = (+ ("CLAUDE.md (CONTEXT_FILE, project-level stable facts)", project.context_file),+ ("MEMORY.md (MEMORY_FILE, session-level evolving preferences)", project.memory_file),+ )+ for label, path in candidates:+ if path is None or not path.exists():+ continue+ block = extract_marker_block(path.read_text())+ if block:+ parts.append((label, block))++ if not parts:+ return ""++ lines = [+ "=== Prior Learned Patterns ===",+ (+ f"These patterns are currently written to {project.name}'s context "+ f"files. They are your starting baseline — see the 'Prior Learned "+ f"Patterns' rule in the system prompt for how to integrate them."+ ),+ "",+ ]+ for label, block in parts:+ lines.append(f"--- From {label} ---")+ lines.append(block)+ lines.append("")+ return "\n".join(lines)++def _build_digest(project: ProjectInfo, sessions: list[SessionData]) -> str:"""Build a token-efficient text digest of all session events.The digest includes:- Project context+ - Prior learned patterns (if any) from CLAUDE.md / MEMORY.md- Per-session summaries with condensed event streams- Error outputs (truncated), success indicators, user messages"""@@ -173,6 +214,12 @@ def _build_digest(project: ProjectInfo, sessions: list[SessionData]) -> str:lines.append(f"Tokens used: {total_tokens_in:,} in / {total_tokens_out:,} out")lines.append("")+ # Prior learned patterns (if any) — gives the LLM the current baseline so+ # it can produce complete updated sections instead of condensed deltas.+ prior_section = _build_prior_patterns_section(project)+ if prior_section:+ lines.append(prior_section)+# Budget tracking — stop adding events when we approach the limit# Rough estimate: 4 chars per tokenchar_budget = _MAX_DIGEST_TOKENS * 4@@ -289,6 +336,24 @@ Rules:- Do NOT produce tautological rules (e.g., "use python3 not python3")- Do NOT produce rules about things that only happened once (transient errors)+Prior Learned Patterns:+- The input may contain a "Prior Learned Patterns" section showing what is+ already written to the project's CLAUDE.md / MEMORY.md. Treat those as the+ starting baseline for your analysis.+- When you re-emit a section heading that appears in the prior block, your+ output REPLACES that prior section wholesale — so your section must be the+ COMPLETE updated version:+ * Preserve prior bullets that remain accurate (copy them forward)+ * Revise bullets when new evidence refines them (merge, don't duplicate)+ * Drop a prior bullet only when contradicted by clear new evidence+- Sections from prior runs that you do NOT re-emit are preserved automatically+ by the writer, so focus only on sections where you have something to add or+ change. Do NOT re-emit a prior section just to echo it verbatim — that wastes+ output tokens without changing the outcome.+- Do NOT write bullets that reference prior siblings you are about to drop+ (e.g., "X is ALSO large — same rule as Y, Z") unless Y and Z are also present+ in your current output or preserved in the prior block.+Return ONLY valid JSON matching this schema — no other text:{"context_file_rules": [diff --git a/headroom/learn/writer.py b/headroom/learn/writer.pyindex b50ef8dad..8d2168ab2 100644--- a/headroom/learn/writer.py+++ b/headroom/learn/writer.py@@ -91,6 +91,17 @@ def _build_section(recommendations: list[Recommendation]) -> str:_TOKENS_ANNOTATION_PATTERN = re.compile(r"\*~([\d,]+) tokens/session saved\*\n?")+def extract_marker_block(file_content: str) -> str | None:+ """Return the raw text of the headroom:learn marker block, or None.++ Unlike _parse_prior_recommendations, this returns the block verbatim+ (including the start/end markers) so it can be fed back to an LLM as+ context without losing formatting. Returns None if no block is present.+ """+ match = _MARKER_PATTERN.search(file_content)+ return match.group(0) if match else None++def _parse_prior_recommendations(existing: str) -> list[Recommendation]:"""Parse recommendations out of a prior marker block.diff --git a/tests/test_learn/test_analyzer.py b/tests/test_learn/test_analyzer.pyindex f15ba9c86..9f3d93cd2 100644--- a/tests/test_learn/test_analyzer.py+++ b/tests/test_learn/test_analyzer.py@@ -153,6 +153,117 @@ class TestDigestBuilder:assert "0 sessions" in digest or "test-project" in digest+# =============================================================================+# Prior Patterns Injection Tests+# =============================================================================+++_MARKER_BLOCK = (+ "<!-- headroom:learn:start -->\n"+ "## Headroom Learned Patterns\n"+ "*Auto-generated by `headroom learn` on 2026-04-01 — do not edit manually*\n"+ "\n"+ "### Large Files\n"+ "- `src/App.tsx` is very large (~40k tokens) — use offset/limit reads\n"+ "- `src/lib.rs` frequently exceeds 10k tokens\n"+ "\n"+ "<!-- headroom:learn:end -->"+)+++def _project_with_files(tmp_path: Path, claude_md_text: str | None, memory_md_text: str | None) -> ProjectInfo:+ """Build a ProjectInfo pointing at temp CLAUDE.md / MEMORY.md files."""+ proj_dir = tmp_path / "proj"+ proj_dir.mkdir()+ data_dir = tmp_path / "data"+ (data_dir / "memory").mkdir(parents=True)++ context_file: Path | None = None+ if claude_md_text is not None:+ context_file = proj_dir / "CLAUDE.md"+ context_file.write_text(claude_md_text)++ memory_file: Path | None = None+ if memory_md_text is not None:+ memory_file = data_dir / "memory" / "MEMORY.md"+ memory_file.write_text(memory_md_text)++ return ProjectInfo(+ name="proj",+ project_path=proj_dir,+ data_path=data_dir,+ context_file=context_file,+ memory_file=memory_file,+ )+++class TestPriorPatternsInjection:+ """The digest should include the prior marker block so the LLM can emit+ COMPLETE updated sections instead of condensed deltas that reference+ now-dropped siblings (the "X is also large — same rule as Y, Z" bug)."""++ def test_digest_includes_prior_block_from_claude_md(self, tmp_path):+ project = _project_with_files(tmp_path, claude_md_text=f"# Project\n\n{_MARKER_BLOCK}\n", memory_md_text=None)+ digest = _build_digest(project, [])+ assert "Prior Learned Patterns" in digest+ assert "### Large Files" in digest+ assert "App.tsx" in digest++ def test_digest_includes_prior_block_from_memory_md(self, tmp_path):+ project = _project_with_files(tmp_path, claude_md_text=None, memory_md_text=f"{_MARKER_BLOCK}\n")+ digest = _build_digest(project, [])+ assert "Prior Learned Patterns" in digest+ assert "MEMORY.md" in digest+ assert "### Large Files" in digest++ def test_digest_omits_section_when_no_files_exist(self, tmp_path):+ project = _project_with_files(tmp_path, claude_md_text=None, memory_md_text=None)+ digest = _build_digest(project, [])+ assert "Prior Learned Patterns" not in digest+ assert "<!-- headroom:learn" not in digest++ def test_digest_omits_section_when_file_has_no_marker_block(self, tmp_path):+ """CLAUDE.md exists but has no headroom block → no prior section emitted."""+ project = _project_with_files(+ tmp_path,+ claude_md_text="# Project\n\nJust a regular readme, no headroom block.\n",+ memory_md_text=None,+ )+ digest = _build_digest(project, [])+ assert "Prior Learned Patterns" not in digest++ def test_digest_surfaces_both_files_when_both_present(self, tmp_path):+ project = _project_with_files(+ tmp_path,+ claude_md_text=f"# Project\n\n{_MARKER_BLOCK}\n",+ memory_md_text=f"{_MARKER_BLOCK}\n",+ )+ digest = _build_digest(project, [])+ assert digest.count("### Large Files") >= 2 # once per file+ assert "CLAUDE.md" in digest+ assert "MEMORY.md" in digest++ @patch("headroom.learn.analyzer._call_llm")+ def test_analyze_passes_prior_block_through_to_llm(self, mock_call_llm: MagicMock, tmp_path):+ """End-to-end: SessionAnalyzer.analyze() → _call_llm receives digest+ containing the prior marker block content."""+ mock_call_llm.return_value = {"context_file_rules": [], "memory_file_rules": []}+ project = _project_with_files(tmp_path, claude_md_text=f"# Project\n\n{_MARKER_BLOCK}\n", memory_md_text=None)+ sessions = [+ SessionData(+ session_id="s1",+ tool_calls=[_tc(msg_index=0, is_error=True, output="error")],+ )+ ]++ SessionAnalyzer(model="test-model").analyze(project, sessions)++ mock_call_llm.assert_called_once()+ digest_arg = mock_call_llm.call_args[0][0]+ assert "Prior Learned Patterns" in digest_arg+ assert "App.tsx" in digest_arg++# =============================================================================# LLM Response Parser Tests# =============================================================================diff --git a/tests/test_learn/test_writer.py b/tests/test_learn/test_writer.pyindex 4ee3c7405..4b2d6d212 100644--- a/tests/test_learn/test_writer.py+++ b/tests/test_learn/test_writer.py@@ -8,6 +8,7 @@ from headroom.learn.writer import (_MARKER_START,ClaudeCodeWriter,_parse_prior_recommendations,+ extract_marker_block,)@@ -239,3 +240,43 @@ class TestParsePriorRecommendations:assert len(recs) == 1assert recs[0].section == "Real Section"assert "real bullet" in recs[0].content+++class TestExtractMarkerBlock:+ """Direct coverage for extract_marker_block."""++ def test_returns_raw_block_when_present(self):+ """Marker block is returned verbatim with delimiters, for LLM prompts."""+ content = (+ "# Project README\n\n"+ "Some text.\n\n"+ f"{_MARKER_START}\n"+ "## Headroom Learned Patterns\n"+ "### Environment\n"+ "- Use uv run python\n"+ f"{_MARKER_END}\n"+ "Trailing text.\n"+ )+ block = extract_marker_block(content)+ assert block is not None+ assert block.startswith(_MARKER_START)+ assert block.endswith(_MARKER_END)+ assert "### Environment" in block+ assert "Use uv run python" in block+ assert "Trailing text." not in block++ def test_returns_none_when_absent(self):+ """File without any marker delimiters yields None."""+ assert extract_marker_block("# Project\n\nJust a regular README.\n") is None++ def test_returns_none_when_only_start_marker(self):+ """Partial/malformed block (start only) yields None — writer expects both delimiters."""+ content = f"prefix\n{_MARKER_START}\n### Something\n- content\n"+ assert extract_marker_block(content) is None++ def test_returns_empty_block_when_markers_are_adjacent(self):+ """A block with nothing between the markers is still returned (caller's choice what to do)."""+ content = f"prefix\n{_MARKER_START}\n{_MARKER_END}\nsuffix\n"+ block = extract_marker_block(content)+ assert block is not None+ assert block == f"{_MARKER_START}\n{_MARKER_END}"