Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(learn): preserve prior recommendations across runs (#231)
Garm0123e499393b
diff --git a/headroom/learn/writer.py b/headroom/learn/writer.pyindex 140e35b1c..b50ef8dad 100644--- a/headroom/learn/writer.py+++ b/headroom/learn/writer.py@@ -87,8 +87,73 @@ def _build_section(recommendations: list[Recommendation]) -> str:return "\n".join(lines)-def _merge_into_file(file_path: Path, section: str) -> str:- """Merge the section into an existing file, replacing any prior section."""+# Matches the "*~N tokens/session saved*" annotation emitted by _build_section.+_TOKENS_ANNOTATION_PATTERN = re.compile(r"\*~([\d,]+) tokens/session saved\*\n?")+++def _parse_prior_recommendations(existing: str) -> list[Recommendation]:+ """Parse recommendations out of a prior marker block.++ Returns [] if no marker block is present or it contains no sections.+ The returned Recommendation objects are round-trip compatible with+ _build_section — target is set to a placeholder since the marker block+ itself doesn't record it (blocks are always per-file and per-target).+ """+ match = _MARKER_PATTERN.search(existing)+ if not match:+ return []+ inner = match.group(0)[len(_MARKER_START) : -len(_MARKER_END)]++ recs: list[Recommendation] = []+ for part in re.split(r"\n### ", "\n" + inner)[1:]:+ heading_line, _, body = part.partition("\n")+ heading = heading_line.strip()+ if not heading:+ continue++ tokens_saved = 0+ tokens_match = _TOKENS_ANNOTATION_PATTERN.match(body)+ if tokens_match:+ tokens_saved = int(tokens_match.group(1).replace(",", ""))+ body = body[tokens_match.end() :]++ recs.append(+ Recommendation(+ target=RecommendationTarget.CONTEXT_FILE,+ section=heading,+ content=body.rstrip(),+ estimated_tokens_saved=tokens_saved,+ )+ )+ return recs+++def _merge_recommendations(+ file_path: Path,+ new_recommendations: list[Recommendation],+) -> list[Recommendation]:+ """Union new recommendations with prior ones whose section is not re-surfaced.++ Sections produced by the current run take precedence over same-named+ prior sections — the latest analysis is authoritative. Prior sections+ whose headings do not reappear in the new run are carried forward so+ a re-run doesn't silently drop accumulated learnings. To fully rebuild+ the block, delete it manually and re-run.+ """+ if not file_path.exists():+ return new_recommendations+ prior = _parse_prior_recommendations(file_path.read_text())+ if not prior:+ return new_recommendations+ new_sections = {r.section for r in new_recommendations}+ carried = [p for p in prior if p.section not in new_sections]+ return list(new_recommendations) + carried+++def _merge_into_file(file_path: Path, new_recommendations: list[Recommendation]) -> str:+ """Merge new recommendations with any existing marker block and rebuild the file."""+ merged = _merge_recommendations(file_path, new_recommendations)+ section = _build_section(merged)if file_path.exists():existing = file_path.read_text()if _MARKER_START in existing:@@ -119,8 +184,7 @@ class ClaudeCodeWriter(ContextWriter):if context_recs:claude_md_path = self._resolve_context_path(project)- section_content = _build_section(context_recs)- full_content = _merge_into_file(claude_md_path, section_content)+ full_content = _merge_into_file(claude_md_path, context_recs)result.add(claude_md_path, full_content)if not dry_run:claude_md_path.parent.mkdir(parents=True, exist_ok=True)@@ -128,8 +192,7 @@ class ClaudeCodeWriter(ContextWriter):if memory_recs:memory_path = self._resolve_memory_path(project)- section_content = _build_section(memory_recs)- full_content = _merge_into_file(memory_path, section_content)+ full_content = _merge_into_file(memory_path, memory_recs)result.add(memory_path, full_content)if not dry_run:memory_path.parent.mkdir(parents=True, exist_ok=True)@@ -174,8 +237,7 @@ class CodexWriter(ContextWriter):if context_recs:agents_md = project.context_file or (project.project_path / "AGENTS.md")- section_content = _build_section(context_recs)- full_content = _merge_into_file(agents_md, section_content)+ full_content = _merge_into_file(agents_md, context_recs)result.add(agents_md, full_content)if not dry_run:agents_md.parent.mkdir(parents=True, exist_ok=True)@@ -183,8 +245,7 @@ class CodexWriter(ContextWriter):if memory_recs:instructions_md = project.memory_file or (project.data_path.parent / "instructions.md")- section_content = _build_section(memory_recs)- full_content = _merge_into_file(instructions_md, section_content)+ full_content = _merge_into_file(instructions_md, memory_recs)result.add(instructions_md, full_content)if not dry_run:instructions_md.parent.mkdir(parents=True, exist_ok=True)@@ -214,8 +275,7 @@ class GeminiWriter(ContextWriter):return resultgemini_md = project.context_file or (project.project_path / "GEMINI.md")- section_content = _build_section(recommendations)- full_content = _merge_into_file(gemini_md, section_content)+ full_content = _merge_into_file(gemini_md, recommendations)result.add(gemini_md, full_content)if not dry_run:gemini_md.parent.mkdir(parents=True, exist_ok=True)diff --git a/tests/test_learn/test_writer.py b/tests/test_learn/test_writer.pyindex 6bb12b832..18b4ea5ed 100644--- a/tests/test_learn/test_writer.py+++ b/tests/test_learn/test_writer.py@@ -80,26 +80,112 @@ class TestClaudeCodeWriter:assert "Existing instructions here" in contentassert "Use uv" in content- def test_replaces_existing_headroom_section(self, tmp_path):+ def test_carries_forward_prior_sections_not_resurfaced(self, tmp_path):+ """Re-running learn must not drop prior sections that the new run didn't re-surface."""proj = _project(tmp_path)claude_md = proj.project_path / "CLAUDE.md"- old_section = (- f"# My Project\n\n{_MARKER_START}\n## Old Patterns\nold stuff\n{_MARKER_END}\n"+ prior_block = (+ f"# My Project\n\n{_MARKER_START}\n"+ "## Headroom Learned Patterns\n"+ "*Auto-generated by `headroom learn` on 2026-01-01 — do not edit manually*\n\n"+ "### Large Files\n"+ "*~15,000 tokens/session saved*\n"+ "- src/App.tsx is huge\n\n"+ "### Build Commands\n"+ "- cargo check from src-tauri/\n\n"+ f"{_MARKER_END}\n")- claude_md.write_text(old_section)+ claude_md.write_text(prior_block)writer = ClaudeCodeWriter()- recs = [_rec(RecommendationTarget.CONTEXT_FILE, "Environment", "- New stuff")]+ recs = [_rec(RecommendationTarget.CONTEXT_FILE, "Environment", "- Use uv")]writer.write(recs, proj, dry_run=False)content = claude_md.read_text()- assert "old stuff" not in content- assert "New stuff" in contentassert "My Project" in content- # Should have exactly one marker pair+ # New section present+ assert "Use uv" in content+ # Prior sections preserved (neither heading re-surfaced by the new run)+ assert "### Large Files" in content+ assert "src/App.tsx is huge" in content+ assert "### Build Commands" in content+ assert "cargo check from src-tauri/" in content+ # Tokens annotation round-tripped+ assert "*~15,000 tokens/session saved*" in content+ # Still exactly one marker pairassert content.count(_MARKER_START) == 1assert content.count(_MARKER_END) == 1+ def test_new_run_overrides_same_named_prior_section(self, tmp_path):+ """When a section appears in both prior and new, the new run wins."""+ proj = _project(tmp_path)+ claude_md = proj.project_path / "CLAUDE.md"+ prior_block = (+ f"{_MARKER_START}\n"+ "## Headroom Learned Patterns\n"+ "*Auto-generated by `headroom learn` on 2026-01-01 — do not edit manually*\n\n"+ "### Environment\n"+ "- old stale environment note\n\n"+ f"{_MARKER_END}\n"+ )+ claude_md.write_text(prior_block)++ writer = ClaudeCodeWriter()+ recs = [_rec(RecommendationTarget.CONTEXT_FILE, "Environment", "- fresh environment note")]+ writer.write(recs, proj, dry_run=False)++ content = claude_md.read_text()+ assert "fresh environment note" in content+ assert "old stale environment note" not in content+ # Only one Environment section in the final block+ assert content.count("### Environment") == 1++ def test_memory_md_carry_forward(self, tmp_path):+ """Carry-forward also works for MEMORY.md."""+ proj = _project(tmp_path)+ memory_md = proj.data_path / "memory" / "MEMORY.md"+ memory_md.write_text(+ f"{_MARKER_START}\n"+ "## Headroom Learned Patterns\n"+ "*Auto-generated by `headroom learn` on 2026-01-01 — do not edit manually*\n\n"+ "### User Workflow Preferences\n"+ "- User rejects sleep-based polling\n\n"+ f"{_MARKER_END}\n"+ )++ writer = ClaudeCodeWriter()+ recs = [_rec(RecommendationTarget.MEMORY_FILE, "Related Codebases", "- web app at ~/Code/web")]+ writer.write(recs, proj, dry_run=False)++ content = memory_md.read_text()+ assert "User rejects sleep-based polling" in content+ assert "web app at ~/Code/web" in content++ def test_section_without_tokens_annotation_round_trips(self, tmp_path):+ """Prior sections emitted without a tokens annotation must still carry forward cleanly."""+ proj = _project(tmp_path)+ claude_md = proj.project_path / "CLAUDE.md"+ claude_md.write_text(+ f"{_MARKER_START}\n"+ "## Headroom Learned Patterns\n"+ "*Auto-generated by `headroom learn` on 2026-01-01 — do not edit manually*\n\n"+ "### Misc\n"+ "- one-liner pattern\n\n"+ f"{_MARKER_END}\n"+ )++ writer = ClaudeCodeWriter()+ recs = [_rec(RecommendationTarget.CONTEXT_FILE, "Other", "- new one")]+ writer.write(recs, proj, dry_run=False)++ content = claude_md.read_text()+ assert "### Misc" in content+ assert "one-liner pattern" in content+ # No spurious tokens annotation injected for a prior that didn't have one+ misc_idx = content.index("### Misc")+ after_misc = content[misc_idx : misc_idx + 200]+ assert "tokens/session saved" not in after_misc+def test_appends_to_existing_memory_md(self, tmp_path):proj = _project(tmp_path)memory_md = proj.data_path / "memory" / "MEMORY.md"