Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(learn): preserve prior recommendations across runs (#231)

Garm0123e499393b
diff --git a/headroom/learn/writer.py b/headroom/learn/writer.py
index 140e35b1c..b50ef8dad 100644
--- a/headroom/learn/writer.py
+++ b/headroom/learn/writer.py
@@ -87,8 +87,73 @@ def _build_section(recommendations: list[Recommendation]) -> str:
return "\n".join(lines)
-def _merge_into_file(file_path: Path, section: str) -> str:
- """Merge the section into an existing file, replacing any prior section."""
+# Matches the "*~N tokens/session saved*" annotation emitted by _build_section.
+_TOKENS_ANNOTATION_PATTERN = re.compile(r"\*~([\d,]+) tokens/session saved\*\n?")
+
+
+def _parse_prior_recommendations(existing: str) -> list[Recommendation]:
+ """Parse recommendations out of a prior marker block.
+
+ Returns [] if no marker block is present or it contains no sections.
+ The returned Recommendation objects are round-trip compatible with
+ _build_section — target is set to a placeholder since the marker block
+ itself doesn't record it (blocks are always per-file and per-target).
+ """
+ match = _MARKER_PATTERN.search(existing)
+ if not match:
+ return []
+ inner = match.group(0)[len(_MARKER_START) : -len(_MARKER_END)]
+
+ recs: list[Recommendation] = []
+ for part in re.split(r"\n### ", "\n" + inner)[1:]:
+ heading_line, _, body = part.partition("\n")
+ heading = heading_line.strip()
+ if not heading:
+ continue
+
+ tokens_saved = 0
+ tokens_match = _TOKENS_ANNOTATION_PATTERN.match(body)
+ if tokens_match:
+ tokens_saved = int(tokens_match.group(1).replace(",", ""))
+ body = body[tokens_match.end() :]
+
+ recs.append(
+ Recommendation(
+ target=RecommendationTarget.CONTEXT_FILE,
+ section=heading,
+ content=body.rstrip(),
+ estimated_tokens_saved=tokens_saved,
+ )
+ )
+ return recs
+
+
+def _merge_recommendations(
+ file_path: Path,
+ new_recommendations: list[Recommendation],
+) -> list[Recommendation]:
+ """Union new recommendations with prior ones whose section is not re-surfaced.
+
+ Sections produced by the current run take precedence over same-named
+ prior sections — the latest analysis is authoritative. Prior sections
+ whose headings do not reappear in the new run are carried forward so
+ a re-run doesn't silently drop accumulated learnings. To fully rebuild
+ the block, delete it manually and re-run.
+ """
+ if not file_path.exists():
+ return new_recommendations
+ prior = _parse_prior_recommendations(file_path.read_text())
+ if not prior:
+ return new_recommendations
+ new_sections = {r.section for r in new_recommendations}
+ carried = [p for p in prior if p.section not in new_sections]
+ return list(new_recommendations) + carried
+
+
+def _merge_into_file(file_path: Path, new_recommendations: list[Recommendation]) -> str:
+ """Merge new recommendations with any existing marker block and rebuild the file."""
+ merged = _merge_recommendations(file_path, new_recommendations)
+ section = _build_section(merged)
if file_path.exists():
existing = file_path.read_text()
if _MARKER_START in existing:
@@ -119,8 +184,7 @@ class ClaudeCodeWriter(ContextWriter):
if context_recs:
claude_md_path = self._resolve_context_path(project)
- section_content = _build_section(context_recs)
- full_content = _merge_into_file(claude_md_path, section_content)
+ full_content = _merge_into_file(claude_md_path, context_recs)
result.add(claude_md_path, full_content)
if not dry_run:
claude_md_path.parent.mkdir(parents=True, exist_ok=True)
@@ -128,8 +192,7 @@ class ClaudeCodeWriter(ContextWriter):
if memory_recs:
memory_path = self._resolve_memory_path(project)
- section_content = _build_section(memory_recs)
- full_content = _merge_into_file(memory_path, section_content)
+ full_content = _merge_into_file(memory_path, memory_recs)
result.add(memory_path, full_content)
if not dry_run:
memory_path.parent.mkdir(parents=True, exist_ok=True)
@@ -174,8 +237,7 @@ class CodexWriter(ContextWriter):
if context_recs:
agents_md = project.context_file or (project.project_path / "AGENTS.md")
- section_content = _build_section(context_recs)
- full_content = _merge_into_file(agents_md, section_content)
+ full_content = _merge_into_file(agents_md, context_recs)
result.add(agents_md, full_content)
if not dry_run:
agents_md.parent.mkdir(parents=True, exist_ok=True)
@@ -183,8 +245,7 @@ class CodexWriter(ContextWriter):
if memory_recs:
instructions_md = project.memory_file or (project.data_path.parent / "instructions.md")
- section_content = _build_section(memory_recs)
- full_content = _merge_into_file(instructions_md, section_content)
+ full_content = _merge_into_file(instructions_md, memory_recs)
result.add(instructions_md, full_content)
if not dry_run:
instructions_md.parent.mkdir(parents=True, exist_ok=True)
@@ -214,8 +275,7 @@ class GeminiWriter(ContextWriter):
return result
gemini_md = project.context_file or (project.project_path / "GEMINI.md")
- section_content = _build_section(recommendations)
- full_content = _merge_into_file(gemini_md, section_content)
+ full_content = _merge_into_file(gemini_md, recommendations)
result.add(gemini_md, full_content)
if not dry_run:
gemini_md.parent.mkdir(parents=True, exist_ok=True)
diff --git a/tests/test_learn/test_writer.py b/tests/test_learn/test_writer.py
index 6bb12b832..18b4ea5ed 100644
--- a/tests/test_learn/test_writer.py
+++ b/tests/test_learn/test_writer.py
@@ -80,26 +80,112 @@ class TestClaudeCodeWriter:
assert "Existing instructions here" in content
assert "Use uv" in content
- def test_replaces_existing_headroom_section(self, tmp_path):
+ def test_carries_forward_prior_sections_not_resurfaced(self, tmp_path):
+ """Re-running learn must not drop prior sections that the new run didn't re-surface."""
proj = _project(tmp_path)
claude_md = proj.project_path / "CLAUDE.md"
- old_section = (
- f"# My Project\n\n{_MARKER_START}\n## Old Patterns\nold stuff\n{_MARKER_END}\n"
+ prior_block = (
+ f"# My Project\n\n{_MARKER_START}\n"
+ "## Headroom Learned Patterns\n"
+ "*Auto-generated by `headroom learn` on 2026-01-01 — do not edit manually*\n\n"
+ "### Large Files\n"
+ "*~15,000 tokens/session saved*\n"
+ "- src/App.tsx is huge\n\n"
+ "### Build Commands\n"
+ "- cargo check from src-tauri/\n\n"
+ f"{_MARKER_END}\n"
)
- claude_md.write_text(old_section)
+ claude_md.write_text(prior_block)
writer = ClaudeCodeWriter()
- recs = [_rec(RecommendationTarget.CONTEXT_FILE, "Environment", "- New stuff")]
+ recs = [_rec(RecommendationTarget.CONTEXT_FILE, "Environment", "- Use uv")]
writer.write(recs, proj, dry_run=False)
content = claude_md.read_text()
- assert "old stuff" not in content
- assert "New stuff" in content
assert "My Project" in content
- # Should have exactly one marker pair
+ # New section present
+ assert "Use uv" in content
+ # Prior sections preserved (neither heading re-surfaced by the new run)
+ assert "### Large Files" in content
+ assert "src/App.tsx is huge" in content
+ assert "### Build Commands" in content
+ assert "cargo check from src-tauri/" in content
+ # Tokens annotation round-tripped
+ assert "*~15,000 tokens/session saved*" in content
+ # Still exactly one marker pair
assert content.count(_MARKER_START) == 1
assert content.count(_MARKER_END) == 1
+ def test_new_run_overrides_same_named_prior_section(self, tmp_path):
+ """When a section appears in both prior and new, the new run wins."""
+ proj = _project(tmp_path)
+ claude_md = proj.project_path / "CLAUDE.md"
+ prior_block = (
+ f"{_MARKER_START}\n"
+ "## Headroom Learned Patterns\n"
+ "*Auto-generated by `headroom learn` on 2026-01-01 — do not edit manually*\n\n"
+ "### Environment\n"
+ "- old stale environment note\n\n"
+ f"{_MARKER_END}\n"
+ )
+ claude_md.write_text(prior_block)
+
+ writer = ClaudeCodeWriter()
+ recs = [_rec(RecommendationTarget.CONTEXT_FILE, "Environment", "- fresh environment note")]
+ writer.write(recs, proj, dry_run=False)
+
+ content = claude_md.read_text()
+ assert "fresh environment note" in content
+ assert "old stale environment note" not in content
+ # Only one Environment section in the final block
+ assert content.count("### Environment") == 1
+
+ def test_memory_md_carry_forward(self, tmp_path):
+ """Carry-forward also works for MEMORY.md."""
+ proj = _project(tmp_path)
+ memory_md = proj.data_path / "memory" / "MEMORY.md"
+ memory_md.write_text(
+ f"{_MARKER_START}\n"
+ "## Headroom Learned Patterns\n"
+ "*Auto-generated by `headroom learn` on 2026-01-01 — do not edit manually*\n\n"
+ "### User Workflow Preferences\n"
+ "- User rejects sleep-based polling\n\n"
+ f"{_MARKER_END}\n"
+ )
+
+ writer = ClaudeCodeWriter()
+ recs = [_rec(RecommendationTarget.MEMORY_FILE, "Related Codebases", "- web app at ~/Code/web")]
+ writer.write(recs, proj, dry_run=False)
+
+ content = memory_md.read_text()
+ assert "User rejects sleep-based polling" in content
+ assert "web app at ~/Code/web" in content
+
+ def test_section_without_tokens_annotation_round_trips(self, tmp_path):
+ """Prior sections emitted without a tokens annotation must still carry forward cleanly."""
+ proj = _project(tmp_path)
+ claude_md = proj.project_path / "CLAUDE.md"
+ claude_md.write_text(
+ f"{_MARKER_START}\n"
+ "## Headroom Learned Patterns\n"
+ "*Auto-generated by `headroom learn` on 2026-01-01 — do not edit manually*\n\n"
+ "### Misc\n"
+ "- one-liner pattern\n\n"
+ f"{_MARKER_END}\n"
+ )
+
+ writer = ClaudeCodeWriter()
+ recs = [_rec(RecommendationTarget.CONTEXT_FILE, "Other", "- new one")]
+ writer.write(recs, proj, dry_run=False)
+
+ content = claude_md.read_text()
+ assert "### Misc" in content
+ assert "one-liner pattern" in content
+ # No spurious tokens annotation injected for a prior that didn't have one
+ misc_idx = content.index("### Misc")
+ after_misc = content[misc_idx : misc_idx + 200]
+ assert "tokens/session saved" not in after_misc
+
def test_appends_to_existing_memory_md(self, tmp_path):
proj = _project(tmp_path)
memory_md = proj.data_path / "memory" / "MEMORY.md"