Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(memory): pin LF on memory writers and guard the learn-writer newline contract (#3706)
Tejas Choprab10dd8db6e71
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex fcd5074c4..637fd094f 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -178,6 +178,59 @@ jobs:path: dist/*.whlretention-days: 1+ test-windows-newline:+ # No unit test has ever run on Windows (the `test` shards are+ # ubuntu-only), which is how a Windows-only newline bug shipped twice+ # (#3594, #3698). A full Windows shard is not worth the minutes; this+ # runs only the handful of tests marked `windows_newline` against the+ # wheel `build-wheel-windows` already produced. The marker keeps the set+ # greppable — add `@pytest.mark.windows_newline` to a test and it runs+ # here, no workflow edit needed.+ #+ # Gated on `packaging` because that is what builds the wheel this job+ # consumes; on a pure-Python PR the ubuntu `test` shards still run the+ # same marked tests (they assert the `newline=` kwarg, which fails on any+ # platform), so this job is the Windows-side backstop, not the only guard.+ needs: [changes, build-wheel-windows]+ if: needs.changes.outputs.packaging == 'true'+ runs-on: windows-latest+ timeout-minutes: 30+ steps:+ - uses: actions/checkout@v7+ - uses: actions/setup-python@v7+ with:+ python-version: ${{ env.PY_VERSION }}+ - name: Download Windows wheel+ uses: actions/download-artifact@v8+ with:+ name: headroom-wheel-windows+ path: dist+ - name: Install wheel + dev deps+ shell: bash+ run: |+ python -m pip install --upgrade pip+ WHEEL="$(ls dist/*.whl)"+ pip install "${WHEEL}[dev]"+ - name: Drop the source tree so the wheel is what gets tested+ shell: bash+ # `actions/checkout` leaves ./headroom/ on sys.path ahead of the+ # installed wheel, and the source tree has no compiled+ # `headroom/_core.abi3.so` -- so every module that reaches the Rust+ # extension dies with `ModuleNotFoundError: No module named+ # 'headroom._core'` (203 collection errors on the first run of this+ # job). The release workflow's smoke-import sidesteps this by running+ # from RUNNER_TEMP; here we need ./tests/, so remove the shadow+ # instead. Testing the installed artifact rather than the source tree+ # is the point of consuming the wheel anyway.+ run: rm -rf headroom+ - name: Run newline-contract tests+ shell: bash+ # Scoped to the trees that hold the marked tests rather than all of+ # ./tests: collecting ~10k modules on Windows took 91s and pulled in+ # unrelated import failures. Add a directory here if you mark a test+ # outside these paths.+ run: pytest tests/test_learn tests/test_memory tests/test_memory_sync.py -m windows_newline --tb=short -q+python-314-wheels:# Partners on centrally managed machines cannot choose their Python, and# many have no compiler. Install the proxy extras on the newest supporteddiff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex 775dc1691..48027f36d 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -123,6 +123,16 @@ Enable or disable automatic Copilot review in **Settings → Rules → Rulesets- Type hints on public functions; Google-style docstrings.- Cover new behavior + edge cases; aim >80% coverage on new code.- Python 3.10+. Optional features go behind extras.+- **Headroom writes LF and normalizes on read, on every platform.** Pass+ `newline="\n"` to every `write_text`/`open` that writes a context, memory,+ or state file, and normalize `\r\n`/`\r` when you read one back. Without+ the pin, `TextIOWrapper` translates `\n` to `\r\n` on Windows, and a reader+ that decodes bytes directly then re-writes accumulates carriage returns+ (#3594); even with a normalizing reader, two subsystems writing the same+ file (`CLAUDE.md`, `AGENTS.md`) flip it between LF and CRLF (#3698). Cover+ new write sites with `@pytest.mark.windows_newline` — those tests assert the+ `newline=` kwarg (an artifact assertion cannot fail on POSIX) and also run+ on `windows-latest` in CI.## Architecture principlesdiff --git a/headroom/memory/bridge.py b/headroom/memory/bridge.pyindex 7ea081d16..45e748a1e 100644--- a/headroom/memory/bridge.py+++ b/headroom/memory/bridge.py@@ -126,6 +126,7 @@ class MemoryBridge:path.write_text(json.dumps(self._sync_state, indent=2, default=str),encoding="utf-8",+ newline="\n",)except OSError as e:logger.warning(f"Bridge: Failed to save sync state to {path}: {e}")@@ -390,7 +391,7 @@ class MemoryBridge:if path:path = Path(path).expanduser()path.parent.mkdir(parents=True, exist_ok=True)- path.write_text(markdown, encoding="utf-8")+ path.write_text(markdown, encoding="utf-8", newline="\n")logger.info(f"Bridge: Exported {len(memories)} memories to {path}")# Update sync state to avoid re-importing our own export@@ -646,7 +647,7 @@ class MemoryBridge:try:path.parent.mkdir(parents=True, exist_ok=True)- path.write_text(new_content, encoding="utf-8")+ path.write_text(new_content, encoding="utf-8", newline="\n")except OSError as e:logger.error(f"Bridge: Failed to write {path}: {e}")return 0diff --git a/headroom/memory/sync.py b/headroom/memory/sync.pyindex ce4f0c596..0b3c80fd8 100644--- a/headroom/memory/sync.py+++ b/headroom/memory/sync.py@@ -134,7 +134,7 @@ def _load_sync_state(state_path: Path) -> dict[str, Any]:def _save_sync_state(state_path: Path, state: dict[str, Any]) -> None:"""Save sync state to disk."""state_path.parent.mkdir(parents=True, exist_ok=True)- state_path.write_text(json.dumps(state, indent=2), encoding="utf-8")+ state_path.write_text(json.dumps(state, indent=2), encoding="utf-8", newline="\n")def _db_fingerprint(memories: list[Any]) -> str:diff --git a/headroom/memory/sync_adapters/claude_code.py b/headroom/memory/sync_adapters/claude_code.pyindex d26151db7..71607ada1 100644--- a/headroom/memory/sync_adapters/claude_code.py+++ b/headroom/memory/sync_adapters/claude_code.py@@ -193,7 +193,7 @@ class ClaudeCodeAdapter(AgentMemoryAdapter):"source_agent": source_agent,})- target.write_text(f"{fm}\n\n{content}\n", encoding="utf-8")+ target.write_text(f"{fm}\n\n{content}\n", encoding="utf-8", newline="\n")written += 1# Track for MEMORY.md index@@ -236,7 +236,7 @@ class ClaudeCodeAdapter(AgentMemoryAdapter):else:content = "# Memory\n" + new_section- memory_md.write_text(content, encoding="utf-8")+ memory_md.write_text(content, encoding="utf-8", newline="\n")def fingerprint(self) -> str:"""Hash of all .md filenames + contents for change detection."""diff --git a/headroom/memory/sync_adapters/codex_agent.py b/headroom/memory/sync_adapters/codex_agent.pyindex 01f9da07a..ae582bf22 100644--- a/headroom/memory/sync_adapters/codex_agent.py+++ b/headroom/memory/sync_adapters/codex_agent.py@@ -121,7 +121,7 @@ class CodexAdapter(AgentMemoryAdapter):self._path.parent.mkdir(parents=True, exist_ok=True)content = section + "\n"- self._path.write_text(content, encoding="utf-8")+ self._path.write_text(content, encoding="utf-8", newline="\n")return addeddef fingerprint(self) -> str:diff --git a/headroom/memory/writers/base.py b/headroom/memory/writers/base.pyindex 5e8ed1636..7d9e961dd 100644--- a/headroom/memory/writers/base.py+++ b/headroom/memory/writers/base.py@@ -165,7 +165,7 @@ class AgentWriter(ABC):if not dry_run:target.parent.mkdir(parents=True, exist_ok=True)- target.write_text(full_content, encoding="utf-8")+ target.write_text(full_content, encoding="utf-8", newline="\n")return resultdiff --git a/headroom/memory/writers/claude_writer.py b/headroom/memory/writers/claude_writer.pyindex 44d971363..4d5c507b3 100644--- a/headroom/memory/writers/claude_writer.py+++ b/headroom/memory/writers/claude_writer.py@@ -116,6 +116,6 @@ class ClaudeCodeMemoryWriter(AgentWriter):if not dry_run:memory_dir.mkdir(parents=True, exist_ok=True)- (memory_dir / filename).write_text(content, encoding="utf-8")+ (memory_dir / filename).write_text(content, encoding="utf-8", newline="\n")return topic_filesdiff --git a/headroom/memory/writers/cursor_writer.py b/headroom/memory/writers/cursor_writer.pyindex ea77b537e..b9fd84b2b 100644--- a/headroom/memory/writers/cursor_writer.py+++ b/headroom/memory/writers/cursor_writer.py@@ -127,6 +127,6 @@ class CursorMemoryWriter(AgentWriter):if not dry_run:target.parent.mkdir(parents=True, exist_ok=True)- target.write_text(full_content, encoding="utf-8")+ target.write_text(full_content, encoding="utf-8", newline="\n")return resultdiff --git a/pyproject.toml b/pyproject.tomlindex f64c8ea68..19a3f5f56 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -532,6 +532,7 @@ markers = ["real_llm: tests that hit real LLM APIs; skipped unless explicitly enabled","live: opt-in multi-turn tests that hit real upstream APIs; require provider keys","proxy_dependency_gate: exercises ensure_proxy_dependencies() without mocking",+ "windows_newline: newline-contract tests (#3698); also run on windows-latest in CI",][tool.coverage.run]diff --git a/tests/test_learn/test_writer.py b/tests/test_learn/test_writer.pyindex c597d55c1..a88c4e66a 100644--- a/tests/test_learn/test_writer.py+++ b/tests/test_learn/test_writer.py@@ -9,6 +9,9 @@ from headroom.learn.writer import (_MARKER_END,_MARKER_START,ClaudeCodeWriter,+ CodexWriter,+ GeminiWriter,+ GrokWriter,_merge_into_file,_parse_prior_recommendations,_read_text_tolerant,@@ -570,3 +573,60 @@ class TestEncodingResilience:raw = memory_md.read_bytes()assert b"\r\r" not in rawassert raw.count(b"\r") == raw.count(b"\r\n")++[email protected]_newline+class TestNewlineContract:+ """Regression guard for #3594 / #3698 — every learn-writer write pins LF.++ This asserts the *call*, not the artifact, on purpose. ``Path.write_text``+ with ``newline=None`` translates ``\n`` through ``TextIOWrapper``, whose+ translation target is chosen at C-compile time (``#ifdef MS_WINDOWS``), not+ read from ``os.linesep`` at runtime. So on POSIX no fixture can make the+ unpinned call emit CRLF, and every artifact-level assertion here passes+ with the fix reverted. Spying on the kwarg fails the moment a pin is+ dropped, on any platform — which is the property #3698 asked for.+ """++ def test_every_learn_writer_write_pins_lf(self, tmp_path, monkeypatch):+ proj = _project(tmp_path)+ context_rec = _rec(RecommendationTarget.CONTEXT_FILE, "Environment", "- Use uv")+ memory_rec = _rec(RecommendationTarget.MEMORY_FILE, "Errors", "- rule 1")++ # Seed a legacy CLAUDE.md that carries a headroom block *and*+ # hand-written prose, so ClaudeCodeWriter's migration branch (which+ # rewrites the cleaned CLAUDE.md) is exercised alongside the rest.+ (proj.project_path / "CLAUDE.md").write_text(+ _legacy_block("Build Commands", "- cargo check"), encoding="utf-8"+ )++ calls: list[tuple[Path, str | None]] = []+ original = Path.write_text++ def spy(self, data, encoding=None, errors=None, newline=None):+ calls.append((self, newline))+ return original(self, data, encoding=encoding, errors=errors, newline=newline)++ monkeypatch.setattr(Path, "write_text", spy)++ ClaudeCodeWriter().write([context_rec, memory_rec], proj, dry_run=False)+ CodexWriter().write([context_rec, memory_rec], proj, dry_run=False)+ GeminiWriter().write([context_rec], proj, dry_run=False)+ GrokWriter().write([context_rec], proj, dry_run=False)++ assert calls, "no writes captured — this test no longer drives the writers"+ unpinned = sorted(str(path) for path, newline in calls if newline != "\n")+ assert not unpinned, f"learn writers wrote without newline='\\n': {unpinned}"++ # All seven write sites in headroom/learn/writer.py are reached above;+ # if a writer grows a new target, this set fails loudly rather than+ # letting an unguarded write site slip in.+ assert {path.name for path, _ in calls} == {+ "CLAUDE.local.md", # ClaudeCodeWriter context target+ "CLAUDE.md", # legacy-migration rewrite+ "MEMORY.md", # ClaudeCodeWriter memory target+ "AGENTS.md", # CodexWriter context target+ "instructions.md", # CodexWriter memory target+ "GEMINI.md", # GeminiWriter+ "GROK.md", # GrokWriter+ }diff --git a/tests/test_memory/test_writers.py b/tests/test_memory/test_writers.pyindex 7cf18591a..fae5f856f 100644--- a/tests/test_memory/test_writers.py+++ b/tests/test_memory/test_writers.py@@ -2,8 +2,13 @@from __future__ import annotations+import asyncioimport time+from datetime import datetime, timezonefrom pathlib import Path+from types import SimpleNamespace++import pytestfrom headroom.memory.writers.base import (MARKER_END,@@ -309,3 +314,100 @@ class TestGenericWriter:assert (tmp_path / "HEADROOM_MEMORY.md").exists()assert result.memories_exported == 3+++# =============================================================================+# LF newline contract (#3698)+# =============================================================================++[email protected]_newline+class TestNewlineContract:+ """Every memory write pins ``newline="\n"`` — see #3698.++ These files overlap with the learn writers (``CLAUDE.md``, ``AGENTS.md``,+ ``MEMORY.md``), so an unpinned write here flips a file the learn writers+ just wrote as LF back to CRLF on Windows, producing spurious whole-file+ diffs in any repo that commits them.++ Asserted at the call, not the artifact: ``TextIOWrapper`` picks its+ translation target at C-compile time (``#ifdef MS_WINDOWS``), so on POSIX+ an unpinned write is indistinguishable from a pinned one on disk.+ """++ @staticmethod+ def _spy(monkeypatch: pytest.MonkeyPatch) -> list[tuple[Path, str | None]]:+ calls: list[tuple[Path, str | None]] = []+ original = Path.write_text++ def spy(self, data, encoding=None, errors=None, newline=None):+ calls.append((self, newline))+ return original(self, data, encoding=encoding, errors=errors, newline=newline)++ monkeypatch.setattr(Path, "write_text", spy)+ return calls++ @staticmethod+ def _assert_all_lf(calls: list[tuple[Path, str | None]]) -> None:+ assert calls, "no writes captured — this test no longer drives the writers"+ unpinned = sorted(str(path) for path, newline in calls if newline != "\n")+ assert not unpinned, f"memory writers wrote without newline='\\n': {unpinned}"++ def test_memory_writers_pin_lf(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch):+ """base.py, cursor_writer.py and claude_writer.py write sites."""+ entries = _make_entries(6)+ calls = self._spy(monkeypatch)++ # base.MemoryWriter.export — shared by the codex/generic writers.+ CodexMemoryWriter(project_path=tmp_path).export(entries, dry_run=False)+ GenericMemoryWriter(project_path=tmp_path).export(entries, dry_run=False)+ # cursor_writer.export overrides base.export with its own write.+ CursorMemoryWriter(project_path=tmp_path).export(entries, dry_run=False)+ # claude_writer.export_topics writes per-category topic files.+ claude = ClaudeCodeMemoryWriter(project_path=tmp_path, memory_dir=tmp_path / "memory")+ claude.export(entries, dry_run=False)+ claude.export_topics(entries, dry_run=False)++ self._assert_all_lf(calls)+ names = {path.name for path, _ in calls}+ assert {"AGENTS.md", "HEADROOM_MEMORY.md", "headroom-memory.mdc", "MEMORY.md"} <= names+ assert any(name.startswith("headroom_") for name in names), names++ def test_bridge_writes_pin_lf(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch):+ """bridge.py write sites: markdown export, markdown append, sync state."""+ from headroom.memory.bridge import MemoryBridge+ from headroom.memory.bridge_config import BridgeConfig++ memories = [+ SimpleNamespace(+ content="use uv, not pip",+ metadata={"topic": "Environment"},+ entity_refs=[],+ importance=0.8,+ created_at=datetime(2026, 1, 1, tzinfo=timezone.utc),+ )+ ]++ async def get_user_memories(user_id, limit=200):+ return list(memories)++ config = BridgeConfig(sync_state_path=tmp_path / "state" / "bridge_state.json")+ bridge = MemoryBridge(+ config=config, backend=SimpleNamespace(get_user_memories=get_user_memories)+ )++ target = tmp_path / "CLAUDE.md"+ target.write_text("# Notes\n\n## Environment\n- existing\n", encoding="utf-8")++ calls = self._spy(monkeypatch)++ # export_to_markdown covers the export write *and* the sync-state write.+ asyncio.run(bridge.export_to_markdown(path=tmp_path / "export" / "MEMORY.md"))+ asyncio.run(bridge._append_to_markdown(target, memories))++ self._assert_all_lf(calls)+ assert {path.name for path, _ in calls} == {+ "MEMORY.md", # export_to_markdown+ "bridge_state.json", # _save_sync_state+ "CLAUDE.md", # _append_to_markdown+ }diff --git a/tests/test_memory_sync.py b/tests/test_memory_sync.pyindex fa6f8c03b..a53a1afb1 100644--- a/tests/test_memory_sync.py+++ b/tests/test_memory_sync.py@@ -24,6 +24,7 @@ import pytestfrom headroom.memory.sync import (_build_sync_backend,+ _save_sync_state,sync,sync_export,sync_import,@@ -758,3 +759,51 @@ def test_sync_backend_uses_onnx_embedder(tmp_path):"""backend = _build_sync_backend(str(tmp_path / "memory.db"))assert backend._config.embedder_backend == "onnx"++[email protected]_newline+class TestSyncNewlineContract:+ """Sync adapters and sync state pin ``newline="\n"`` — see #3698.++ AGENTS.md and the Claude memory files are also written by+ ``headroom/learn/writer.py``, which pins LF. An unpinned write here flips+ the same file back to CRLF on Windows on the next sync, so the two+ subsystems fight over the line endings of a committed file.++ Asserted at the call, not the artifact: ``TextIOWrapper`` picks its newline+ translation target at C-compile time (``#ifdef MS_WINDOWS``), so on POSIX+ an unpinned write looks identical on disk to a pinned one.+ """++ @pytest.mark.asyncio+ async def test_adapter_and_state_writes_pin_lf(self, tmp_path, monkeypatch):+ claude_dir = tmp_path / "claude"+ claude_dir.mkdir()+ (claude_dir / "MEMORY.md").write_text("# Memory\n\n## User\n- existing\n")+ agents_md = tmp_path / "AGENTS.md"+ agents_md.write_text("# Existing instructions\n")++ calls: list[tuple[Path, str | None]] = []+ original = Path.write_text++ def spy(self, data, encoding=None, errors=None, newline=None):+ calls.append((self, newline))+ return original(self, data, encoding=encoding, errors=errors, newline=newline)++ monkeypatch.setattr(Path, "write_text", spy)++ await ClaudeCodeAdapter(claude_dir).write_memories(+ [{"content": "Port 8787 is default", "id": "mem_0001"}]+ )+ await CodexAdapter(agents_md).write_memories([{"content": "Uses ruff for linting"}])+ _save_sync_state(tmp_path / "state" / "sync_state.json", {"version": 1})++ assert calls, "no writes captured — this test no longer drives the adapters"+ unpinned = sorted(str(path) for path, newline in calls if newline != "\n")+ assert not unpinned, f"sync writes without newline='\\n': {unpinned}"+ assert {path.name for path, _ in calls} >= {+ "MEMORY.md", # claude_code._update_memory_md_index+ "AGENTS.md", # codex_agent.write_memories+ "sync_state.json", # sync._save_sync_state+ }+ assert any(name.startswith("headroom_") for name in {p.name for p, _ in calls})