Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(memory): pin LF on memory writers and guard the learn-writer newline contract (#3706)

Tejas Choprab10dd8db6e71
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index fcd5074c4..637fd094f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -178,6 +178,59 @@ jobs:
path: dist/*.whl
retention-days: 1
+ test-windows-newline:
+ # No unit test has ever run on Windows (the `test` shards are
+ # ubuntu-only), which is how a Windows-only newline bug shipped twice
+ # (#3594, #3698). A full Windows shard is not worth the minutes; this
+ # runs only the handful of tests marked `windows_newline` against the
+ # wheel `build-wheel-windows` already produced. The marker keeps the set
+ # greppable — add `@pytest.mark.windows_newline` to a test and it runs
+ # here, no workflow edit needed.
+ #
+ # Gated on `packaging` because that is what builds the wheel this job
+ # consumes; on a pure-Python PR the ubuntu `test` shards still run the
+ # same marked tests (they assert the `newline=` kwarg, which fails on any
+ # platform), so this job is the Windows-side backstop, not the only guard.
+ needs: [changes, build-wheel-windows]
+ if: needs.changes.outputs.packaging == 'true'
+ runs-on: windows-latest
+ timeout-minutes: 30
+ steps:
+ - uses: actions/checkout@v7
+ - uses: actions/setup-python@v7
+ with:
+ python-version: ${{ env.PY_VERSION }}
+ - name: Download Windows wheel
+ uses: actions/download-artifact@v8
+ with:
+ name: headroom-wheel-windows
+ path: dist
+ - name: Install wheel + dev deps
+ shell: bash
+ run: |
+ python -m pip install --upgrade pip
+ WHEEL="$(ls dist/*.whl)"
+ pip install "${WHEEL}[dev]"
+ - name: Drop the source tree so the wheel is what gets tested
+ shell: bash
+ # `actions/checkout` leaves ./headroom/ on sys.path ahead of the
+ # installed wheel, and the source tree has no compiled
+ # `headroom/_core.abi3.so` -- so every module that reaches the Rust
+ # extension dies with `ModuleNotFoundError: No module named
+ # 'headroom._core'` (203 collection errors on the first run of this
+ # job). The release workflow's smoke-import sidesteps this by running
+ # from RUNNER_TEMP; here we need ./tests/, so remove the shadow
+ # instead. Testing the installed artifact rather than the source tree
+ # is the point of consuming the wheel anyway.
+ run: rm -rf headroom
+ - name: Run newline-contract tests
+ shell: bash
+ # Scoped to the trees that hold the marked tests rather than all of
+ # ./tests: collecting ~10k modules on Windows took 91s and pulled in
+ # unrelated import failures. Add a directory here if you mark a test
+ # outside these paths.
+ run: pytest tests/test_learn tests/test_memory tests/test_memory_sync.py -m windows_newline --tb=short -q
+
python-314-wheels:
# Partners on centrally managed machines cannot choose their Python, and
# many have no compiler. Install the proxy extras on the newest supported
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 775dc1691..48027f36d 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -123,6 +123,16 @@ Enable or disable automatic Copilot review in **Settings → Rules → Rulesets
- Type hints on public functions; Google-style docstrings.
- Cover new behavior + edge cases; aim >80% coverage on new code.
- Python 3.10+. Optional features go behind extras.
+- **Headroom writes LF and normalizes on read, on every platform.** Pass
+ `newline="\n"` to every `write_text`/`open` that writes a context, memory,
+ or state file, and normalize `\r\n`/`\r` when you read one back. Without
+ the pin, `TextIOWrapper` translates `\n` to `\r\n` on Windows, and a reader
+ that decodes bytes directly then re-writes accumulates carriage returns
+ (#3594); even with a normalizing reader, two subsystems writing the same
+ file (`CLAUDE.md`, `AGENTS.md`) flip it between LF and CRLF (#3698). Cover
+ new write sites with `@pytest.mark.windows_newline` — those tests assert the
+ `newline=` kwarg (an artifact assertion cannot fail on POSIX) and also run
+ on `windows-latest` in CI.
## Architecture principles
diff --git a/headroom/memory/bridge.py b/headroom/memory/bridge.py
index 7ea081d16..45e748a1e 100644
--- a/headroom/memory/bridge.py
+++ b/headroom/memory/bridge.py
@@ -126,6 +126,7 @@ class MemoryBridge:
path.write_text(
json.dumps(self._sync_state, indent=2, default=str),
encoding="utf-8",
+ newline="\n",
)
except OSError as e:
logger.warning(f"Bridge: Failed to save sync state to {path}: {e}")
@@ -390,7 +391,7 @@ class MemoryBridge:
if path:
path = Path(path).expanduser()
path.parent.mkdir(parents=True, exist_ok=True)
- path.write_text(markdown, encoding="utf-8")
+ path.write_text(markdown, encoding="utf-8", newline="\n")
logger.info(f"Bridge: Exported {len(memories)} memories to {path}")
# Update sync state to avoid re-importing our own export
@@ -646,7 +647,7 @@ class MemoryBridge:
try:
path.parent.mkdir(parents=True, exist_ok=True)
- path.write_text(new_content, encoding="utf-8")
+ path.write_text(new_content, encoding="utf-8", newline="\n")
except OSError as e:
logger.error(f"Bridge: Failed to write {path}: {e}")
return 0
diff --git a/headroom/memory/sync.py b/headroom/memory/sync.py
index ce4f0c596..0b3c80fd8 100644
--- a/headroom/memory/sync.py
+++ b/headroom/memory/sync.py
@@ -134,7 +134,7 @@ def _load_sync_state(state_path: Path) -> dict[str, Any]:
def _save_sync_state(state_path: Path, state: dict[str, Any]) -> None:
"""Save sync state to disk."""
state_path.parent.mkdir(parents=True, exist_ok=True)
- state_path.write_text(json.dumps(state, indent=2), encoding="utf-8")
+ state_path.write_text(json.dumps(state, indent=2), encoding="utf-8", newline="\n")
def _db_fingerprint(memories: list[Any]) -> str:
diff --git a/headroom/memory/sync_adapters/claude_code.py b/headroom/memory/sync_adapters/claude_code.py
index d26151db7..71607ada1 100644
--- a/headroom/memory/sync_adapters/claude_code.py
+++ b/headroom/memory/sync_adapters/claude_code.py
@@ -193,7 +193,7 @@ class ClaudeCodeAdapter(AgentMemoryAdapter):
"source_agent": source_agent,
}
)
- target.write_text(f"{fm}\n\n{content}\n", encoding="utf-8")
+ target.write_text(f"{fm}\n\n{content}\n", encoding="utf-8", newline="\n")
written += 1
# Track for MEMORY.md index
@@ -236,7 +236,7 @@ class ClaudeCodeAdapter(AgentMemoryAdapter):
else:
content = "# Memory\n" + new_section
- memory_md.write_text(content, encoding="utf-8")
+ memory_md.write_text(content, encoding="utf-8", newline="\n")
def fingerprint(self) -> str:
"""Hash of all .md filenames + contents for change detection."""
diff --git a/headroom/memory/sync_adapters/codex_agent.py b/headroom/memory/sync_adapters/codex_agent.py
index 01f9da07a..ae582bf22 100644
--- a/headroom/memory/sync_adapters/codex_agent.py
+++ b/headroom/memory/sync_adapters/codex_agent.py
@@ -121,7 +121,7 @@ class CodexAdapter(AgentMemoryAdapter):
self._path.parent.mkdir(parents=True, exist_ok=True)
content = section + "\n"
- self._path.write_text(content, encoding="utf-8")
+ self._path.write_text(content, encoding="utf-8", newline="\n")
return added
def fingerprint(self) -> str:
diff --git a/headroom/memory/writers/base.py b/headroom/memory/writers/base.py
index 5e8ed1636..7d9e961dd 100644
--- a/headroom/memory/writers/base.py
+++ b/headroom/memory/writers/base.py
@@ -165,7 +165,7 @@ class AgentWriter(ABC):
if not dry_run:
target.parent.mkdir(parents=True, exist_ok=True)
- target.write_text(full_content, encoding="utf-8")
+ target.write_text(full_content, encoding="utf-8", newline="\n")
return result
diff --git a/headroom/memory/writers/claude_writer.py b/headroom/memory/writers/claude_writer.py
index 44d971363..4d5c507b3 100644
--- a/headroom/memory/writers/claude_writer.py
+++ b/headroom/memory/writers/claude_writer.py
@@ -116,6 +116,6 @@ class ClaudeCodeMemoryWriter(AgentWriter):
if not dry_run:
memory_dir.mkdir(parents=True, exist_ok=True)
- (memory_dir / filename).write_text(content, encoding="utf-8")
+ (memory_dir / filename).write_text(content, encoding="utf-8", newline="\n")
return topic_files
diff --git a/headroom/memory/writers/cursor_writer.py b/headroom/memory/writers/cursor_writer.py
index ea77b537e..b9fd84b2b 100644
--- a/headroom/memory/writers/cursor_writer.py
+++ b/headroom/memory/writers/cursor_writer.py
@@ -127,6 +127,6 @@ class CursorMemoryWriter(AgentWriter):
if not dry_run:
target.parent.mkdir(parents=True, exist_ok=True)
- target.write_text(full_content, encoding="utf-8")
+ target.write_text(full_content, encoding="utf-8", newline="\n")
return result
diff --git a/pyproject.toml b/pyproject.toml
index f64c8ea68..19a3f5f56 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -532,6 +532,7 @@ markers = [
"real_llm: tests that hit real LLM APIs; skipped unless explicitly enabled",
"live: opt-in multi-turn tests that hit real upstream APIs; require provider keys",
"proxy_dependency_gate: exercises ensure_proxy_dependencies() without mocking",
+ "windows_newline: newline-contract tests (#3698); also run on windows-latest in CI",
]
[tool.coverage.run]
diff --git a/tests/test_learn/test_writer.py b/tests/test_learn/test_writer.py
index c597d55c1..a88c4e66a 100644
--- a/tests/test_learn/test_writer.py
+++ b/tests/test_learn/test_writer.py
@@ -9,6 +9,9 @@ from headroom.learn.writer import (
_MARKER_END,
_MARKER_START,
ClaudeCodeWriter,
+ CodexWriter,
+ GeminiWriter,
+ GrokWriter,
_merge_into_file,
_parse_prior_recommendations,
_read_text_tolerant,
@@ -570,3 +573,60 @@ class TestEncodingResilience:
raw = memory_md.read_bytes()
assert b"\r\r" not in raw
assert raw.count(b"\r") == raw.count(b"\r\n")
+
+
+class TestNewlineContract:
+ """Regression guard for #3594 / #3698 — every learn-writer write pins LF.
+
+ This asserts the *call*, not the artifact, on purpose. ``Path.write_text``
+ with ``newline=None`` translates ``\n`` through ``TextIOWrapper``, whose
+ translation target is chosen at C-compile time (``#ifdef MS_WINDOWS``), not
+ read from ``os.linesep`` at runtime. So on POSIX no fixture can make the
+ unpinned call emit CRLF, and every artifact-level assertion here passes
+ with the fix reverted. Spying on the kwarg fails the moment a pin is
+ dropped, on any platform — which is the property #3698 asked for.
+ """
+
+ def test_every_learn_writer_write_pins_lf(self, tmp_path, monkeypatch):
+ proj = _project(tmp_path)
+ context_rec = _rec(RecommendationTarget.CONTEXT_FILE, "Environment", "- Use uv")
+ memory_rec = _rec(RecommendationTarget.MEMORY_FILE, "Errors", "- rule 1")
+
+ # Seed a legacy CLAUDE.md that carries a headroom block *and*
+ # hand-written prose, so ClaudeCodeWriter's migration branch (which
+ # rewrites the cleaned CLAUDE.md) is exercised alongside the rest.
+ (proj.project_path / "CLAUDE.md").write_text(
+ _legacy_block("Build Commands", "- cargo check"), encoding="utf-8"
+ )
+
+ calls: list[tuple[Path, str | None]] = []
+ original = Path.write_text
+
+ def spy(self, data, encoding=None, errors=None, newline=None):
+ calls.append((self, newline))
+ return original(self, data, encoding=encoding, errors=errors, newline=newline)
+
+ monkeypatch.setattr(Path, "write_text", spy)
+
+ ClaudeCodeWriter().write([context_rec, memory_rec], proj, dry_run=False)
+ CodexWriter().write([context_rec, memory_rec], proj, dry_run=False)
+ GeminiWriter().write([context_rec], proj, dry_run=False)
+ GrokWriter().write([context_rec], proj, dry_run=False)
+
+ assert calls, "no writes captured — this test no longer drives the writers"
+ unpinned = sorted(str(path) for path, newline in calls if newline != "\n")
+ assert not unpinned, f"learn writers wrote without newline='\\n': {unpinned}"
+
+ # All seven write sites in headroom/learn/writer.py are reached above;
+ # if a writer grows a new target, this set fails loudly rather than
+ # letting an unguarded write site slip in.
+ assert {path.name for path, _ in calls} == {
+ "CLAUDE.local.md", # ClaudeCodeWriter context target
+ "CLAUDE.md", # legacy-migration rewrite
+ "MEMORY.md", # ClaudeCodeWriter memory target
+ "AGENTS.md", # CodexWriter context target
+ "instructions.md", # CodexWriter memory target
+ "GEMINI.md", # GeminiWriter
+ "GROK.md", # GrokWriter
+ }
diff --git a/tests/test_memory/test_writers.py b/tests/test_memory/test_writers.py
index 7cf18591a..fae5f856f 100644
--- a/tests/test_memory/test_writers.py
+++ b/tests/test_memory/test_writers.py
@@ -2,8 +2,13 @@
from __future__ import annotations
+import asyncio
import time
+from datetime import datetime, timezone
from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
from headroom.memory.writers.base import (
MARKER_END,
@@ -309,3 +314,100 @@ class TestGenericWriter:
assert (tmp_path / "HEADROOM_MEMORY.md").exists()
assert result.memories_exported == 3
+
+
+# =============================================================================
+# LF newline contract (#3698)
+# =============================================================================
+
+
+class TestNewlineContract:
+ """Every memory write pins ``newline="\n"`` — see #3698.
+
+ These files overlap with the learn writers (``CLAUDE.md``, ``AGENTS.md``,
+ ``MEMORY.md``), so an unpinned write here flips a file the learn writers
+ just wrote as LF back to CRLF on Windows, producing spurious whole-file
+ diffs in any repo that commits them.
+
+ Asserted at the call, not the artifact: ``TextIOWrapper`` picks its
+ translation target at C-compile time (``#ifdef MS_WINDOWS``), so on POSIX
+ an unpinned write is indistinguishable from a pinned one on disk.
+ """
+
+ @staticmethod
+ def _spy(monkeypatch: pytest.MonkeyPatch) -> list[tuple[Path, str | None]]:
+ calls: list[tuple[Path, str | None]] = []
+ original = Path.write_text
+
+ def spy(self, data, encoding=None, errors=None, newline=None):
+ calls.append((self, newline))
+ return original(self, data, encoding=encoding, errors=errors, newline=newline)
+
+ monkeypatch.setattr(Path, "write_text", spy)
+ return calls
+
+ @staticmethod
+ def _assert_all_lf(calls: list[tuple[Path, str | None]]) -> None:
+ assert calls, "no writes captured — this test no longer drives the writers"
+ unpinned = sorted(str(path) for path, newline in calls if newline != "\n")
+ assert not unpinned, f"memory writers wrote without newline='\\n': {unpinned}"
+
+ def test_memory_writers_pin_lf(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
+ """base.py, cursor_writer.py and claude_writer.py write sites."""
+ entries = _make_entries(6)
+ calls = self._spy(monkeypatch)
+
+ # base.MemoryWriter.export — shared by the codex/generic writers.
+ CodexMemoryWriter(project_path=tmp_path).export(entries, dry_run=False)
+ GenericMemoryWriter(project_path=tmp_path).export(entries, dry_run=False)
+ # cursor_writer.export overrides base.export with its own write.
+ CursorMemoryWriter(project_path=tmp_path).export(entries, dry_run=False)
+ # claude_writer.export_topics writes per-category topic files.
+ claude = ClaudeCodeMemoryWriter(project_path=tmp_path, memory_dir=tmp_path / "memory")
+ claude.export(entries, dry_run=False)
+ claude.export_topics(entries, dry_run=False)
+
+ self._assert_all_lf(calls)
+ names = {path.name for path, _ in calls}
+ assert {"AGENTS.md", "HEADROOM_MEMORY.md", "headroom-memory.mdc", "MEMORY.md"} <= names
+ assert any(name.startswith("headroom_") for name in names), names
+
+ def test_bridge_writes_pin_lf(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
+ """bridge.py write sites: markdown export, markdown append, sync state."""
+ from headroom.memory.bridge import MemoryBridge
+ from headroom.memory.bridge_config import BridgeConfig
+
+ memories = [
+ SimpleNamespace(
+ content="use uv, not pip",
+ metadata={"topic": "Environment"},
+ entity_refs=[],
+ importance=0.8,
+ created_at=datetime(2026, 1, 1, tzinfo=timezone.utc),
+ )
+ ]
+
+ async def get_user_memories(user_id, limit=200):
+ return list(memories)
+
+ config = BridgeConfig(sync_state_path=tmp_path / "state" / "bridge_state.json")
+ bridge = MemoryBridge(
+ config=config, backend=SimpleNamespace(get_user_memories=get_user_memories)
+ )
+
+ target = tmp_path / "CLAUDE.md"
+ target.write_text("# Notes\n\n## Environment\n- existing\n", encoding="utf-8")
+
+ calls = self._spy(monkeypatch)
+
+ # export_to_markdown covers the export write *and* the sync-state write.
+ asyncio.run(bridge.export_to_markdown(path=tmp_path / "export" / "MEMORY.md"))
+ asyncio.run(bridge._append_to_markdown(target, memories))
+
+ self._assert_all_lf(calls)
+ assert {path.name for path, _ in calls} == {
+ "MEMORY.md", # export_to_markdown
+ "bridge_state.json", # _save_sync_state
+ "CLAUDE.md", # _append_to_markdown
+ }
diff --git a/tests/test_memory_sync.py b/tests/test_memory_sync.py
index fa6f8c03b..a53a1afb1 100644
--- a/tests/test_memory_sync.py
+++ b/tests/test_memory_sync.py
@@ -24,6 +24,7 @@ import pytest
from headroom.memory.sync import (
_build_sync_backend,
+ _save_sync_state,
sync,
sync_export,
sync_import,
@@ -758,3 +759,51 @@ def test_sync_backend_uses_onnx_embedder(tmp_path):
"""
backend = _build_sync_backend(str(tmp_path / "memory.db"))
assert backend._config.embedder_backend == "onnx"
+
+
+class TestSyncNewlineContract:
+ """Sync adapters and sync state pin ``newline="\n"`` — see #3698.
+
+ AGENTS.md and the Claude memory files are also written by
+ ``headroom/learn/writer.py``, which pins LF. An unpinned write here flips
+ the same file back to CRLF on Windows on the next sync, so the two
+ subsystems fight over the line endings of a committed file.
+
+ Asserted at the call, not the artifact: ``TextIOWrapper`` picks its newline
+ translation target at C-compile time (``#ifdef MS_WINDOWS``), so on POSIX
+ an unpinned write looks identical on disk to a pinned one.
+ """
+
+ @pytest.mark.asyncio
+ async def test_adapter_and_state_writes_pin_lf(self, tmp_path, monkeypatch):
+ claude_dir = tmp_path / "claude"
+ claude_dir.mkdir()
+ (claude_dir / "MEMORY.md").write_text("# Memory\n\n## User\n- existing\n")
+ agents_md = tmp_path / "AGENTS.md"
+ agents_md.write_text("# Existing instructions\n")
+
+ calls: list[tuple[Path, str | None]] = []
+ original = Path.write_text
+
+ def spy(self, data, encoding=None, errors=None, newline=None):
+ calls.append((self, newline))
+ return original(self, data, encoding=encoding, errors=errors, newline=newline)
+
+ monkeypatch.setattr(Path, "write_text", spy)
+
+ await ClaudeCodeAdapter(claude_dir).write_memories(
+ [{"content": "Port 8787 is default", "id": "mem_0001"}]
+ )
+ await CodexAdapter(agents_md).write_memories([{"content": "Uses ruff for linting"}])
+ _save_sync_state(tmp_path / "state" / "sync_state.json", {"version": 1})
+
+ assert calls, "no writes captured — this test no longer drives the adapters"
+ unpinned = sorted(str(path) for path, newline in calls if newline != "\n")
+ assert not unpinned, f"sync writes without newline='\\n': {unpinned}"
+ assert {path.name for path, _ in calls} >= {
+ "MEMORY.md", # claude_code._update_memory_md_index
+ "AGENTS.md", # codex_agent.write_memories
+ "sync_state.json", # sync._save_sync_state
+ }
+ assert any(name.startswith("headroom_") for name in {p.name for p, _ in calls})