Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Fix `headroom learn` crashing/no-op on Windows from missing UTF-8 encoding (#1239)
jichaowang02-lang6129808462f4
diff --git a/headroom/cli/learn.py b/headroom/cli/learn.pyindex 2b69d89c9..6d5a19981 100644--- a/headroom/cli/learn.py+++ b/headroom/cli/learn.py@@ -236,9 +236,15 @@ def learn(click.echo(f"Path: {proj.project_path}")click.echo(f"{'=' * 60}")- sessions = plugin.scan_project(- proj, max_workers=max_workers, include_subagents=not main_only- )+ try:+ sessions = plugin.scan_project(+ proj, max_workers=max_workers, include_subagents=not main_only+ )+ except Exception as exc:+ # One unreadable agent/project must not abort the whole+ # cross-agent run; skip it with a warning and continue.+ click.echo(f" Skipping (could not scan sessions): {exc}")+ continueif not sessions:click.echo(" No conversation data found.")continuediff --git a/headroom/learn/analyzer.py b/headroom/learn/analyzer.pyindex e939aea3c..110ba6046 100644--- a/headroom/learn/analyzer.py+++ b/headroom/learn/analyzer.py@@ -198,7 +198,7 @@ def _build_prior_patterns_section(project: ProjectInfo) -> str:for label, path in candidates:if path is None or not path.exists():continue- block = extract_marker_block(path.read_text())+ block = extract_marker_block(path.read_text(encoding="utf-8", errors="replace"))if block:parts.append((label, block))@@ -486,6 +486,8 @@ def _call_cli_llm(digest: str, model: str) -> dict:input=prompt,capture_output=True,text=True,+ encoding="utf-8",+ errors="replace",timeout=hard_cap,)except FileNotFoundError:@@ -541,6 +543,8 @@ def _call_claude_cli_streaming(stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,+ encoding="utf-8",+ errors="replace",bufsize=1, # line-buffered)except FileNotFoundError:diff --git a/headroom/learn/plugins/claude.py b/headroom/learn/plugins/claude.pyindex 8cc5f7dcf..c790bfdf1 100644--- a/headroom/learn/plugins/claude.py+++ b/headroom/learn/plugins/claude.py@@ -167,7 +167,7 @@ class ClaudeCodePlugin(LearnPlugin, ConversationScanner):msg_index = 0try:- with open(jsonl_path) as f:+ with open(jsonl_path, encoding="utf-8", errors="replace") as f:for line in f:try:d = json.loads(line)diff --git a/headroom/learn/plugins/codex.py b/headroom/learn/plugins/codex.pyindex 8b70ae748..4908aa74d 100644--- a/headroom/learn/plugins/codex.py+++ b/headroom/learn/plugins/codex.py@@ -126,7 +126,7 @@ class CodexPlugin(LearnPlugin, ConversationScanner):def _scan_json_session(self, json_path: Path) -> SessionData | None:"""Parse a single Codex session file."""try:- with open(json_path) as f:+ with open(json_path, encoding="utf-8", errors="replace") as f:data = json.load(f)except (OSError, json.JSONDecodeError) as e:logger.debug("Failed to read Codex session %s: %s", json_path, e)@@ -209,7 +209,7 @@ class CodexPlugin(LearnPlugin, ConversationScanner):msg_index = 0try:- with open(jsonl_path) as f:+ with open(jsonl_path, encoding="utf-8", errors="replace") as f:for line in f:try:entry = json.loads(line)diff --git a/headroom/learn/plugins/gemini.py b/headroom/learn/plugins/gemini.pyindex 7f8a1b901..15e589a68 100644--- a/headroom/learn/plugins/gemini.py+++ b/headroom/learn/plugins/gemini.py@@ -143,7 +143,7 @@ class GeminiPlugin(LearnPlugin, ConversationScanner):def _scan_json_session(self, json_path: Path) -> SessionData | None:"""Parse a Gemini JSON session file."""try:- with open(json_path) as f:+ with open(json_path, encoding="utf-8", errors="replace") as f:data = json.load(f)except (OSError, json.JSONDecodeError) as e:logger.debug("Failed to read Gemini session %s: %s", json_path, e)@@ -167,7 +167,7 @@ class GeminiPlugin(LearnPlugin, ConversationScanner):messages: list[dict] = []try:- with open(jsonl_path) as f:+ with open(jsonl_path, encoding="utf-8", errors="replace") as f:for line in f:try:entry = json.loads(line)@@ -314,7 +314,7 @@ class GeminiPlugin(LearnPlugin, ConversationScanner):def _detect_project_path(self, session_path: Path) -> Path | None:"""Try to detect the project path from a session file."""try:- with open(session_path) as f:+ with open(session_path, encoding="utf-8", errors="replace") as f:data = json.load(f)except (OSError, json.JSONDecodeError):return Nonediff --git a/headroom/learn/writer.py b/headroom/learn/writer.pyindex ddad3e529..ade64a02a 100644--- a/headroom/learn/writer.py+++ b/headroom/learn/writer.py@@ -26,6 +26,23 @@ _MARKER_PATTERN = re.compile()+def _read_text_tolerant(file_path: Path) -> str:+ """Read an existing context file that we are about to rewrite as UTF-8.++ These files are predominantly valid UTF-8 but may carry a stray legacy+ byte (e.g. a cp1252 em-dash ``0x97``). Strict UTF-8 decoding aborts the+ whole ``--apply`` on a single such byte, so fall back to UTF-8 with+ replacement: this preserves the valid UTF-8 content — a full-file cp1252+ fallback would instead turn every genuine UTF-8 em-dash into mojibake —+ and the subsequent ``write_text(encoding="utf-8")`` self-heals the file.+ """+ raw = file_path.read_bytes()+ try:+ return raw.decode("utf-8")+ except UnicodeDecodeError:+ return raw.decode("utf-8", errors="replace")++# =============================================================================# Abstract Writer# =============================================================================@@ -153,7 +170,7 @@ def _merge_recommendations("""if not file_path.exists():return new_recommendations- prior = _parse_prior_recommendations(file_path.read_text(encoding="utf-8"))+ prior = _parse_prior_recommendations(_read_text_tolerant(file_path))if not prior:return new_recommendationsnew_sections = {r.section for r in new_recommendations}@@ -166,7 +183,7 @@ def _merge_into_file(file_path: Path, new_recommendations: list[Recommendation])merged = _merge_recommendations(file_path, new_recommendations)section = _build_section(merged)if file_path.exists():- existing = file_path.read_text(encoding="utf-8")+ existing = _read_text_tolerant(file_path)if _MARKER_START in existing:return _MARKER_PATTERN.sub(lambda _match: section, existing)return existing.rstrip() + "\n\n" + section + "\n"diff --git a/tests/test_learn/test_plugin_encoding.py b/tests/test_learn/test_plugin_encoding.pynew file mode 100644index 000000000..6c15ea7ae--- /dev/null+++ b/tests/test_learn/test_plugin_encoding.py@@ -0,0 +1,51 @@+"""Regression tests for #1202 — the ``learn`` session scanners must read agent+transcripts as UTF-8 with replacement, so a stray non-UTF-8 byte cannot abort+(or silently drop) a scan.++``0x9d`` is undefined in cp1252 *and* an invalid UTF-8 start byte, so a bare+``open()`` fails on it regardless of the host locale. Before the fix this made+the Codex JSONL scanner raise ``UnicodeDecodeError`` (the scan caught only+``OSError``), aborting the whole cross-agent run, while the Claude scanner+caught it and silently dropped the session.+"""++from __future__ import annotations++import json+from pathlib import Path++from headroom.learn.models import SessionData+from headroom.learn.plugins.claude import ClaudeCodePlugin+from headroom.learn.plugins.codex import CodexPlugin+++def _stray_byte_line() -> bytes:+ # A line that is neither valid UTF-8 nor decodable in cp1252.+ return b"\x9d arrow \xe2\x86\x92 junk\n"+++def test_claude_scan_recovers_session_with_stray_byte(tmp_path: Path) -> None:+ jsonl = tmp_path / "session.jsonl"+ valid = json.dumps(+ {"type": "assistant", "message": {"usage": {"input_tokens": 5}}, "text": "em — arrow →"}+ )+ jsonl.write_bytes(valid.encode() + b"\n" + _stray_byte_line())++ result = ClaudeCodePlugin(claude_dir=tmp_path)._scan_session(jsonl)++ # Before the fix this returned None (session silently dropped); now the+ # valid line is read and the stray-byte line is skipped, not fatal.+ assert result is not None+ assert result.session_id == "session"+ assert result.total_input_tokens == 5+++def test_codex_jsonl_scan_does_not_crash_on_stray_byte(tmp_path: Path) -> None:+ jsonl = tmp_path / "rollout.jsonl"+ meta = json.dumps({"type": "session_meta", "payload": {"id": "abc"}})+ jsonl.write_bytes(meta.encode() + b"\n" + _stray_byte_line())++ # Before the fix this raised UnicodeDecodeError and aborted the run.+ result = CodexPlugin()._scan_jsonl_session(jsonl)++ assert result is None or isinstance(result, SessionData)diff --git a/tests/test_learn/test_writer.py b/tests/test_learn/test_writer.pyindex 80b98ad40..8e4ef956a 100644--- a/tests/test_learn/test_writer.py+++ b/tests/test_learn/test_writer.py@@ -2,6 +2,8 @@from pathlib import Path+import pytest+from headroom.learn.models import ProjectInfo, Recommendation, RecommendationTargetfrom headroom.learn.writer import (_MARKER_END,@@ -9,6 +11,7 @@ from headroom.learn.writer import (ClaudeCodeWriter,_merge_into_file,_parse_prior_recommendations,+ _read_text_tolerant,extract_marker_block,)@@ -308,3 +311,41 @@ class TestExtractMarkerBlock:block = extract_marker_block(content)assert block is not Noneassert block == f"{_MARKER_START}\n{_MARKER_END}"+++class TestEncodingResilience:+ """Regression tests for #1202 — ``learn --apply`` must not crash merging into+ an existing context file that carries a stray non-UTF-8 byte (e.g. a legacy+ cp1252 em-dash ``0x97``)."""++ def test_read_text_tolerant_preserves_valid_utf8(self, tmp_path):+ path = tmp_path / "AGENTS.md"+ path.write_text("Use em-dashes — and arrows →.", encoding="utf-8")+ assert _read_text_tolerant(path) == "Use em-dashes — and arrows →."++ def test_read_text_tolerant_survives_stray_legacy_byte(self, tmp_path):+ # Predominantly valid UTF-8 (genuine em-dash E2 80 94) plus one stray+ # cp1252 em-dash byte (0x97) that strict UTF-8 cannot decode.+ path = tmp_path / "AGENTS.md"+ path.write_bytes("real em-dash — here\n".encode() + b"legacy \x97 byte\n")++ # The old strict read aborts the whole --apply on that single byte.+ with pytest.raises(UnicodeDecodeError):+ path.read_text(encoding="utf-8")++ text = _read_text_tolerant(path)+ # Valid UTF-8 content is preserved (no cp1252 "â€" mojibake) and the+ # stray byte is replaced rather than fatal.+ assert "real em-dash — here" in text+ assert "\x97" not in text+ assert "â€" not in text++ def test_merge_into_file_applies_over_file_with_stray_byte(self, tmp_path):+ path = tmp_path / "AGENTS.md"+ path.write_bytes("# Notes — existing\n".encode() + b"stray \x97 byte\n")+ recs = [_rec(RecommendationTarget.CONTEXT_FILE, "Environment", "- Use uv")]++ merged = _merge_into_file(path, recs)++ assert "Use uv" in merged+ assert "Notes — existing" in merged