Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(io): use UTF-8 with locale fallback and preserve line endings on config/text I/O (#1498)
Parideboy1baa04ef6576
diff --git a/headroom/cli/mcp.py b/headroom/cli/mcp.pyindex 2f83aea96..b5aede3a5 100644--- a/headroom/cli/mcp.py+++ b/headroom/cli/mcp.py@@ -35,7 +35,7 @@ def load_mcp_config() -> dict[str, Any]:"""Load existing MCP config or return empty structure."""if MCP_CONFIG_PATH.exists():try:- with open(MCP_CONFIG_PATH) as f:+ with open(MCP_CONFIG_PATH, encoding="utf-8") as f:result: dict[str, Any] = json.load(f)return resultexcept (json.JSONDecodeError, OSError):@@ -46,7 +46,7 @@ def load_mcp_config() -> dict[str, Any]:def save_mcp_config(config: dict) -> None:"""Save MCP config, creating directory if needed."""CLAUDE_CONFIG_DIR.mkdir(parents=True, exist_ok=True)- with open(MCP_CONFIG_PATH, "w") as f:+ with open(MCP_CONFIG_PATH, "w", encoding="utf-8") as f:json.dump(config, f, indent=2)f.write("\n") # Trailing newlinediff --git a/headroom/cli/memory.py b/headroom/cli/memory.pyindex c02798c18..91e717c94 100644--- a/headroom/cli/memory.py+++ b/headroom/cli/memory.py@@ -12,6 +12,7 @@ from typing import Anyimport click+from .. import fsutilfrom ..memory.adapters.sqlite import SQLiteMemoryStorefrom ..memory.models import Memory, ScopeLevelfrom ..memory.ports import MemoryFilter@@ -874,7 +875,7 @@ def export_memories(ctx: click.Context, db_path: str, output: str | None) -> Nonif output:output_path = Path(output)- output_path.write_text(json_output)+ fsutil.write_text(output_path, json_output)print_success(f"Exported {len(memories)} memory(ies) to {output_path}")else:click.echo(json_output)@@ -904,7 +905,7 @@ def import_memories(ctx: click.Context, db_path: str, file: str, force: bool) ->try:# Read and parse file- content = file_path.read_text()+ content = fsutil.read_text(file_path)memories_data = json.loads(content)if not isinstance(memories_data, list):diff --git a/headroom/cli/wrap.py b/headroom/cli/wrap.pyindex 51b280faf..b7ff1cd36 100644--- a/headroom/cli/wrap.py+++ b/headroom/cli/wrap.py@@ -43,6 +43,7 @@ if sys.platform == "win32" and hasattr(sys.stdout, "buffer"):import click+from headroom import fsutilfrom headroom._version import __version__ as _HEADROOM_VERSIONfrom headroom.agent_savings import (apply_agent_savings_env_defaults,@@ -128,19 +129,18 @@ from .main import maindef _read_text(path: Path) -> str:- """Read a text file with explicit UTF-8 encoding."""- return path.read_text(encoding="utf-8")+ """Read a text file as UTF-8, falling back to the system locale encoding."""+ return fsutil.read_text(path)def _write_text(path: Path, content: str) -> None:- """Write a text file with explicit UTF-8 encoding."""- path.write_text(content, encoding="utf-8")+ """Write a text file as UTF-8 without translating line endings (preserves CRLF)."""+ fsutil.write_text(path, content)def _append_text(path: Path, content: str) -> None:- """Append to a text file with explicit UTF-8 encoding."""- with open(path, "a", encoding="utf-8") as f:- f.write(content)+ """Append to a text file as UTF-8 without translating line endings."""+ fsutil.append_text(path, content)_CONTEXT_TOOL_ENV = "HEADROOM_CONTEXT_TOOL"@@ -556,7 +556,7 @@ def _patch_rtk_hook_absolute_path(rtk_path: Path, hook_script_path: Path | Noneif not hook_script_path.exists():return False- original = hook_script_path.read_text()+ original = _read_text(hook_script_path)# Quote the absolute path safely for POSIX shells. This matters because# paths containing spaces or other shell-special characters (e.g.@@ -576,7 +576,7 @@ def _patch_rtk_hook_absolute_path(rtk_path: Path, hook_script_path: Path | None)if count and patched != original:- hook_script_path.write_text(patched)+ _write_text(hook_script_path, patched)return Truereturn Falsediff --git a/headroom/fsutil.py b/headroom/fsutil.pynew file mode 100644index 000000000..b88fb4d99--- /dev/null+++ b/headroom/fsutil.py@@ -0,0 +1,81 @@+"""Encoding- and newline-safe text file I/O.++``Path.read_text()`` / ``Path.write_text()`` and the builtin ``open()`` default+to the *system locale* encoding and, in text mode, translate ``\\n`` to+``os.linesep`` on write. On non-UTF-8 Windows locales (e.g. GBK / cp936 on+zh-CN) this corrupts config files two ways:++1. Reading a UTF-8 file as GBK — or a GBK file as UTF-8 — raises+ ``UnicodeDecodeError``.+2. Writing re-translates ``\\n`` to ``\\r\\n``; content that already has+ ``\\r\\n`` becomes ``\\r\\r\\n``, which TOML parsers reject with+ "carriage return must be followed by newline".++These helpers always use UTF-8, fall back to the locale encoding when a file+predates the fix (tools may have written it in the locale encoding), and write+with ``newline=""`` so existing line endings pass through unchanged.++See issue #733.+"""++from __future__ import annotations++import locale+import os+from pathlib import Path++# Sentinel so ``default=None`` can be a real return value if a caller wants it.+_RAISE = object()+++def read_text(path: str | os.PathLike[str], *, default: object = _RAISE) -> str:+ """Read text, preferring UTF-8 and falling back to the locale encoding.++ Decoding order: UTF-8 (strict) → locale preferred encoding (strict) →+ UTF-8 with ``errors="replace"`` (never raises on content). If the file+ cannot be opened (missing/unreadable) and ``default`` is given, it is+ returned; otherwise the ``OSError`` propagates.++ Line endings are normalised to ``\\n`` (universal-newline semantics,+ matching the stdlib text-mode default) so callers that search or rewrite+ the text work on a single ending, and a later :func:`write_text` cannot+ re-double an existing ``\\r\\n``.+ """+ try:+ raw = Path(path).read_bytes()+ except OSError:+ if default is not _RAISE:+ return default # type: ignore[return-value]+ raise++ text: str | None = None+ try:+ text = raw.decode("utf-8")+ except UnicodeDecodeError:+ loc = locale.getpreferredencoding(False)+ if loc and loc.lower().replace("-", "") != "utf8":+ try:+ text = raw.decode(loc)+ except (UnicodeDecodeError, LookupError):+ text = None+ if text is None:+ text = raw.decode("utf-8", errors="replace")++ return text.replace("\r\n", "\n").replace("\r", "\n")+++def write_text(path: str | os.PathLike[str], content: str) -> None:+ """Write text as UTF-8 without translating line endings.++ ``newline=""`` disables the platform ``\\n`` → ``\\r\\n`` rewrite, so the+ bytes written match ``content`` exactly and existing ``\\r\\n`` endings are+ never doubled.+ """+ with Path(path).open("w", encoding="utf-8", newline="") as f:+ f.write(content)+++def append_text(path: str | os.PathLike[str], content: str) -> None:+ """Append text as UTF-8 without translating line endings (see ``write_text``)."""+ with Path(path).open("a", encoding="utf-8", newline="") as f:+ f.write(content)diff --git a/headroom/install/providers.py b/headroom/install/providers.pyindex a976411d5..fd1fabfd8 100644--- a/headroom/install/providers.py+++ b/headroom/install/providers.py@@ -7,6 +7,7 @@ import reimport subprocessfrom pathlib import Path+from headroom import fsutilfrom headroom._subprocess import runfrom headroom.providers.install_registry import (apply_provider_scope_mutations,@@ -29,7 +30,7 @@ _ENV_PATTERN = re.compile(def _merge_marker_block(file_path: Path, block: str, pattern: re.Pattern[str], marker: str) -> str:if file_path.exists():- existing = file_path.read_text()+ existing = fsutil.read_text(file_path)if marker in existing:return pattern.sub(block, existing)return existing.rstrip() + "\n\n" + block + "\n"@@ -66,7 +67,7 @@ def _apply_unix_env_scope(manifest: DeploymentManifest) -> list[ManagedMutation]for path in targets:path.parent.mkdir(parents=True, exist_ok=True)merged = _merge_marker_block(path, block, _ENV_PATTERN, _ENV_MARKER_START)- path.write_text(merged)+ fsutil.write_text(path, merged)mutations.append(ManagedMutation(target="env", kind="shell-block", path=str(path)))return mutations@@ -78,10 +79,10 @@ def _remove_unix_env_scope(mutations: list[ManagedMutation]) -> None:path = Path(mutation.path)if not path.exists():continue- content = path.read_text()+ content = fsutil.read_text(path)if _ENV_MARKER_START not in content:continue- path.write_text(_ENV_PATTERN.sub("", content).strip() + "\n")+ fsutil.write_text(path, _ENV_PATTERN.sub("", content).strip() + "\n")def _apply_windows_env_scope(manifest: DeploymentManifest) -> list[ManagedMutation]:diff --git a/headroom/mcp_registry/claude.py b/headroom/mcp_registry/claude.pyindex 939a6c141..c537e679b 100644--- a/headroom/mcp_registry/claude.py+++ b/headroom/mcp_registry/claude.py@@ -213,7 +213,7 @@ def _read_json(path: Path) -> dict[str, Any]:if not path.exists():return {}try:- with open(path) as f:+ with open(path, encoding="utf-8") as f:data = json.load(f)except (OSError, json.JSONDecodeError):return {}@@ -224,7 +224,7 @@ def _read_json(path: Path) -> dict[str, Any]:def _write_json(path: Path, data: dict[str, Any]) -> None:path.parent.mkdir(parents=True, exist_ok=True)- with open(path, "w") as f:+ with open(path, "w", encoding="utf-8") as f:json.dump(data, f, indent=2)f.write("\n")diff --git a/headroom/mcp_registry/codex.py b/headroom/mcp_registry/codex.pyindex e4b675d64..190ad96df 100644--- a/headroom/mcp_registry/codex.py+++ b/headroom/mcp_registry/codex.py@@ -17,6 +17,8 @@ import sysfrom pathlib import Pathfrom typing import Any+from headroom import fsutil+from .base import MCPRegistrar, RegisterResult, RegisterStatus, ServerSpecif sys.version_info >= (3, 11):@@ -132,7 +134,7 @@ class CodexRegistrar(MCPRegistrar):else:new_content = (before or after).rstrip("\n") + ("\n" if (before or after) else "")try:- self._config_file.write_text(new_content)+ fsutil.write_text(self._config_file, new_content)except OSError:return Falsereturn True@@ -145,17 +147,16 @@ class CodexRegistrar(MCPRegistrar):if not self._config_file.exists():return {}try:- with open(self._config_file, "rb") as f:- data = tomllib.load(f)+ # Read via fsutil (UTF-8 with locale fallback) so a config that a+ # tool wrote in the system locale (e.g. GBK) still parses instead+ # of failing tomllib's UTF-8 requirement. See #733.+ data = tomllib.loads(fsutil.read_text(self._config_file))except (tomllib.TOMLDecodeError, OSError):return {}return data if isinstance(data, dict) else {}def _read_text(self) -> str:- try:- return self._config_file.read_text()- except OSError:- return ""+ return fsutil.read_text(self._config_file, default="")def _write_block(self, spec: ServerSpec) -> RegisterResult:block = _render_block(spec)@@ -178,7 +179,7 @@ class CodexRegistrar(MCPRegistrar):content = content.rstrip("\n") + "\n\n" + block + "\n"else:content = block + "\n"- self._config_file.write_text(content)+ fsutil.write_text(self._config_file, content)except OSError as exc:return RegisterResult(RegisterStatus.FAILED, f"could not write {self._config_file}: {exc}"diff --git a/headroom/mcp_registry/opencode.py b/headroom/mcp_registry/opencode.pyindex 93dda3e6d..6ab6cca91 100644--- a/headroom/mcp_registry/opencode.py+++ b/headroom/mcp_registry/opencode.py@@ -39,7 +39,7 @@ def _read_json(path: Path) -> dict[str, Any]:if not path.exists():return {}try:- with open(path) as f:+ with open(path, encoding="utf-8") as f:data = json.load(f)except (OSError, json.JSONDecodeError):return {}@@ -50,7 +50,7 @@ def _read_json(path: Path) -> dict[str, Any]:def _write_json(path: Path, data: dict[str, Any]) -> None:path.parent.mkdir(parents=True, exist_ok=True)- with open(path, "w") as f:+ with open(path, "w", encoding="utf-8") as f:json.dump(data, f, indent=2)f.write("\n")diff --git a/headroom/providers/anthropic.py b/headroom/providers/anthropic.pyindex 93d3d1bd6..e1ff0126c 100644--- a/headroom/providers/anthropic.py+++ b/headroom/providers/anthropic.py@@ -199,7 +199,7 @@ def _load_custom_model_config() -> dict[str, Any]:try:# Check if it's a file pathif os.path.isfile(env_config):- with open(env_config) as f:+ with open(env_config, encoding="utf-8") as f:loaded = json.load(f)else:# Try to parse as JSON string@@ -225,7 +225,7 @@ def _load_custom_model_config() -> dict[str, Any]:config_file = legacy_modelsif config_file.exists():try:- with open(config_file) as f:+ with open(config_file, encoding="utf-8") as f:loaded = json.load(f)# Only load anthropic-specific configdiff --git a/headroom/providers/openai.py b/headroom/providers/openai.pyindex b4c688b15..320fb59e6 100644--- a/headroom/providers/openai.py+++ b/headroom/providers/openai.py@@ -157,7 +157,7 @@ def _load_custom_model_config() -> dict[str, Any]:try:# Check if it's a file pathif os.path.isfile(env_config):- with open(env_config) as f:+ with open(env_config, encoding="utf-8") as f:loaded = json.load(f)else:# Try to parse as JSON string@@ -184,7 +184,7 @@ def _load_custom_model_config() -> dict[str, Any]:config_file = legacy_modelsif config_file.exists():try:- with open(config_file) as f:+ with open(config_file, encoding="utf-8") as f:loaded = json.load(f)openai_config = loaded.get("openai", {})diff --git a/headroom/providers/opencode/config.py b/headroom/providers/opencode/config.pyindex 3579fabbf..097022dcc 100644--- a/headroom/providers/opencode/config.py+++ b/headroom/providers/opencode/config.py@@ -11,6 +11,7 @@ from typing import Anyimport click+from headroom import fsutilfrom headroom.install.paths import opencode_config_pathfrom headroom.mcp_registry.install import DEFAULT_PROXY_URL@@ -58,7 +59,7 @@ def snapshot_opencode_config_if_unwrapped(config_file: Path, backup_file: Path)if not config_file.exists():returntry:- content = config_file.read_text()+ content = fsutil.read_text(config_file)except OSError:returnif _PROVIDER_MARKER_START in content or _MCP_MARKER_START in content:@@ -190,7 +191,7 @@ def inject_opencode_provider_config(port: int) -> None:snapshot_opencode_config_if_unwrapped(config_file, backup_file)if config_file.exists():- content = config_file.read_text()+ content = fsutil.read_text(config_file)data = _parse_json_loose(content)else:content = ""diff --git a/headroom/providers/opencode/install.py b/headroom/providers/opencode/install.pyindex 1209bc27a..38bc951f9 100644--- a/headroom/providers/opencode/install.py+++ b/headroom/providers/opencode/install.py@@ -6,6 +6,7 @@ import jsonimport shutilfrom pathlib import Path+from headroom import fsutilfrom headroom.install.models import ConfigScope, DeploymentManifest, ManagedMutation, ToolTargetfrom headroom.install.paths import opencode_config_path@@ -38,7 +39,7 @@ def apply_provider_scope(manifest: DeploymentManifest) -> ManagedMutation | None)if config_file.exists():- content = config_file.read_text()+ content = fsutil.read_text(config_file)data = _parse_json_loose(content)else:data = {}@@ -80,7 +81,7 @@ def revert_provider_scope(mutation: ManagedMutation, manifest: DeploymentManifespassif not path.exists():return- content = path.read_text()+ content = fsutil.read_text(path)cleaned = strip_opencode_headroom_blocks(content)if cleaned:path.write_text(cleaned + "\n", encoding="utf-8")diff --git a/tests/test_fsutil.py b/tests/test_fsutil.pynew file mode 100644index 000000000..9bf5dfb1e--- /dev/null+++ b/tests/test_fsutil.py@@ -0,0 +1,73 @@+"""Tests for headroom.fsutil — encoding- and newline-safe text I/O (#733)."""++from __future__ import annotations++import pytest++from headroom import fsutil+++def test_write_text_does_not_double_existing_crlf(tmp_path):+ """A string containing \\r\\n must be written verbatim, never as \\r\\r\\n."""+ p = tmp_path / "config.toml"+ fsutil.write_text(p, 'model = "gpt-5"\r\nport = 8787\r\n')+ raw = p.read_bytes()+ assert b"\r\r\n" not in raw+ assert raw == b'model = "gpt-5"\r\nport = 8787\r\n'+++def test_write_text_does_not_translate_lf(tmp_path):+ """\\n must stay \\n on every platform (no \\r\\n rewrite)."""+ p = tmp_path / "hook.sh"+ fsutil.write_text(p, "#!/bin/sh\necho hi\n")+ assert p.read_bytes() == b"#!/bin/sh\necho hi\n"+++def test_read_text_normalises_crlf(tmp_path):+ """read_text returns universal-newline (\\n) text, so a round trip can't double CRLF."""+ p = tmp_path / "config.toml"+ p.write_bytes(b"a = 1\r\nb = 2\r\n")+ text = fsutil.read_text(p)+ assert text == "a = 1\nb = 2\n"+ fsutil.write_text(p, text)+ assert b"\r" not in p.read_bytes()+++def test_read_text_roundtrips_utf8_non_ascii(tmp_path):+ p = tmp_path / "config.toml"+ fsutil.write_text(p, 'project = "比赛/机器人"\n')+ assert fsutil.read_text(p) == 'project = "比赛/机器人"\n'+++def test_read_text_falls_back_to_locale_encoding(tmp_path, monkeypatch):+ """A file a tool wrote in the locale encoding (e.g. GBK) still decodes."""+ monkeypatch.setattr(fsutil.locale, "getpreferredencoding", lambda *_: "gbk")+ p = tmp_path / "config.toml"+ p.write_bytes('path = "模型"\n'.encode("gbk")) # not valid UTF-8+ assert fsutil.read_text(p) == 'path = "模型"\n'+++def test_read_text_replace_fallback_never_raises(tmp_path, monkeypatch):+ """When neither UTF-8 nor the locale encoding decodes, fall back to replace."""+ monkeypatch.setattr(fsutil.locale, "getpreferredencoding", lambda *_: "ascii")+ p = tmp_path / "config.toml"+ p.write_bytes(b"\xff\xfe bad bytes")+ # Must not raise; returns *something* decodable.+ assert isinstance(fsutil.read_text(p), str)+++def test_read_text_missing_returns_default(tmp_path):+ p = tmp_path / "nope.toml"+ assert fsutil.read_text(p, default="") == ""+++def test_read_text_missing_raises_without_default(tmp_path):+ with pytest.raises(OSError):+ fsutil.read_text(tmp_path / "nope.toml")+++def test_append_text_preserves_endings(tmp_path):+ p = tmp_path / "AGENTS.md"+ fsutil.write_text(p, "line1\n")+ fsutil.append_text(p, "line2\n")+ assert p.read_bytes() == b"line1\nline2\n"diff --git a/tests/test_mcp_registry/test_codex_registrar.py b/tests/test_mcp_registry/test_codex_registrar.pyindex 6a48b4d2b..937777610 100644--- a/tests/test_mcp_registry/test_codex_registrar.py+++ b/tests/test_mcp_registry/test_codex_registrar.py@@ -359,3 +359,40 @@ def test_round_trip(tmp_path: Path, spec: ServerSpec) -> None:assert got.command == spec.commandassert got.args == spec.argsassert got.env == spec.env+++# ---------------------------------------------------------------------------+# #733 — encoding / line-ending safety on GBK / CRLF Windows configs+# ---------------------------------------------------------------------------+++def test_register_does_not_double_crlf(tmp_path: Path) -> None:+ """A pre-existing CRLF config must not gain ``\\r\\r\\n`` after register."""+ import tomllib++ cfg = _config_path(tmp_path)+ cfg.parent.mkdir(parents=True, exist_ok=True)+ cfg.write_bytes(b'model = "gpt-5"\r\nworkers = 1\r\n')++ result = _make_registrar(tmp_path).register_server(_spec())++ assert result.status == RegisterStatus.REGISTERED+ raw = cfg.read_bytes()+ assert b"\r\r\n" not in raw+ tomllib.loads(raw.decode("utf-8")) # still valid TOML+++def test_register_preserves_non_ascii_values(tmp_path: Path) -> None:+ """A config with non-ASCII (Chinese) values survives register and stays parseable."""+ import tomllib++ cfg = _config_path(tmp_path)+ cfg.parent.mkdir(parents=True, exist_ok=True)+ cfg.write_text('model = "gpt-5"\nproject = "比赛/机器人"\n', encoding="utf-8")++ result = _make_registrar(tmp_path).register_server(_spec())++ assert result.status == RegisterStatus.REGISTERED+ data = tomllib.loads(cfg.read_text(encoding="utf-8"))+ assert data["project"] == "比赛/机器人"+ assert "headroom" in data.get("mcp_servers", {})