Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix: decode/encode owned config, state and template assets as UTF-8

supermario_leo2f1538a641dd
diff --git a/headroom/dashboard/__init__.py b/headroom/dashboard/__init__.py
index 74010394d..b02c3bcf5 100644
--- a/headroom/dashboard/__init__.py
+++ b/headroom/dashboard/__init__.py
@@ -9,4 +9,4 @@ TEMPLATES_DIR = DASHBOARD_DIR / "templates"
def get_dashboard_html() -> str:
"""Load the dashboard HTML template."""
template_path = TEMPLATES_DIR / "dashboard.html"
- return template_path.read_text()
+ return template_path.read_text(encoding="utf-8")
diff --git a/headroom/install/state.py b/headroom/install/state.py
index 06b3bf023..5ef9f0f35 100644
--- a/headroom/install/state.py
+++ b/headroom/install/state.py
@@ -24,7 +24,7 @@ def save_manifest(manifest: DeploymentManifest) -> None:
root.mkdir(parents=True, exist_ok=True)
manifest.updated_at = iso_utc_now()
path = manifest_path(manifest.profile)
- path.write_text(json.dumps(asdict(manifest), indent=2) + "\n")
+ path.write_text(json.dumps(asdict(manifest), indent=2) + "\n", encoding="utf-8")
except OSError as e:
logger.warning("Cannot save deployment manifest: %s — continuing without persistence", e)
@@ -35,7 +35,7 @@ def load_manifest(profile: str = "default") -> DeploymentManifest | None:
path = manifest_path(profile)
if not path.exists():
return None
- payload = json.loads(path.read_text())
+ payload = json.loads(path.read_text(encoding="utf-8"))
payload["mutations"] = [ManagedMutation(**item) for item in payload.get("mutations", [])]
payload["artifacts"] = [ArtifactRecord(**item) for item in payload.get("artifacts", [])]
return DeploymentManifest(**payload)
@@ -51,7 +51,7 @@ def list_manifests() -> list[DeploymentManifest]:
manifests: list[DeploymentManifest] = []
for candidate in sorted(root.glob("*/manifest.json")):
try:
- payload = json.loads(candidate.read_text())
+ payload = json.loads(candidate.read_text(encoding="utf-8"))
payload["mutations"] = [
ManagedMutation(**item) for item in payload.get("mutations", [])
]
diff --git a/headroom/memory/sync.py b/headroom/memory/sync.py
index 4ab48dbc0..8f78e12c5 100644
--- a/headroom/memory/sync.py
+++ b/headroom/memory/sync.py
@@ -124,7 +124,7 @@ def _load_sync_state(state_path: Path) -> dict[str, Any]:
"""Load sync state from disk."""
if state_path.exists():
try:
- result: dict[str, Any] = json.loads(state_path.read_text())
+ result: dict[str, Any] = json.loads(state_path.read_text(encoding="utf-8"))
return result
except (json.JSONDecodeError, OSError):
pass
@@ -134,7 +134,7 @@ def _load_sync_state(state_path: Path) -> dict[str, Any]:
def _save_sync_state(state_path: Path, state: dict[str, Any]) -> None:
"""Save sync state to disk."""
state_path.parent.mkdir(parents=True, exist_ok=True)
- state_path.write_text(json.dumps(state, indent=2))
+ state_path.write_text(json.dumps(state, indent=2), encoding="utf-8")
def _db_fingerprint(memories: list[Any]) -> str:
diff --git a/headroom/providers/claude/install.py b/headroom/providers/claude/install.py
index 97f166018..30b6928c2 100644
--- a/headroom/providers/claude/install.py
+++ b/headroom/providers/claude/install.py
@@ -26,14 +26,14 @@ def apply_provider_scope(manifest: DeploymentManifest) -> ManagedMutation | None
path.parent.mkdir(parents=True, exist_ok=True)
payload: dict[str, object] = {}
if path.exists():
- payload = json.loads(path.read_text())
+ payload = json.loads(path.read_text(encoding="utf-8"))
env = payload.get("env")
env_map = dict(env) if isinstance(env, dict) else {}
values = manifest.tool_envs.get(ToolTarget.CLAUDE.value, {})
previous = {name: env_map.get(name) for name in values}
env_map.update(values)
payload["env"] = env_map
- path.write_text(json.dumps(payload, indent=2) + "\n")
+ path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
return ManagedMutation(
target=ToolTarget.CLAUDE.value,
kind="json-env",
@@ -49,7 +49,7 @@ def revert_provider_scope(mutation: ManagedMutation, manifest: DeploymentManifes
path = Path(mutation.path)
if not path.exists():
return
- payload = json.loads(path.read_text())
+ payload = json.loads(path.read_text(encoding="utf-8"))
env = payload.get("env")
env_map = dict(env) if isinstance(env, dict) else {}
previous: dict[str, object] = mutation.data.get("previous", {})
@@ -60,4 +60,4 @@ def revert_provider_scope(mutation: ManagedMutation, manifest: DeploymentManifes
else:
env_map[name] = previous[name]
payload["env"] = env_map
- path.write_text(json.dumps(payload, indent=2) + "\n")
+ path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
diff --git a/headroom/providers/codex/install.py b/headroom/providers/codex/install.py
index cfd731b20..24b1ef0a0 100644
--- a/headroom/providers/codex/install.py
+++ b/headroom/providers/codex/install.py
@@ -80,14 +80,14 @@ def apply_provider_scope(manifest: DeploymentManifest) -> ManagedMutation | None
+ f"{_CODEX_MARKER_END}\n"
)
if path.exists():
- existing = path.read_text()
+ existing = path.read_text(encoding="utf-8")
if _CODEX_MARKER_START in existing:
merged = _CODEX_PATTERN.sub(section, existing)
else:
merged = existing.rstrip() + "\n\n" + section + "\n"
else:
merged = section + "\n"
- path.write_text(merged)
+ path.write_text(merged, encoding="utf-8")
return ManagedMutation(target=ToolTarget.CODEX.value, kind="toml-block", path=str(path))
@@ -99,7 +99,7 @@ def revert_provider_scope(mutation: ManagedMutation, manifest: DeploymentManifes
path = Path(mutation.path)
if not path.exists():
return
- content = path.read_text()
+ content = path.read_text(encoding="utf-8")
# Remove the managed marker block.
if _CODEX_MARKER_START in content:
content = _CODEX_PATTERN.sub("", content)
@@ -108,4 +108,4 @@ def revert_provider_scope(mutation: ManagedMutation, manifest: DeploymentManifes
content = _ORPHAN_MODEL_PROVIDER.sub("", content)
content = _ORPHAN_OPENAI_BASE_URL.sub("", content)
content = _ORPHAN_HEADROOM_TABLE.sub("", content)
- path.write_text(content.strip() + "\n")
+ path.write_text(content.strip() + "\n", encoding="utf-8")
diff --git a/headroom/telemetry/reporter.py b/headroom/telemetry/reporter.py
index cf72fab2d..eb23afd66 100644
--- a/headroom/telemetry/reporter.py
+++ b/headroom/telemetry/reporter.py
@@ -357,7 +357,9 @@ class UsageReporter:
return
try:
self._cache_path.parent.mkdir(parents=True, exist_ok=True)
- self._cache_path.write_text(json.dumps(self._license_info.to_dict(), indent=2))
+ self._cache_path.write_text(
+ json.dumps(self._license_info.to_dict(), indent=2), encoding="utf-8"
+ )
except OSError:
logger.warning("Could not save license cache to %s", self._cache_path)
@@ -365,7 +367,7 @@ class UsageReporter:
"""Load cached license info, or return a default if expired/missing."""
try:
if self._cache_path.exists():
- data = json.loads(self._cache_path.read_text())
+ data = json.loads(self._cache_path.read_text(encoding="utf-8"))
cached = LicenseInfo.from_dict(data)
age = (datetime.now(timezone.utc) - cached.validated_at).total_seconds()
if age < GRACE_PERIOD_SECONDS:
diff --git a/tests/test_owned_asset_encoding.py b/tests/test_owned_asset_encoding.py
new file mode 100644
index 000000000..4d2f25362
--- /dev/null
+++ b/tests/test_owned_asset_encoding.py
@@ -0,0 +1,59 @@
+"""Regression tests for UTF-8 decoding/encoding of headroom-owned assets.
+
+These guard against ``UnicodeDecodeError`` on systems whose default text
+encoding is not UTF-8 (e.g. Windows ``cp949``/``cp1252`` locales). Headroom
+ships and writes its own templates, JSON state and config files as UTF-8, so
+they must be read and written with an explicit ``encoding="utf-8"`` rather than
+relying on the platform default codec. See issue #533.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from headroom.dashboard import TEMPLATES_DIR, get_dashboard_html
+from headroom.memory.sync import _load_sync_state, _save_sync_state
+
+
+def test_dashboard_template_contains_non_ascii() -> None:
+ """The bundled template has non-ASCII bytes, so the bug is reproducible."""
+ raw = (TEMPLATES_DIR / "dashboard.html").read_bytes()
+ assert any(byte > 0x7F for byte in raw), "template expected to contain non-ASCII bytes"
+
+
+def test_get_dashboard_html_reads_as_utf8(monkeypatch) -> None:
+ """get_dashboard_html must decode the template as UTF-8, not the OS default.
+
+ Before the fix, ``read_text()`` used the platform default codec and raised
+ ``UnicodeDecodeError`` on non-UTF-8 locales. We assert the explicit encoding
+ is passed so the regression cannot silently return (a utf-8 CI host would
+ otherwise mask it).
+ """
+ captured: dict[str, object] = {}
+ original = Path.read_text
+
+ def _spy(self: Path, *args: object, **kwargs: object) -> str:
+ captured["encoding"] = kwargs.get("encoding")
+ return original(self, *args, **kwargs) # type: ignore[arg-type]
+
+ monkeypatch.setattr(Path, "read_text", _spy)
+
+ html = get_dashboard_html()
+
+ assert captured["encoding"] == "utf-8"
+ assert html # non-empty
+ # Content must equal an explicit UTF-8 decode of the raw template.
+ expected = (TEMPLATES_DIR / "dashboard.html").read_bytes().decode("utf-8")
+ assert html == expected
+
+
+def test_sync_state_round_trips_non_ascii(tmp_path) -> None:
+ """JSON sync state with non-ASCII values must survive a save/load round-trip."""
+ state_path = tmp_path / "nested" / "sync_state.json"
+ state = {"agent": "café", "note": "한국어 메모", "emoji": "🚀"}
+
+ _save_sync_state(state_path, state)
+
+ # Persisted bytes must be valid UTF-8 regardless of the platform default.
+ assert state_path.read_bytes().decode("utf-8")
+ assert _load_sync_state(state_path) == state