Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(cli): strip unmarked headroom_memory TOML table before injecting (#3490)

Bernardo Gomes08dda4cb0b5f
diff --git a/headroom/cli/wrap.py b/headroom/cli/wrap.py
index de0836a3c..2c6275d42 100644
--- a/headroom/cli/wrap.py
+++ b/headroom/cli/wrap.py
@@ -39,6 +39,11 @@ from functools import wraps
from pathlib import Path
from typing import Any, NamedTuple, cast
+try:
+ import tomllib
+except ModuleNotFoundError: # pragma: no cover - Python < 3.11
+ import tomli as tomllib
+
from headroom._subprocess import identity_mismatch as _shared_identity_mismatch
from headroom._subprocess import pid_alive, proc_identity, run
@@ -2892,16 +2897,23 @@ def _strip_codex_headroom_blocks(
content,
)
- # Strip any orphaned `[model_providers.headroom]` table with the fields we
- # write. We only remove it if the table is recognisably ours (base_url
- # mentions localhost and a Headroom proxy port). This protects users who
- # happen to have a differently configured `headroom` provider.
- orphan_headroom_table = re.compile(
- r"(?ms)^\[model_providers\.headroom\][^\[]*?"
- r'base_url[ \t]*=[ \t]*"http://127\.0\.0\.1:\d+/v1"[^\[]*?'
- r"(?=^\[|\Z)"
- )
- content = orphan_headroom_table.sub("", content)
+ # Remove an orphaned local-proxy provider structurally. Text-level table
+ # matching cannot safely distinguish comments, multiline strings, and
+ # arrays of tables from a provider's body.
+ import tomlkit
+
+ try:
+ document = tomlkit.parse(content)
+ except tomlkit.exceptions.ParseError:
+ pass
+ else:
+ providers = document.get("model_providers")
+ if isinstance(providers, MutableMapping):
+ provider = providers.get("headroom")
+ base_url = provider.get("base_url") if isinstance(provider, Mapping) else None
+ if isinstance(base_url, str) and re.fullmatch(r"http://127\.0\.0\.1:\d+/v1", base_url):
+ del providers["headroom"]
+ content = tomlkit.dumps(document)
return content.lstrip("\n").rstrip() + "\n" if content.strip() else ""
@@ -2915,17 +2927,51 @@ _REDIRECTABLE_KEYS: tuple[str, ...] = ("model_provider", "openai_base_url")
def _strip_existing_codex_headroom_provider_table(content: str) -> str:
- """Remove a pre-existing ``[model_providers.headroom]`` table before wrap."""
- if "[model_providers.headroom]" not in content:
+ """Remove a legacy Headroom provider table without touching other TOML.
+
+ ``tomlkit`` identifies tables structurally, preserving comments, string
+ values, and arrays of tables that text-level table-boundary matching can
+ mistake for provider content. The injector owns this provider name, so an
+ existing table is replaced by the current Headroom configuration.
+ """
+ import tomlkit
+
+ try:
+ document = tomlkit.parse(content)
+ except tomlkit.exceptions.ParseError:
+ return content
+
+ providers = document.get("model_providers")
+ if not isinstance(providers, MutableMapping):
+ return content
+ provider = providers.get("headroom")
+ if not isinstance(provider, Mapping):
return content
+ del providers["headroom"]
+ return str(tomlkit.dumps(document))
- import re # local import to match surrounding helper convention
- provider_table = re.compile(
- r"(?ms)^[ \t]*\[model_providers\.headroom\][^\n]*\n.*?(?=^[ \t]*\[|\Z)"
- )
- content = provider_table.sub("", content)
- return content.lstrip("\n").rstrip() + "\n" if content.strip() else ""
+def _strip_existing_codex_memory_mcp_table(content: str) -> str:
+ """Remove an unmarked memory MCP table without text-level TOML parsing.
+
+ TOMLKit resolves quoted keys and finds table boundaries structurally, so
+ comments, multiline strings, and arrays of tables remain untouched.
+ Invalid source TOML is deliberately left unchanged; the caller validates
+ its complete generated candidate before it can overwrite the file.
+ """
+ import tomlkit
+
+ try:
+ document = tomlkit.parse(content)
+ except tomlkit.exceptions.ParseError:
+ return content
+
+ mcp_servers = document.get("mcp_servers")
+ if not isinstance(mcp_servers, MutableMapping) or "headroom_memory" not in mcp_servers:
+ return content
+
+ del mcp_servers["headroom_memory"]
+ return str(tomlkit.dumps(document))
def _redirect_existing_top_level_keys(content: str, port: int) -> str:
@@ -3347,6 +3393,8 @@ def _inject_codex_provider_config(port: int) -> str | None:
f"\n{provider_section}"
)
+ # Never replace a usable config with a malformed generated candidate.
+ tomllib.loads(content)
_write_text(config_file, content)
click.echo(f" Codex config: injected Headroom provider (WS + HTTP) into {config_file}")
if custom_upstream_base_url:
@@ -3574,10 +3622,21 @@ def _inject_memory_mcp_config(user_id: str) -> None:
end = content.index(_MEMORY_MCP_END) + len(_MEMORY_MCP_END)
content = content[:start].rstrip("\n") + mcp_section + content[end:].lstrip("\n")
else:
+ # No marker yet. A `[mcp_servers.headroom_memory]` table can
+ # still exist unmarked — e.g. a user copied the snippet from
+ # this module's docstring by hand, or the table survived
+ # from before markers guarded this injection. Strip it first
+ # so we don't end up with two `[mcp_servers.headroom_memory]`
+ # tables, which is invalid TOML and silently breaks Codex's
+ # MCP config parsing (last-table-wins or hard failure
+ # depending on the TOML parser).
+ content = _strip_existing_codex_memory_mcp_table(content)
content = content.rstrip() + "\n" + mcp_section
else:
content = mcp_section
+ # Never replace a usable config with a malformed generated candidate.
+ tomllib.loads(content)
_write_text(config_file, content)
click.echo(f" Memory MCP: registered in {config_file}")
except Exception as e:
diff --git a/tests/test_cli/test_wrap_codex.py b/tests/test_cli/test_wrap_codex.py
index f26f931f8..8a2dbd96f 100644
--- a/tests/test_cli/test_wrap_codex.py
+++ b/tests/test_cli/test_wrap_codex.py
@@ -263,6 +263,176 @@ class TestCodexMemoryMcpConfig:
assert "--db" not in content
assert 'model = "gpt-4o"' in content
+ def test_inject_replaces_unmarked_memory_block(
+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+ ) -> None:
+ """A `[mcp_servers.headroom_memory]` table without the injection
+ markers (e.g. hand-copied from the `mcp_server.py` docstring, or left
+ over from before this table was marker-guarded) must be replaced in
+ place, not duplicated alongside a second, marked table.
+ """
+ _set_test_home(monkeypatch, tmp_path)
+ config_file = tmp_path / ".codex" / "config.toml"
+ config_file.parent.mkdir(parents=True)
+ config_file.write_text(
+ '[profiles.default]\nmodel = "gpt-4o"\n\n'
+ "[mcp_servers.headroom_memory]\n"
+ 'command = "python"\n'
+ 'args = ["-m", "headroom.memory.mcp_server", "--user", "old-user"]\n'
+ "startup_timeout_sec = 30\n"
+ "tool_timeout_sec = 30\n\n"
+ "[mcp_servers.other]\n"
+ 'command = "other-tool"\n'
+ )
+
+ wrap_mod._inject_memory_mcp_config("codex-user")
+
+ content = config_file.read_text()
+ assert content.count("[mcp_servers.headroom_memory]") == 1
+ assert content.count(wrap_mod._MEMORY_MCP_MARKER) == 1
+ assert '"--user", "codex-user"' in content
+ assert "old-user" not in content
+ assert 'model = "gpt-4o"' in content
+ assert "[mcp_servers.other]" in content
+ assert 'command = "other-tool"' in content
+
+ # The result must also be valid, parseable TOML — the original bug
+ # produced two `[mcp_servers.headroom_memory]` tables, which is
+ # invalid TOML (duplicate key) and breaks Codex startup.
+ parsed = tomllib.loads(content)
+ assert parsed["mcp_servers"]["headroom_memory"]["args"][-1] == "codex-user"
+
+ def test_inject_preserves_table_looking_text_in_multiline_string(
+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+ ) -> None:
+ """A `[mcp_servers.headroom_memory]`-looking line embedded inside an
+ unrelated multiline string value must not be mistaken for a real
+ table and stripped -- doing so previously deleted the string's
+ closing delimiter and any settings after it, corrupting the file.
+ """
+ _set_test_home(monkeypatch, tmp_path)
+ config_file = tmp_path / ".codex" / "config.toml"
+ config_file.parent.mkdir(parents=True)
+ original = (
+ 'instructions = """\n'
+ "[mcp_servers.headroom_memory]\n"
+ "keep this instruction\n"
+ '"""\n'
+ 'model="gpt-4o"\n'
+ )
+ config_file.write_text(original)
+
+ wrap_mod._inject_memory_mcp_config("codex-user")
+
+ content = config_file.read_text()
+ # The unrelated multiline string and the setting after it must
+ # survive untouched.
+ assert "keep this instruction" in content
+ assert 'model="gpt-4o"' in content
+ # The real, marker-wrapped memory MCP block must still be injected.
+ assert content.count(wrap_mod._MEMORY_MCP_MARKER) == 1
+
+ parsed = tomllib.loads(content)
+ assert parsed["model"] == "gpt-4o"
+ assert parsed["instructions"] == "[mcp_servers.headroom_memory]\nkeep this instruction\n"
+ assert parsed["mcp_servers"]["headroom_memory"]["args"][-1] == "codex-user"
+
+ def test_inject_replaces_quoted_key_memory_table(
+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+ ) -> None:
+ """TOML allows `[mcp_servers."headroom_memory"]` as an equivalent
+ spelling of `[mcp_servers.headroom_memory]`; an unmarked table using
+ this spelling must still be replaced in place, not left behind
+ alongside a second, semantically-duplicate table.
+ """
+ _set_test_home(monkeypatch, tmp_path)
+ config_file = tmp_path / ".codex" / "config.toml"
+ config_file.parent.mkdir(parents=True)
+ config_file.write_text(
+ '[profiles.default]\nmodel = "gpt-4o"\n\n'
+ '[mcp_servers."headroom_memory"]\n'
+ 'command = "python"\n'
+ 'args = ["-m", "headroom.memory.mcp_server", "--user", "old-user"]\n\n'
+ "[mcp_servers.other]\n"
+ 'command = "other-tool"\n'
+ )
+
+ wrap_mod._inject_memory_mcp_config("codex-user")
+
+ content = config_file.read_text()
+ assert content.count(wrap_mod._MEMORY_MCP_MARKER) == 1
+ assert '"--user", "codex-user"' in content
+ assert "old-user" not in content
+ assert 'model = "gpt-4o"' in content
+ assert "[mcp_servers.other]" in content
+ assert 'command = "other-tool"' in content
+
+ # Must parse as valid TOML with exactly one `mcp_servers.headroom_memory`
+ # table -- the original bug left the quoted-key table behind, and
+ # `tomllib.loads` raises `Cannot declare (\'mcp_servers\', \'headroom_memory\')
+ # twice` once the marker-wrapped table is appended alongside it.
+ parsed = tomllib.loads(content)
+ assert parsed["mcp_servers"]["headroom_memory"]["args"][-1] == "codex-user"
+ assert parsed["mcp_servers"]["other"]["command"] == "other-tool"
+
+ def test_inject_preserves_array_table_after_unmarked_memory_table(
+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+ ) -> None:
+ """Removing an old memory table must stop before an unrelated array table."""
+ _set_test_home(monkeypatch, tmp_path)
+ config_file = tmp_path / ".codex" / "config.toml"
+ config_file.parent.mkdir(parents=True)
+ config_file.write_text(
+ "[mcp_servers.headroom_memory]\n"
+ 'command = "old"\n\n'
+ "[[profiles.custom.entries]]\n"
+ 'name = "preserve"\n'
+ )
+
+ wrap_mod._inject_memory_mcp_config("codex-user")
+
+ parsed = tomllib.loads(config_file.read_text())
+ assert parsed["profiles"]["custom"]["entries"] == [{"name": "preserve"}]
+ assert parsed["mcp_servers"]["headroom_memory"]["args"][-1] == "codex-user"
+
+ def test_inject_ignores_triple_quote_in_comment_before_memory_table(
+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+ ) -> None:
+ """A comment delimiter cannot hide the real old memory table from cleanup."""
+ _set_test_home(monkeypatch, tmp_path)
+ config_file = tmp_path / ".codex" / "config.toml"
+ config_file.parent.mkdir(parents=True)
+ config_file.write_text(
+ '# Example triple quote: """\n[mcp_servers.headroom_memory]\ncommand = "old"\n'
+ )
+
+ wrap_mod._inject_memory_mcp_config("codex-user")
+
+ content = config_file.read_text()
+ parsed = tomllib.loads(content)
+ assert content.count("[mcp_servers.headroom_memory]") == 1
+ assert parsed["mcp_servers"]["headroom_memory"]["args"][-1] == "codex-user"
+
+ def test_inject_leaves_file_unchanged_when_candidate_is_invalid(
+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+ ) -> None:
+ """An invalid candidate must never overwrite the user's configuration."""
+ _set_test_home(monkeypatch, tmp_path)
+ config_file = tmp_path / ".codex" / "config.toml"
+ config_file.parent.mkdir(parents=True)
+ original = (
+ "broken = [\n"
+ f"{wrap_mod._MEMORY_MCP_MARKER}\n"
+ "[mcp_servers.headroom_memory]\n"
+ 'command = "old"\n'
+ f"{wrap_mod._MEMORY_MCP_END}\n"
+ )
+ config_file.write_text(original)
+
+ wrap_mod._inject_memory_mcp_config("codex-user")
+
+ assert config_file.read_text() == original
+
class TestInjectAndRestoreRoundTrip:
"""End-to-end wrap → unwrap cycle operating directly on a temp $HOME."""
@@ -944,6 +1114,86 @@ class TestInjectAvoidsDuplicateTopLevelKeys:
assert "[profiles.default]" in content
assert 'model = "gpt-5"' in content
+ def test_inject_replaces_legacy_nested_subtable_format(
+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+ ) -> None:
+ """Legacy ``wrap`` output used a nested
+ ``[model_providers.headroom.env_http_headers]`` sub-table instead of
+ today's inline table. The strip must consume the sub-table as well:
+ leaving it behind makes the re-injected inline ``env_http_headers``
+ key a duplicate TOML key and invalidates the whole config (seen in
+ the wild as ``TOMLDecodeError: Cannot declare ('model_providers',
+ 'headroom') twice``).
+ """
+ _set_test_home(monkeypatch, tmp_path)
+ config_dir = tmp_path / ".codex"
+ config_dir.mkdir()
+ config_file = config_dir / "config.toml"
+ config_file.write_text(
+ "[profiles.default]\n"
+ 'model = "gpt-5"\n'
+ "\n"
+ "[model_providers.headroom]\n"
+ 'base_url = "http://127.0.0.1:8787/v1"\n'
+ 'name = "OpenAI via Headroom proxy"\n'
+ "requires_openai_auth = true\n"
+ "supports_websockets = true\n"
+ "\n"
+ "[model_providers.headroom.env_http_headers]\n"
+ 'X-Headroom-Project = "HEADROOM_PROJECT"\n'
+ "\n"
+ "[notice]\n"
+ "hide_rate_limit_model_nudge = true\n"
+ )
+
+ wrap_mod._inject_codex_provider_config(8787)
+ content = config_file.read_text()
+
+ parsed = tomllib.loads(content) # must not raise: the original bug did
+ assert content.count("[model_providers.headroom]") == 1
+ assert "[model_providers.headroom.env_http_headers]" not in content
+ assert content.count("env_http_headers") == 1 # inline form from the injection
+ assert parsed["model_providers"]["headroom"]["base_url"] == "http://127.0.0.1:8787/v1"
+ assert parsed["model_providers"]["headroom"]["env_http_headers"] == {
+ "X-Headroom-Project": "HEADROOM_PROJECT"
+ }
+ assert parsed["profiles"]["default"]["model"] == "gpt-5"
+ assert parsed["notice"] == {"hide_rate_limit_model_nudge": True}
+
+ def test_inject_preserves_array_table_after_legacy_provider_table(
+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+ ) -> None:
+ """Provider cleanup must stop before an unrelated TOML array table."""
+ _set_test_home(monkeypatch, tmp_path)
+ config_file = tmp_path / ".codex" / "config.toml"
+ config_file.parent.mkdir()
+ config_file.write_text(
+ "[model_providers.headroom]\n"
+ 'base_url = "http://127.0.0.1:8787/v1"\n\n'
+ "[[profiles.custom.entries]]\n"
+ 'name = "preserve"\n'
+ )
+
+ wrap_mod._inject_codex_provider_config(8787)
+
+ parsed = tomllib.loads(config_file.read_text())
+ assert parsed["profiles"]["custom"]["entries"] == [{"name": "preserve"}]
+ assert parsed["model_providers"]["headroom"]["base_url"] == "http://127.0.0.1:8787/v1"
+
+ def test_inject_leaves_invalid_provider_config_unchanged(
+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+ ) -> None:
+ """Provider injection must not overwrite a config that cannot validate."""
+ _set_test_home(monkeypatch, tmp_path)
+ config_file = tmp_path / ".codex" / "config.toml"
+ config_file.parent.mkdir()
+ original = 'broken = [\n[model_providers.headroom]\nbase_url = "http://127.0.0.1:8787/v1"\n'
+ config_file.write_text(original)
+
+ wrap_mod._inject_codex_provider_config(8787)
+
+ assert config_file.read_text() == original
+
def test_unwrap_restores_prior_headroom_provider_table(
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None: