Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(cli): strip unmarked headroom_memory TOML table before injecting (#3490)
Bernardo Gomes08dda4cb0b5f
diff --git a/headroom/cli/wrap.py b/headroom/cli/wrap.pyindex de0836a3c..2c6275d42 100644--- a/headroom/cli/wrap.py+++ b/headroom/cli/wrap.py@@ -39,6 +39,11 @@ from functools import wrapsfrom pathlib import Pathfrom typing import Any, NamedTuple, cast+try:+ import tomllib+except ModuleNotFoundError: # pragma: no cover - Python < 3.11+ import tomli as tomllib+from headroom._subprocess import identity_mismatch as _shared_identity_mismatchfrom headroom._subprocess import pid_alive, proc_identity, run@@ -2892,16 +2897,23 @@ def _strip_codex_headroom_blocks(content,)- # Strip any orphaned `[model_providers.headroom]` table with the fields we- # write. We only remove it if the table is recognisably ours (base_url- # mentions localhost and a Headroom proxy port). This protects users who- # happen to have a differently configured `headroom` provider.- orphan_headroom_table = re.compile(- r"(?ms)^\[model_providers\.headroom\][^\[]*?"- r'base_url[ \t]*=[ \t]*"http://127\.0\.0\.1:\d+/v1"[^\[]*?'- r"(?=^\[|\Z)"- )- content = orphan_headroom_table.sub("", content)+ # Remove an orphaned local-proxy provider structurally. Text-level table+ # matching cannot safely distinguish comments, multiline strings, and+ # arrays of tables from a provider's body.+ import tomlkit++ try:+ document = tomlkit.parse(content)+ except tomlkit.exceptions.ParseError:+ pass+ else:+ providers = document.get("model_providers")+ if isinstance(providers, MutableMapping):+ provider = providers.get("headroom")+ base_url = provider.get("base_url") if isinstance(provider, Mapping) else None+ if isinstance(base_url, str) and re.fullmatch(r"http://127\.0\.0\.1:\d+/v1", base_url):+ del providers["headroom"]+ content = tomlkit.dumps(document)return content.lstrip("\n").rstrip() + "\n" if content.strip() else ""@@ -2915,17 +2927,51 @@ _REDIRECTABLE_KEYS: tuple[str, ...] = ("model_provider", "openai_base_url")def _strip_existing_codex_headroom_provider_table(content: str) -> str:- """Remove a pre-existing ``[model_providers.headroom]`` table before wrap."""- if "[model_providers.headroom]" not in content:+ """Remove a legacy Headroom provider table without touching other TOML.++ ``tomlkit`` identifies tables structurally, preserving comments, string+ values, and arrays of tables that text-level table-boundary matching can+ mistake for provider content. The injector owns this provider name, so an+ existing table is replaced by the current Headroom configuration.+ """+ import tomlkit++ try:+ document = tomlkit.parse(content)+ except tomlkit.exceptions.ParseError:+ return content++ providers = document.get("model_providers")+ if not isinstance(providers, MutableMapping):+ return content+ provider = providers.get("headroom")+ if not isinstance(provider, Mapping):return content+ del providers["headroom"]+ return str(tomlkit.dumps(document))- import re # local import to match surrounding helper convention- provider_table = re.compile(- r"(?ms)^[ \t]*\[model_providers\.headroom\][^\n]*\n.*?(?=^[ \t]*\[|\Z)"- )- content = provider_table.sub("", content)- return content.lstrip("\n").rstrip() + "\n" if content.strip() else ""+def _strip_existing_codex_memory_mcp_table(content: str) -> str:+ """Remove an unmarked memory MCP table without text-level TOML parsing.++ TOMLKit resolves quoted keys and finds table boundaries structurally, so+ comments, multiline strings, and arrays of tables remain untouched.+ Invalid source TOML is deliberately left unchanged; the caller validates+ its complete generated candidate before it can overwrite the file.+ """+ import tomlkit++ try:+ document = tomlkit.parse(content)+ except tomlkit.exceptions.ParseError:+ return content++ mcp_servers = document.get("mcp_servers")+ if not isinstance(mcp_servers, MutableMapping) or "headroom_memory" not in mcp_servers:+ return content++ del mcp_servers["headroom_memory"]+ return str(tomlkit.dumps(document))def _redirect_existing_top_level_keys(content: str, port: int) -> str:@@ -3347,6 +3393,8 @@ def _inject_codex_provider_config(port: int) -> str | None:f"\n{provider_section}")+ # Never replace a usable config with a malformed generated candidate.+ tomllib.loads(content)_write_text(config_file, content)click.echo(f" Codex config: injected Headroom provider (WS + HTTP) into {config_file}")if custom_upstream_base_url:@@ -3574,10 +3622,21 @@ def _inject_memory_mcp_config(user_id: str) -> None:end = content.index(_MEMORY_MCP_END) + len(_MEMORY_MCP_END)content = content[:start].rstrip("\n") + mcp_section + content[end:].lstrip("\n")else:+ # No marker yet. A `[mcp_servers.headroom_memory]` table can+ # still exist unmarked — e.g. a user copied the snippet from+ # this module's docstring by hand, or the table survived+ # from before markers guarded this injection. Strip it first+ # so we don't end up with two `[mcp_servers.headroom_memory]`+ # tables, which is invalid TOML and silently breaks Codex's+ # MCP config parsing (last-table-wins or hard failure+ # depending on the TOML parser).+ content = _strip_existing_codex_memory_mcp_table(content)content = content.rstrip() + "\n" + mcp_sectionelse:content = mcp_section+ # Never replace a usable config with a malformed generated candidate.+ tomllib.loads(content)_write_text(config_file, content)click.echo(f" Memory MCP: registered in {config_file}")except Exception as e:diff --git a/tests/test_cli/test_wrap_codex.py b/tests/test_cli/test_wrap_codex.pyindex f26f931f8..8a2dbd96f 100644--- a/tests/test_cli/test_wrap_codex.py+++ b/tests/test_cli/test_wrap_codex.py@@ -263,6 +263,176 @@ class TestCodexMemoryMcpConfig:assert "--db" not in contentassert 'model = "gpt-4o"' in content+ def test_inject_replaces_unmarked_memory_block(+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path+ ) -> None:+ """A `[mcp_servers.headroom_memory]` table without the injection+ markers (e.g. hand-copied from the `mcp_server.py` docstring, or left+ over from before this table was marker-guarded) must be replaced in+ place, not duplicated alongside a second, marked table.+ """+ _set_test_home(monkeypatch, tmp_path)+ config_file = tmp_path / ".codex" / "config.toml"+ config_file.parent.mkdir(parents=True)+ config_file.write_text(+ '[profiles.default]\nmodel = "gpt-4o"\n\n'+ "[mcp_servers.headroom_memory]\n"+ 'command = "python"\n'+ 'args = ["-m", "headroom.memory.mcp_server", "--user", "old-user"]\n'+ "startup_timeout_sec = 30\n"+ "tool_timeout_sec = 30\n\n"+ "[mcp_servers.other]\n"+ 'command = "other-tool"\n'+ )++ wrap_mod._inject_memory_mcp_config("codex-user")++ content = config_file.read_text()+ assert content.count("[mcp_servers.headroom_memory]") == 1+ assert content.count(wrap_mod._MEMORY_MCP_MARKER) == 1+ assert '"--user", "codex-user"' in content+ assert "old-user" not in content+ assert 'model = "gpt-4o"' in content+ assert "[mcp_servers.other]" in content+ assert 'command = "other-tool"' in content++ # The result must also be valid, parseable TOML — the original bug+ # produced two `[mcp_servers.headroom_memory]` tables, which is+ # invalid TOML (duplicate key) and breaks Codex startup.+ parsed = tomllib.loads(content)+ assert parsed["mcp_servers"]["headroom_memory"]["args"][-1] == "codex-user"++ def test_inject_preserves_table_looking_text_in_multiline_string(+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path+ ) -> None:+ """A `[mcp_servers.headroom_memory]`-looking line embedded inside an+ unrelated multiline string value must not be mistaken for a real+ table and stripped -- doing so previously deleted the string's+ closing delimiter and any settings after it, corrupting the file.+ """+ _set_test_home(monkeypatch, tmp_path)+ config_file = tmp_path / ".codex" / "config.toml"+ config_file.parent.mkdir(parents=True)+ original = (+ 'instructions = """\n'+ "[mcp_servers.headroom_memory]\n"+ "keep this instruction\n"+ '"""\n'+ 'model="gpt-4o"\n'+ )+ config_file.write_text(original)++ wrap_mod._inject_memory_mcp_config("codex-user")++ content = config_file.read_text()+ # The unrelated multiline string and the setting after it must+ # survive untouched.+ assert "keep this instruction" in content+ assert 'model="gpt-4o"' in content+ # The real, marker-wrapped memory MCP block must still be injected.+ assert content.count(wrap_mod._MEMORY_MCP_MARKER) == 1++ parsed = tomllib.loads(content)+ assert parsed["model"] == "gpt-4o"+ assert parsed["instructions"] == "[mcp_servers.headroom_memory]\nkeep this instruction\n"+ assert parsed["mcp_servers"]["headroom_memory"]["args"][-1] == "codex-user"++ def test_inject_replaces_quoted_key_memory_table(+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path+ ) -> None:+ """TOML allows `[mcp_servers."headroom_memory"]` as an equivalent+ spelling of `[mcp_servers.headroom_memory]`; an unmarked table using+ this spelling must still be replaced in place, not left behind+ alongside a second, semantically-duplicate table.+ """+ _set_test_home(monkeypatch, tmp_path)+ config_file = tmp_path / ".codex" / "config.toml"+ config_file.parent.mkdir(parents=True)+ config_file.write_text(+ '[profiles.default]\nmodel = "gpt-4o"\n\n'+ '[mcp_servers."headroom_memory"]\n'+ 'command = "python"\n'+ 'args = ["-m", "headroom.memory.mcp_server", "--user", "old-user"]\n\n'+ "[mcp_servers.other]\n"+ 'command = "other-tool"\n'+ )++ wrap_mod._inject_memory_mcp_config("codex-user")++ content = config_file.read_text()+ assert content.count(wrap_mod._MEMORY_MCP_MARKER) == 1+ assert '"--user", "codex-user"' in content+ assert "old-user" not in content+ assert 'model = "gpt-4o"' in content+ assert "[mcp_servers.other]" in content+ assert 'command = "other-tool"' in content++ # Must parse as valid TOML with exactly one `mcp_servers.headroom_memory`+ # table -- the original bug left the quoted-key table behind, and+ # `tomllib.loads` raises `Cannot declare (\'mcp_servers\', \'headroom_memory\')+ # twice` once the marker-wrapped table is appended alongside it.+ parsed = tomllib.loads(content)+ assert parsed["mcp_servers"]["headroom_memory"]["args"][-1] == "codex-user"+ assert parsed["mcp_servers"]["other"]["command"] == "other-tool"++ def test_inject_preserves_array_table_after_unmarked_memory_table(+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path+ ) -> None:+ """Removing an old memory table must stop before an unrelated array table."""+ _set_test_home(monkeypatch, tmp_path)+ config_file = tmp_path / ".codex" / "config.toml"+ config_file.parent.mkdir(parents=True)+ config_file.write_text(+ "[mcp_servers.headroom_memory]\n"+ 'command = "old"\n\n'+ "[[profiles.custom.entries]]\n"+ 'name = "preserve"\n'+ )++ wrap_mod._inject_memory_mcp_config("codex-user")++ parsed = tomllib.loads(config_file.read_text())+ assert parsed["profiles"]["custom"]["entries"] == [{"name": "preserve"}]+ assert parsed["mcp_servers"]["headroom_memory"]["args"][-1] == "codex-user"++ def test_inject_ignores_triple_quote_in_comment_before_memory_table(+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path+ ) -> None:+ """A comment delimiter cannot hide the real old memory table from cleanup."""+ _set_test_home(monkeypatch, tmp_path)+ config_file = tmp_path / ".codex" / "config.toml"+ config_file.parent.mkdir(parents=True)+ config_file.write_text(+ '# Example triple quote: """\n[mcp_servers.headroom_memory]\ncommand = "old"\n'+ )++ wrap_mod._inject_memory_mcp_config("codex-user")++ content = config_file.read_text()+ parsed = tomllib.loads(content)+ assert content.count("[mcp_servers.headroom_memory]") == 1+ assert parsed["mcp_servers"]["headroom_memory"]["args"][-1] == "codex-user"++ def test_inject_leaves_file_unchanged_when_candidate_is_invalid(+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path+ ) -> None:+ """An invalid candidate must never overwrite the user's configuration."""+ _set_test_home(monkeypatch, tmp_path)+ config_file = tmp_path / ".codex" / "config.toml"+ config_file.parent.mkdir(parents=True)+ original = (+ "broken = [\n"+ f"{wrap_mod._MEMORY_MCP_MARKER}\n"+ "[mcp_servers.headroom_memory]\n"+ 'command = "old"\n'+ f"{wrap_mod._MEMORY_MCP_END}\n"+ )+ config_file.write_text(original)++ wrap_mod._inject_memory_mcp_config("codex-user")++ assert config_file.read_text() == original+class TestInjectAndRestoreRoundTrip:"""End-to-end wrap → unwrap cycle operating directly on a temp $HOME."""@@ -944,6 +1114,86 @@ class TestInjectAvoidsDuplicateTopLevelKeys:assert "[profiles.default]" in contentassert 'model = "gpt-5"' in content+ def test_inject_replaces_legacy_nested_subtable_format(+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path+ ) -> None:+ """Legacy ``wrap`` output used a nested+ ``[model_providers.headroom.env_http_headers]`` sub-table instead of+ today's inline table. The strip must consume the sub-table as well:+ leaving it behind makes the re-injected inline ``env_http_headers``+ key a duplicate TOML key and invalidates the whole config (seen in+ the wild as ``TOMLDecodeError: Cannot declare ('model_providers',+ 'headroom') twice``).+ """+ _set_test_home(monkeypatch, tmp_path)+ config_dir = tmp_path / ".codex"+ config_dir.mkdir()+ config_file = config_dir / "config.toml"+ config_file.write_text(+ "[profiles.default]\n"+ 'model = "gpt-5"\n'+ "\n"+ "[model_providers.headroom]\n"+ 'base_url = "http://127.0.0.1:8787/v1"\n'+ 'name = "OpenAI via Headroom proxy"\n'+ "requires_openai_auth = true\n"+ "supports_websockets = true\n"+ "\n"+ "[model_providers.headroom.env_http_headers]\n"+ 'X-Headroom-Project = "HEADROOM_PROJECT"\n'+ "\n"+ "[notice]\n"+ "hide_rate_limit_model_nudge = true\n"+ )++ wrap_mod._inject_codex_provider_config(8787)+ content = config_file.read_text()++ parsed = tomllib.loads(content) # must not raise: the original bug did+ assert content.count("[model_providers.headroom]") == 1+ assert "[model_providers.headroom.env_http_headers]" not in content+ assert content.count("env_http_headers") == 1 # inline form from the injection+ assert parsed["model_providers"]["headroom"]["base_url"] == "http://127.0.0.1:8787/v1"+ assert parsed["model_providers"]["headroom"]["env_http_headers"] == {+ "X-Headroom-Project": "HEADROOM_PROJECT"+ }+ assert parsed["profiles"]["default"]["model"] == "gpt-5"+ assert parsed["notice"] == {"hide_rate_limit_model_nudge": True}++ def test_inject_preserves_array_table_after_legacy_provider_table(+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path+ ) -> None:+ """Provider cleanup must stop before an unrelated TOML array table."""+ _set_test_home(monkeypatch, tmp_path)+ config_file = tmp_path / ".codex" / "config.toml"+ config_file.parent.mkdir()+ config_file.write_text(+ "[model_providers.headroom]\n"+ 'base_url = "http://127.0.0.1:8787/v1"\n\n'+ "[[profiles.custom.entries]]\n"+ 'name = "preserve"\n'+ )++ wrap_mod._inject_codex_provider_config(8787)++ parsed = tomllib.loads(config_file.read_text())+ assert parsed["profiles"]["custom"]["entries"] == [{"name": "preserve"}]+ assert parsed["model_providers"]["headroom"]["base_url"] == "http://127.0.0.1:8787/v1"++ def test_inject_leaves_invalid_provider_config_unchanged(+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path+ ) -> None:+ """Provider injection must not overwrite a config that cannot validate."""+ _set_test_home(monkeypatch, tmp_path)+ config_file = tmp_path / ".codex" / "config.toml"+ config_file.parent.mkdir()+ original = 'broken = [\n[model_providers.headroom]\nbase_url = "http://127.0.0.1:8787/v1"\n'+ config_file.write_text(original)++ wrap_mod._inject_codex_provider_config(8787)++ assert config_file.read_text() == original+def test_unwrap_restores_prior_headroom_provider_table(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: