Experience-Driven Black-Box Cost Control for Coding Agents
Python agentic-aiagentic-codingai-codingai-coding-assistantanthropicartificial-intelligenceclaude-codecodexcoding-agentcoding-agentscost-optimizationcost-reductiondeveloper-toolsgithub-copilotllmopen-sourceopenaiproductivitytoken-optimization
"""The Codex lane bills the OpenAI API directly, and the pieces that must agree about it.
OPENAI_API_KEY the credential -- required; the lane refuses at construction without it
OPENAI_BASE_URL OpenAI's own regional endpoint, for a region-scoped key; the global
endpoint otherwise
Nothing else selects an endpoint. Pinned here, because a launch banner stays green when any
of them is wrong:
* the RESOLUTION, including that no other variable moves the lane off the OpenAI API;
* the REQUEST CONTRACT config.toml expresses, and that it carries the key by NAME only;
* the CACHE ARITHMETIC in Codex's usage block, where `input_tokens` is the GRAND total
and forgetting to subtract the cache fields prices ~90% cached re-reads at 1.0x.
All offline: an explicit env dict, no network, no CLI binary, no credentials.
"""
import json
import os
import subprocess
import pytest
from te_harness import codex
from te_harness.guards import net_guard
from te_bench import spiral_stage as spiral_mech
from tokeneconomizer.toolprune import codex as codex_untool, stage as tooling
_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
_KEY = "sk-test-key-abcd"
_DIRECT = {"base_url": codex.OPENAI_DIRECT, "api_key": _KEY}
@pytest.mark.parametrize("env,base", [
({"OPENAI_API_KEY": _KEY}, "https://api.openai.com/v1"),
({"OPENAI_API_KEY": _KEY, "OPENAI_BASE_URL": "https://us.api.openai.com/v1/"},
"https://us.api.openai.com/v1"),
# Not consulted: nothing but OPENAI_BASE_URL names the endpoint.
({"OPENAI_API_KEY": _KEY, "ANTHROPIC_BASE_URL": "https://example.invalid"},
"https://api.openai.com/v1"),
])
def test_the_lane_resolves_to_the_openai_api(env, base):
assert codex.resolve_codex_routing(dict(env)) == {"base_url": base, "api_key": _KEY}
@pytest.mark.parametrize("env", [
{}, {"OPENAI_BASE_URL": "https://us.api.openai.com/v1", "ANTHROPIC_API_KEY": "sk-ant-x"}])
def test_without_the_openai_key_the_lane_is_refused_at_construction(env):
"""Not once per task: a lane with nothing to bill must die before task 0, and no other
variable stands in for the key."""
with pytest.raises(RuntimeError, match="OPENAI_API_KEY"):
codex.resolve_codex_routing(dict(env))
def test_validate_priceable_refuses_an_unknown_id():
"""`weights_for` hard-fails on an unpriced id at the END of a paid rollout, so the
runner checks at construction instead."""
codex.validate_priceable("gpt-5.6-sol", "codex") # must not raise
with pytest.raises(ValueError):
codex.validate_priceable("not-a-model", "codex")
# ---- the request contract -------------------------------------------------------------
def test_config_toggles_cover_exactly_the_mechanism_s_config_capabilities():
"""Layer 1 nominates the packages; the runner owns the mapping. They must line up.
A capability the mechanism reports as config-pruned but the runner has no toggle for is
an OVERSTATED saving on the arm whose entire claim is the saving.
"""
assert codex.CONFIG_CAPABILITIES == codex_untool.CONFIG_CAPABILITIES
def test_config_toggles_refuse_a_protected_capability():
with pytest.raises(RuntimeError, match="protected"):
codex.config_toggles(["collaboration", "apply_patch"])
def test_web_is_blocked_on_every_arm_including_an_empty_mask():
"""Contamination control, not a mechanism: the answer key is a fetch away.
Mirrors `swebench.network_tool_block` on the other two lanes. The consequence is
deliberate and documented -- with web off in the baseline too, the `web` package
contributes no untooling saving on this benchmark.
"""
top, tables = codex.config_toggles([])
# The TOP-LEVEL spelling only. codex-cli 0.153.1 deprecated `[features].web_search`
# ("web search is enabled by default; set `web_search` to disabled at the top level")
# and emits an error item into the transcript when it is present, so no table entry is
# written -- see codex._TABLE_TOGGLES["web"].
assert 'web_search = "disabled"' in top
assert not any(t == "features" and "web_search" in line for t, line in tables)
assert not any(t == "tools" and "web_search" in line for t, line in tables)
# ...and asking for `web` as well must not duplicate a key: Codex rejects that.
top2, tables2 = codex.config_toggles(["web"])
assert len(top2) == len(set(top2)) and len(tables2) == len(set(tables2))
assert sorted(top) == sorted(top2)
def test_network_block_can_be_lifted(monkeypatch):
monkeypatch.setenv("TE_ALLOW_NETWORK_TOOLS", "1")
top, tables = codex.config_toggles([])
assert top == [] and tables == []
def test_config_toml_names_the_openai_api_and_the_key_by_name_only():
txt = codex.config_toml(routing=_DIRECT, model="gpt-5.6-sol", effort="high",
capabilities=())
assert 'model_provider = "openai_direct"' in txt
assert "[model_providers.openai_direct]" in txt
assert txt.count("[model_providers.") == 1
assert f'base_url = "{codex.OPENAI_DIRECT}"' in txt
assert 'env_key = "OPENAI_API_KEY"' in txt and codex.KEY_ENV == "OPENAI_API_KEY"
assert 'wire_api = "responses"' in txt
assert 'trust_level = "trusted"' in txt
assert "requires_openai_auth" not in txt
# The credential VALUE never goes in the file -- only the variable name.
assert _KEY not in txt
def test_a_regional_endpoint_is_written_as_given():
r = codex.resolve_codex_routing({"OPENAI_API_KEY": _KEY,
"OPENAI_BASE_URL": "https://eu.api.openai.com/v1"})
txt = codex.config_toml(routing=r, model="gpt-5.6-sol", effort="high")
assert 'base_url = "https://eu.api.openai.com/v1"' in txt
def test_the_key_reaches_the_container_by_name_only():
assert codex.KEY_ENV not in codex.run_env()
assert _KEY not in codex.run_env().values()
# ---- accounting: the cache arithmetic -------------------------------------------------
def _stream(*events):
return "\n".join(json.dumps(e) for e in events)
def test_row_usage_decomposes_the_grand_input_total():
"""`input_tokens` INCLUDES both cache fields; fresh input is what is left.
Folding the cache fields into input_tokens instead would price cached re-reads at 1.0x
where the vendor bills 0.10x -- a one-directional error that would make this lane's rows
look more expensive than the other lanes' on identical work.
"""
row = codex.row_usage(_stream(
{"type": "thread.started", "thread_id": "th-1"},
{"type": "item.completed", "item": {"type": "command_execution",
"aggregated_output": "ok"}},
{"type": "item.completed", "item": {"type": "file_change"}},
{"type": "item.completed", "item": {"type": "agent_message"}},
{"type": "turn.completed", "usage": {
"input_tokens": 10_000, "cached_input_tokens": 9_000,
"cache_write_input_tokens": 400, "output_tokens": 300,
"reasoning_output_tokens": 120}},
))
assert row["input_tokens"] == 600 # 10000 - 9000 - 400
assert row["cache_read_input_tokens"] == 9_000
assert row["cache_creation_input_tokens"] == 400
assert row["output_tokens"] == 420 # output + reasoning, as billed
assert row["tokens_total"] == 600 + 9_000 + 400 + 420
assert row["num_turns"] == 3 # steps, not turn.completed events
assert row["turns_basis"] == "codex_steps"
assert row["thread_id"] == "th-1"
assert row["tokens_source"] == "stream_usage"
def test_row_usage_accumulates_across_turns():
row = codex.row_usage(_stream(
{"type": "turn.completed", "usage": {"input_tokens": 100,
"cached_input_tokens": 10,
"output_tokens": 5}},
{"type": "turn.completed", "usage": {"input_tokens": 200,
"cached_input_tokens": 20,
"output_tokens": 7}},
))
assert row["input_tokens"] == 270 and row["cache_read_input_tokens"] == 30
assert row["output_tokens"] == 12
def test_a_killed_stream_records_unknown_not_zero():
"""A guard/timeout kill takes the usage block with it.
None, not 0: a task recorded as free would be averaged in as a cheap success on
precisely the guarded arm the spiral mechanism must show saving on.
"""
row = codex.row_usage(_stream(
{"type": "item.completed", "item": {"type": "command_execution"}}))
assert row["input_tokens"] is None
assert row["tokens_total"] is None
assert row["tokens_source"] == "no_turn_event"
def test_merge_usage_propagates_unknown():
good = codex.row_usage(_stream({"type": "turn.completed", "usage": {
"input_tokens": 100, "cached_input_tokens": 10, "output_tokens": 5}}))
dead = codex.row_usage("")
merged = codex.merge_usage(good, dead)
assert merged["input_tokens"] is None, "a half-known total must not read as known"
assert merged["tokens_source"] != "stream_usage"
both = codex.merge_usage(good, good)
assert both["input_tokens"] == 180 and both["tokens_source"] == "stream_usage"
def test_merge_keeps_the_first_thread_id():
"""A later resume must continue the FIRST segment's thread, never a new session."""
a = codex.row_usage(_stream({"type": "thread.started", "thread_id": "th-A"},
{"type": "turn.completed", "usage": {"input_tokens": 1}}))
b = codex.row_usage(_stream({"type": "thread.started", "thread_id": "th-B"},
{"type": "turn.completed", "usage": {"input_tokens": 1}}))
assert codex.merge_usage(a, b)["thread_id"] == "th-A"
# ---- the net guard's payload spelling -------------------------------------------------
@pytest.mark.parametrize("payload", [
{"tool_name": "Bash", "tool_input": {"command": "curl https://github.com/x"}},
{"toolName": "bash", "toolArgs": {"command": "curl https://github.com/x"}},
{"tool": {"name": "shell",
"input": {"command": ["bash", "-lc", "curl https://github.com/x"]}}},
{"item": {"type": "command_execution", "command": "curl https://github.com/x"}},
])
def test_guard_reads_every_payload_spelling(payload):
"""Codex's `pre_tool_use` spelling is not captured off a live hook, and its hooks table
is not strictly validated -- so the guard must not depend on one key name. An
unreadable payload means `decide("")`, exit 0, and a lane that looks armed and guards
nothing."""
cmd = net_guard.hook_command(payload)
assert cmd, payload
assert net_guard.decide(cmd)[0] is True
def test_guard_still_returns_empty_for_a_payload_with_no_command():
assert net_guard.hook_command({"nothing": 1}) == ""
assert net_guard.decide("")[0] is False
# ---- the mechanism gates --------------------------------------------------------------
def test_the_codex_label_gets_the_codex_mechanisms():
"""The harness registers exactly one Codex label, and both mechanism gates must open on
it -- a gate keyed on a label the registry does not have would construct no controller
while every row still carried the arm's name."""
from te_harness import swebench, webbench
assert "codex" in tooling._CODEX_AGENTS and "codex" in spiral_mech._CODEX_AGENTS
for runners in (swebench.RUNNERS, webbench.RUNNERS):
assert [a for a in runners if a.startswith("codex")] == ["codex"]
# ---- the results tree ---------------------------------------------------------------
def _sh(script):
p = subprocess.run(["bash", "-c", ". scripts/launch/_env.sh; " + script], cwd=_REPO,
capture_output=True, text=True)
assert p.returncode == 0, p.stderr
return p.stdout.strip()
def test_the_codex_tree_is_named_by_agent_domain_and_model():
assert _sh('te_tree_name codex swebench "$(te_model_tag gpt-5.6-sol)"') == \
"codex_swebench_gpt56sol"
def test_a_model_tag_never_contains_a_path_separator():
"""A slash in the tag would turn one tree into a nested PATH and split the cell."""
for model in ("vendor/gpt-5.6-sol", "gpt-5.6-sol"):
assert "/" not in _sh(f'te_model_tag "{model}"'), model
def test_claude_model_tags_are_unchanged():
"""The tag helper is shared, so the Codex lane must not move any stored tree's name."""
for model, expect in (("claude-opus-4-8", "opus48"), ("claude-opus-4.8", "opus48"),
("claude-sonnet-5", "sonnet5"), ("auto", ""), ("", "")):
assert _sh(f'te_model_tag "{model}"') == expect, model
def test_the_mechanism_call_is_a_tool_free_responses_request_to_the_openai_api(monkeypatch):
"""The Codex twin of the Claude lane's Messages call: the lane's own model, on its own
key, at its own endpoint, with no tools offered -- a pure text completion."""
import io, urllib.request
seen = {}
class _Resp(io.BytesIO):
def __enter__(self):
return self
def __exit__(self, *a):
return False
def fake_urlopen(req, timeout=0):
seen["body"] = json.loads(req.data.decode())
seen["url"] = req.full_url
seen["headers"] = {k.lower(): v for k, v in req.header_items()}
return _Resp(json.dumps({"output": [{"type": "message", "content": [
{"type": "output_text", "text": "ok"}]}], "usage": {"input_tokens": 3,
"output_tokens": 1}}).encode())
monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
text, eff = codex.optimizer_call("Reply with the single word: ok",
configured_model="gpt-5.6-terra",
routing=_DIRECT, max_tokens=16)
assert text == "ok" and eff > 0
assert seen["url"] == "https://api.openai.com/v1/responses"
assert seen["headers"]["authorization"] == f"Bearer {_KEY}"
assert "user-agent" not in seen["headers"]
body = seen["body"]
assert body["model"] == "gpt-5.6-terra"
assert "tools" not in body and "tool_choice" not in body
assert body["max_output_tokens"] == 16 and body["store"] is False
def test_the_model_check_reads_the_openai_catalog(monkeypatch):
import io, urllib.request
class _Resp(io.BytesIO):
def __enter__(self):
return self
def __exit__(self, *a):
return False
seen = {}
def fake_urlopen(req, timeout=0):
seen["url"] = req.full_url
seen["auth"] = req.get_header("Authorization")
return _Resp(json.dumps({"object": "list", "data": [
{"id": "gpt-5.6-sol"}, {"id": "gpt-5.6-terra"}]}).encode())
codex._CATALOG.clear()
monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
assert codex.verify_model("gpt-5.6-sol", _DIRECT) == (True, "")
ok, why = codex.verify_model("gpt-9", _DIRECT)
assert ok is False and "gpt-5.6-terra" in why
assert seen == {"url": "https://api.openai.com/v1/models", "auth": f"Bearer {_KEY}"}
codex._CATALOG.clear()
def test_an_unreadable_catalog_is_inconclusive_not_a_refusal(monkeypatch):
import urllib.request
def _boom(*a, **k):
raise OSError("connection refused")
codex._CATALOG.clear()
monkeypatch.setattr(urllib.request, "urlopen", _boom)
served, why = codex.verify_model("gpt-5.6-sol", _DIRECT)
assert served is None and "unreadable" in why
codex._CATALOG.clear()
def test_the_skills_package_is_pruned_by_disabling_every_bundled_system_skill():
"""0.153.1 has no switch for the <skills_instructions> listing, but disabling each
bundled skill through `[[skills.config]]` removes the whole block. The entries name
in-container paths and come last in the file, because an array-of-tables header
swallows any key after it."""
txt = codex.config_toml(routing=_DIRECT, model="gpt-5.6-terra", effort="high",
capabilities=["skills"], system_skills=("alpha", "beta"))
blocks = txt.split("[[skills.config]]")
assert len(blocks) == 3
assert f'path = "{codex.C_HOME}/skills/.system/alpha/SKILL.md"' in blocks[1]
assert "enabled = false" in blocks[1] and "enabled = false" in blocks[2]
assert txt.index("[projects.") < txt.index("[[skills.config]]")
# without the package in the mask no entry is written; the default set covers 0.153.1
plain = codex.config_toml(routing=_DIRECT, model="gpt-5.6-terra", effort="high",
capabilities=[])
assert "skills.config" not in plain
assert len(codex.skills_config_entries(None)) == len(codex.DEFAULT_SYSTEM_SKILLS) == 6
assert "skills" in codex.CONFIG_CAPABILITIES
def _alive(pid):
"""True while `pid` runs (a zombie awaiting its reaper counts as gone)."""
try:
with open(f"/proc/{pid}/stat") as fh:
return fh.read().split()[2] != "Z"
except OSError:
return False
def _fake_cli(tmp_path, body):
"""A stand-in for the npm `codex` shim: it starts a long-lived CHILD, as the real shim
execs the native ELF, records the child's pid, then runs `body`."""
pidfile = tmp_path / "child.pid"
shim = tmp_path / "codex"
shim.write_text(f"#!/bin/sh\nsleep 600 &\necho $! > {pidfile}\n{body}\n")
shim.chmod(0o755)
return str(shim), pidfile
def _assert_reaped(pidfile):
import time
pid = int(pidfile.read_text())
for _ in range(50):
if not _alive(pid):
return
time.sleep(0.1)
raise AssertionError(f"the CLI's child {pid} outlived the probe")
@pytest.mark.skipif(not os.path.isdir("/proc"), reason="needs /proc")
def test_the_skills_probe_kills_the_cli_it_started_not_just_the_shim(tmp_path):
"""`proc.kill()` reaches only the Node shim, and the native CLI the shim started would
sit "waiting for network" for good on an offline host. The probe owns a process group
and kills all of it."""
shim, pidfile = _fake_cli(tmp_path, (
'mkdir -p "$CODEX_HOME/skills/.system/alpha"\n'
'touch "$CODEX_HOME/skills/.system/alpha/SKILL.md"\n'
'touch "$CODEX_HOME/skills/.system/.codex-system-skills.marker"\n'
"wait"))
assert codex.bundled_system_skills(binary=shim, home=str(tmp_path), timeout=20) == ["alpha"]
_assert_reaped(pidfile)
@pytest.mark.skipif(not os.path.isdir("/proc"), reason="needs /proc")
def test_the_config_validator_kills_a_hung_cli_and_its_children(tmp_path):
shim, pidfile = _fake_cli(tmp_path, "sleep 600")
err = codex.validate_config('model = "gpt-5.6-sol"\n', binary=shim, home=str(tmp_path),
timeout=2)
assert err.startswith("could not run the config validator")
_assert_reaped(pidfile)
def test_bundled_system_skills_are_read_from_the_installed_cli():
"""The skills toggle disables bundled skills BY PATH, so the set must come from the CLI
that will run, not from a list written for one version. Offline: the throwaway run is
killed at the skills marker before it can wait for a network it does not have."""
import shutil
if not (os.environ.get("TE_CODEX_BIN") or shutil.which("codex")):
pytest.skip("codex-cli not installed")
found = codex.bundled_system_skills()
assert found, "discovery returned nothing; the runner would fall back to the static set"
assert all("/" not in n for n in found)
# 0.153.1 ships these; a newer CLI may add to the set, which is exactly why it is read.
assert {"skill-creator", "skill-installer"} <= set(found)
# ---------------------------------------------------------------------------------------
# The instruction package (2026-09-11): chat-session guidance in the base prompt is
# removed through a lean copy written to model_instructions_file; operational sections
# stay byte-for-byte.
# ---------------------------------------------------------------------------------------
_FAKE_BASE = (
"You are Codex.\n\n# Personality\n\nBe charming.\n\n## Writing style\n\nShort.\n\n"
"# Working with the user\n\nChannels.\n\nYou end your turn by sending a final message.\n\n"
"When you run out of context, continue.\n\n"
"# Rules for getting work done\n\n- use rg\n\n## Autonomy and persistence\n\nAct.\n\n"
"# Using skills\n\nSkills intro.\n\n### How to use skills\n\nRead SKILL.md.\n")
def test_lean_instructions_drops_only_the_pruned_sections_and_keeps_the_rest_verbatim():
lean = codex.lean_instructions(_FAKE_BASE, ["interactive_guidance"])
assert "# Personality" not in lean and "## Writing style" not in lean
assert "# Working with the user" not in lean and "Channels." not in lean
assert "When you run out of context, continue." in lean # compaction paragraph kept
# the stopping rule is operational and is kept too: without it the agent has no
# statement of how a turn ends, and runs more model calls than its baseline.
assert "You end your turn by sending a final message." in lean
for kept in ("You are Codex.", "# Rules for getting work done\n\n- use rg",
"## Autonomy and persistence\n\nAct.", "# Using skills", "### How to use skills"):
assert kept in lean
both = codex.lean_instructions(_FAKE_BASE, ["interactive_guidance", "skills"])
assert "# Using skills" not in both and "Read SKILL.md." not in both
assert "# Rules for getting work done" in both
# nothing to drop -> no file, the CLI keeps its built-in prompt
assert codex.lean_instructions(_FAKE_BASE, ["collaboration", "goals"]) is None
assert codex.lean_instructions(None, ["interactive_guidance"]) is None
def test_the_captured_base_prompt_exists_for_the_pinned_cli_and_for_no_other():
base = codex.base_prompt_for("codex-cli 0.153.1")
assert base and base.startswith("You are Codex") and len(base) // 4 > 4000
assert codex.base_prompt_for("codex-cli 9.9.9") is None
assert codex.base_prompt_for("") is None
lean = codex.lean_instructions(base, ["interactive_guidance", "skills"])
assert len(lean) < 0.45 * len(base)
for header in ("# Rules for getting work done", "## File editing constraints",
"## Autonomy and persistence", "# Destructive Actions"):
assert header in lean
for header in codex.INSTRUCTION_SECTIONS["interactive_guidance"] + codex.INSTRUCTION_SECTIONS["skills"]:
assert ("\n" + header + "\n") not in lean
def test_config_toml_names_the_instructions_file_only_when_a_lean_prompt_exists():
with_file = codex.config_toml(routing=_DIRECT, model="gpt-5.6-terra", effort="high",
capabilities=["interactive_guidance"],
instructions="lean text")
assert f'model_instructions_file = "{codex.INSTRUCTIONS_PATH}"' in with_file
without = codex.config_toml(routing=_DIRECT, model="gpt-5.6-terra", effort="high",
capabilities=["interactive_guidance"], instructions=None)
assert "model_instructions_file" not in without
assert "interactive_guidance" in codex.CONFIG_CAPABILITIES
def test_codex_trimech_names_no_endpoint():
"""The arm is endpoint-agnostic by construction: nothing in its config names one.
Stated as a test because the three mechanisms reach the runner through three different
channels (prompt prefix, config.toml, live stream) and each one could have grown an
endpoint assumption; the endpoint is the runner's to resolve.
"""
from te_bench.arms import ARMS
import dataclasses
cfg = ARMS["codex-trimech"].cfg()
flat = repr(dataclasses.asdict(cfg)) if dataclasses.is_dataclass(cfg) else repr(cfg)
for token in ("openai", "copilot", "http", "base_url", "api_key"):
assert token not in flat.lower(), f"{token!r} leaked into the arm's config"
def test_rtk_compliance_is_read_off_the_trace(tmp_path):
"""On Codex rtk is an instruction, not a hook, so whether it did anything is whether the
agent typed `rtk`. Counted from the transcript like answer_key_attempts; None when there
is no transcript, so a lane of zeros stays distinguishable from a lane of no data."""
from te_harness import codex
from fixtures import write_codex_transcript
p = str(tmp_path / "t.jsonl")
write_codex_transcript(p, [
{"type": "command_execution", "command": '/bin/bash -lc "rtk git status"',
"aggregated_output": "ok", "exit_code": 0},
{"type": "command_execution", "command": '/bin/bash -lc "pytest -q"',
"aggregated_output": "1 passed", "exit_code": 0},
{"type": "command_execution", "command": "/bin/bash -lc '/usr/local/bin/rtk proxy make'",
"aggregated_output": "", "exit_code": 0},
{"type": "file_change", "changes": [{"path": "a.py", "kind": "update"}]},
])
got = codex.rtk_compliance(p)
assert got == {"rtk_shell_commands": 3, "rtk_prefixed_commands": 2,
"rtk_compliance": 0.667}
assert codex.rtk_compliance(str(tmp_path / "missing.jsonl")) == {
"rtk_shell_commands": None, "rtk_prefixed_commands": None, "rtk_compliance": None}
assert codex._unwrap_shell('/bin/bash -lc "rtk ls"') == "rtk ls"
assert codex._unwrap_shell("sh -c 'rtk ls'") == "rtk ls"
assert codex._unwrap_shell("rtk ls") == "rtk ls"
# ---- MCP servers reach Codex through config.toml -----------------------------------------
def _spec():
return {"serena": {"command": "/tmp/agent-home/.local/bin/serena",
"args": ["start-mcp-server", "--project", "/testbed", "--context", "codex"],
"env": {"PATH": "/tmp/agent-home/.local/bin:/usr/bin"}}}
def test_config_toml_writes_an_mcp_server_table_the_cli_validates():
"""The block is emitted before the array-of-tables skills entries (which must be last),
with args as a TOML array and env as a sub-table. On 0.153.1 `--strict-config` accepts
command/args/env/cwd/startup_timeout_sec and refuses an unknown key, so the acceptance
is checked against the real CLI when one is installed."""
from te_harness import codex
t = codex.config_toml(routing=_DIRECT, model="gpt-5.6-terra", effort="high",
capabilities=("skills",), mcp_servers=_spec())
i = t.index("[mcp_servers.serena]")
assert t.index("[[skills.config]]") > i
body = t[i:]
assert 'command = "/tmp/agent-home/.local/bin/serena"' in body
assert 'args = ["start-mcp-server", "--project", "/testbed", "--context", "codex"]' in body
assert "startup_timeout_sec = 90" in body
assert "[mcp_servers.serena.env]" in body and 'PATH = "/tmp/agent-home/.local/bin:/usr/bin"' in body
# no servers -> no table at all: the baseline config is unchanged by this feature
assert "mcp_servers" not in codex.config_toml(routing=_DIRECT, model="gpt-5.6-terra",
effort="high")
import shutil, os, tempfile
if shutil.which("codex") or os.environ.get("TE_CODEX_BIN"):
home = tempfile.mkdtemp(dir=os.path.dirname(os.path.abspath(__file__)))
try:
err = codex.validate_config(t, home=home)
assert not (err or "").strip(), err
finally:
shutil.rmtree(home, ignore_errors=True)
def test_mcp_tool_calls_counts_completed_items_and_keeps_a_sample_of_what_it_cannot_read(tmp_path):
"""A configured server is not a used one. The reader counts `mcp_tool_call` items at
completion, per server/tool; an item whose tool name sits somewhere unexpected is counted
as unrecognised and ONE raw sample is kept.
The shape below is the one codex-cli 0.153.1 emits (`item.completed`, item keys: id,
type, server, tool, arguments, result, error, status). Codex also makes calls of its OWN,
such as `list_mcp_resources` on server `codex`, which is why the count is split per
server: `mcp_tool_calls` includes it, `serena_tool_calls` does not."""
from te_harness import codex
import json
p = str(tmp_path / "t.jsonl")
live_shape = {"id": "item_2", "type": "mcp_tool_call", "server": "codex",
"tool": "list_mcp_resources", "arguments": {},
"result": {"content": [{"type": "text", "text": "{\"resources\":[]}"}],
"structured_content": None},
"error": None, "status": "completed"}
with open(p, "w") as fh:
fh.write(json.dumps({"type": "item.started", "item": {"type": "mcp_tool_call", "server": "serena", "tool": "find_symbol"}}) + "\n")
fh.write(json.dumps({"type": "item.completed", "item": {"type": "mcp_tool_call", "server": "serena", "tool": "find_symbol", "status": "completed"}}) + "\n")
fh.write(json.dumps({"type": "item.completed", "item": live_shape}) + "\n")
fh.write(json.dumps({"type": "item.completed", "item": {"type": "mcp_tool_call", "invocation": {"server": "serena", "tool": "get_symbols_overview"}}}) + "\n")
fh.write(json.dumps({"type": "item.completed", "item": {"type": "mcp_tool_call", "weird": 1}}) + "\n")
fh.write(json.dumps({"type": "item.completed", "item": {"type": "command_execution", "command": "ls"}}) + "\n")
got = codex.mcp_tool_calls(p, server="serena")
assert got["mcp_tool_calls"] == 3 and got["serena_tool_calls"] == 2
assert got["mcp_tools_used"] == {"serena/find_symbol": 1, "serena/get_symbols_overview": 1,
"codex/list_mcp_resources": 1}
assert got["mcp_unrecognised_items"] == 1 and '"weird"' in got["mcp_item_sample"]
none = codex.mcp_tool_calls(str(tmp_path / "missing.jsonl"), server="serena")
assert none["mcp_tool_calls"] is None and none["serena_tool_calls"] is None
def test_the_codex_lane_hosts_every_scaffold_and_wires_an_mcp_one_through_the_config(
tmp_path, monkeypatch):
"""The Codex lane refuses no scaffold: rtk runs in its own Codex mode, repomap writes
AGENTS.md, and serena -- an MCP server -- is written into config.toml after setup,
because `_start` writes the config before the scaffold exists. Both halves pinned: no
scaffold name is refused at construction, and a scaffold that returns `mcp_servers`
causes exactly one config rewrite carrying that spec."""
from te_harness import swebench
assert not hasattr(swebench, "_CODEX_UNAVAILABLE_SCAFFOLDS")
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
for name in ("rtk", "repomap", "serena"):
with pytest.raises(RuntimeError, match="nothing to bill"): # past the scaffold gate
swebench.CodexSwebenchRunner(agent="codex", scaffold=name)
r = object.__new__(swebench.CodexSwebenchRunner)
writes = []
monkeypatch.setattr(swebench.ClaudeSwebenchRunner, "_apply_scaffold",
lambda self, name, task, rd: ({"arm_setup": "serena", "mcp_servers": _spec(),
"mcp_config": "/tmp/agent-home/scaffold.mcp.json"},
"/tmp/agent-home/scaffold.mcp.json"))
r._write_config = lambda name: writes.append((name, r._mcp_servers))
notes, cfg = r._apply_scaffold("c1", {"instance_id": "x"}, str(tmp_path))
assert writes == [("c1", _spec())]
assert "mcp_servers" not in notes # a dict never lands in row.json
assert notes["mcp_wiring"] == "config.toml" and notes["mcp_servers_configured"] == ["serena"]
assert cfg == "/tmp/agent-home/scaffold.mcp.json"
# and a scaffold that asks for nothing rewrites nothing
writes.clear()
monkeypatch.setattr(swebench.ClaudeSwebenchRunner, "_apply_scaffold",
lambda self, name, task, rd: ({"arm_setup": "rtk"}, None))
r._apply_scaffold("c1", {"instance_id": "x"}, str(tmp_path))
assert writes == [] and r._mcp_servers is None