Experience-Driven Black-Box Cost Control for Coding Agents
Python agentic-aiagentic-codingai-codingai-coding-assistantanthropicartificial-intelligenceclaude-codecodexcoding-agentcoding-agentscost-optimizationcost-reductiondeveloper-toolsgithub-copilotllmopen-sourceopenaiproductivitytoken-optimization
measure: an empty Codex render raises, and the CLI reports a failed probe in one line
Useless0076dfdc7adfdab
diff --git a/tests/test_live_hooks.py b/tests/test_live_hooks.pyindex 8c70bf0..5c37b98 100644--- a/tests/test_live_hooks.py+++ b/tests/test_live_hooks.py@@ -1496,3 +1496,15 @@ def test_an_answer_to_a_failed_call_names_the_failure_event(env, repo):answered.append((p["hook_event_name"], o["hookSpecificOutput"]["hookEventName"]))assert answered and all(sent == named for sent, named in answered)assert any(sent == "PostToolUseFailure" for sent, _ in answered)+++def test_measure_reports_a_failed_probe_in_one_line(env, repo, monkeypatch, capsys):+ from tokeneconomizer.plugin import cli+ for k, v in env.items():+ monkeypatch.setenv(k, v)++ def fail(tools, **kw):+ raise RuntimeError("`claude -p` exited 1: boom")+ monkeypatch.setattr(toolcheck, "measure", fail)+ assert cli.main(["--host", "claude", "--cwd", repo, "toolprune", "measure", "Workflow"]) == 1+ assert capsys.readouterr().out.strip() == "not measured: `claude -p` exited 1: boom"diff --git a/tests/test_toolcheck_measure.py b/tests/test_toolcheck_measure.pyindex a4cd8ca..355d6f6 100644--- a/tests/test_toolcheck_measure.py+++ b/tests/test_toolcheck_measure.py@@ -58,5 +58,15 @@ def test_a_probe_that_exits_non_zero_raises_even_with_usage(monkeypatch):monkeypatch.setattr(toolcheck.subprocess, "run", run)monkeypatch.setattr(toolcheck, "claude_flags", lambda cli=None: [])- with pytest.raises(RuntimeError, match="boom"):+ with pytest.raises(RuntimeError, match="exited 1: boom"):toolcheck.measure(["mcp__x__y"], cli="claude")++[email protected]("lean_stdout", ["", None])+def test_a_codex_probe_that_prints_nothing_raises(monkeypatch, lean_stdout):+ # An empty lean render would otherwise count as 0 characters: the whole request "saved".+ replies = iter(["x" * 1000, lean_stdout])+ monkeypatch.setattr(toolcheck.subprocess, "run", lambda cmd, **kw: subprocess.CompletedProcess(+ cmd, 0, stdout=next(replies), stderr=""))+ with pytest.raises(RuntimeError):+ toolcheck.measure(["web_search"], cli="codex", host="codex")diff --git a/tokeneconomizer/plugin/cli.py b/tokeneconomizer/plugin/cli.pyindex 864bc43..30b9c74 100644--- a/tokeneconomizer/plugin/cli.py+++ b/tokeneconomizer/plugin/cli.py@@ -380,7 +380,11 @@ def cmd_toolprune(args):if not tools:print("no proposal to measure")return 1- res = toolcheck.measure(tools, host=host)+ try:+ res = toolcheck.measure(tools, host=host)+ except RuntimeError as e:+ print(f"not measured: {e}")+ return 1print(json.dumps(res, indent=2))return 0return 2diff --git a/tokeneconomizer/plugin/toolcheck.py b/tokeneconomizer/plugin/toolcheck.pyindex c3e04e0..adbc66b 100644--- a/tokeneconomizer/plugin/toolcheck.py+++ b/tokeneconomizer/plugin/toolcheck.py@@ -232,7 +232,8 @@ def _prompt_tokens(extra_args, cli=None, timeout=180):"cache_creation_input_tokens"))if out.returncode or data.get("is_error") or not total:# A failed probe would otherwise count as 0 tokens, and the whole request as saved.- raise RuntimeError("`claude -p` reported no usage: "+ what = f"exited {out.returncode}" if out.returncode else "reported no usage"+ raise RuntimeError(f"`claude -p` {what}: "+ ((out.stderr or out.stdout or "").strip()[:300] or "no output"))return total, data.get("total_cost_usd")@@ -241,10 +242,11 @@ def _codex_prompt_chars(overrides, cli="codex", timeout=60):cmd = [cli, "debug", "prompt-input"]for table, key, value in overrides:cmd += ["-c", f"{table}.{key}={value}" if table else f"{key}={value}"]- with tempfile.TemporaryDirectory(prefix="te-measure-") as cwd:+ with tempfile.TemporaryDirectory(prefix="te-measure-", ignore_cleanup_errors=True) as cwd:out = subprocess.run(cmd + ["OK"], capture_output=True, encoding="utf-8", errors="replace",cwd=cwd, timeout=timeout, env=dict(os.environ, TE_SIDECALL="1"))- return len(out.stdout or "") if out.returncode == 0 else None+ # Empty output is a failed probe too: counted as 0, the whole request would read as saved.+ return len(out.stdout) if out.returncode == 0 and out.stdout else Nonedef measure(tools, cli=None, host="claude"):