Skip to content

Experience-Driven Black-Box Cost Control for Coding Agents

measure: an empty Codex render raises, and the CLI reports a failed probe in one line

Useless0076dfdc7adfdab
diff --git a/tests/test_live_hooks.py b/tests/test_live_hooks.py
index 8c70bf0..5c37b98 100644
--- a/tests/test_live_hooks.py
+++ b/tests/test_live_hooks.py
@@ -1496,3 +1496,15 @@ def test_an_answer_to_a_failed_call_names_the_failure_event(env, repo):
answered.append((p["hook_event_name"], o["hookSpecificOutput"]["hookEventName"]))
assert answered and all(sent == named for sent, named in answered)
assert any(sent == "PostToolUseFailure" for sent, _ in answered)
+
+
+def test_measure_reports_a_failed_probe_in_one_line(env, repo, monkeypatch, capsys):
+ from tokeneconomizer.plugin import cli
+ for k, v in env.items():
+ monkeypatch.setenv(k, v)
+
+ def fail(tools, **kw):
+ raise RuntimeError("`claude -p` exited 1: boom")
+ monkeypatch.setattr(toolcheck, "measure", fail)
+ assert cli.main(["--host", "claude", "--cwd", repo, "toolprune", "measure", "Workflow"]) == 1
+ assert capsys.readouterr().out.strip() == "not measured: `claude -p` exited 1: boom"
diff --git a/tests/test_toolcheck_measure.py b/tests/test_toolcheck_measure.py
index a4cd8ca..355d6f6 100644
--- a/tests/test_toolcheck_measure.py
+++ b/tests/test_toolcheck_measure.py
@@ -58,5 +58,15 @@ def test_a_probe_that_exits_non_zero_raises_even_with_usage(monkeypatch):
monkeypatch.setattr(toolcheck.subprocess, "run", run)
monkeypatch.setattr(toolcheck, "claude_flags", lambda cli=None: [])
- with pytest.raises(RuntimeError, match="boom"):
+ with pytest.raises(RuntimeError, match="exited 1: boom"):
toolcheck.measure(["mcp__x__y"], cli="claude")
+
+
[email protected]("lean_stdout", ["", None])
+def test_a_codex_probe_that_prints_nothing_raises(monkeypatch, lean_stdout):
+ # An empty lean render would otherwise count as 0 characters: the whole request "saved".
+ replies = iter(["x" * 1000, lean_stdout])
+ monkeypatch.setattr(toolcheck.subprocess, "run", lambda cmd, **kw: subprocess.CompletedProcess(
+ cmd, 0, stdout=next(replies), stderr=""))
+ with pytest.raises(RuntimeError):
+ toolcheck.measure(["web_search"], cli="codex", host="codex")
diff --git a/tokeneconomizer/plugin/cli.py b/tokeneconomizer/plugin/cli.py
index 864bc43..30b9c74 100644
--- a/tokeneconomizer/plugin/cli.py
+++ b/tokeneconomizer/plugin/cli.py
@@ -380,7 +380,11 @@ def cmd_toolprune(args):
if not tools:
print("no proposal to measure")
return 1
- res = toolcheck.measure(tools, host=host)
+ try:
+ res = toolcheck.measure(tools, host=host)
+ except RuntimeError as e:
+ print(f"not measured: {e}")
+ return 1
print(json.dumps(res, indent=2))
return 0
return 2
diff --git a/tokeneconomizer/plugin/toolcheck.py b/tokeneconomizer/plugin/toolcheck.py
index c3e04e0..adbc66b 100644
--- a/tokeneconomizer/plugin/toolcheck.py
+++ b/tokeneconomizer/plugin/toolcheck.py
@@ -232,7 +232,8 @@ def _prompt_tokens(extra_args, cli=None, timeout=180):
"cache_creation_input_tokens"))
if out.returncode or data.get("is_error") or not total:
# A failed probe would otherwise count as 0 tokens, and the whole request as saved.
- raise RuntimeError("`claude -p` reported no usage: "
+ what = f"exited {out.returncode}" if out.returncode else "reported no usage"
+ raise RuntimeError(f"`claude -p` {what}: "
+ ((out.stderr or out.stdout or "").strip()[:300] or "no output"))
return total, data.get("total_cost_usd")
@@ -241,10 +242,11 @@ def _codex_prompt_chars(overrides, cli="codex", timeout=60):
cmd = [cli, "debug", "prompt-input"]
for table, key, value in overrides:
cmd += ["-c", f"{table}.{key}={value}" if table else f"{key}={value}"]
- with tempfile.TemporaryDirectory(prefix="te-measure-") as cwd:
+ with tempfile.TemporaryDirectory(prefix="te-measure-", ignore_cleanup_errors=True) as cwd:
out = subprocess.run(cmd + ["OK"], capture_output=True, encoding="utf-8", errors="replace",
cwd=cwd, timeout=timeout, env=dict(os.environ, TE_SIDECALL="1"))
- return len(out.stdout or "") if out.returncode == 0 else None
+ # Empty output is a failed probe too: counted as 0, the whole request would read as saved.
+ return len(out.stdout) if out.returncode == 0 and out.stdout else None
def measure(tools, cli=None, host="claude"):