Skip to content

Experience-Driven Black-Box Cost Control for Coding Agents

Windows: te toolprune measure no longer reports a failed probe as savings

Useless00789d577dee6ee
diff --git a/tests/test_toolcheck_measure.py b/tests/test_toolcheck_measure.py
new file mode 100644
index 0000000..d4fb1b2
--- /dev/null
+++ b/tests/test_toolcheck_measure.py
@@ -0,0 +1,46 @@
+"""`te toolprune measure` for Claude Code: what the two `claude -p` probes are started with."""
+import json
+import subprocess
+
+import pytest
+
+from tokeneconomizer.plugin import toolcheck
+
+
+def _fake_claude(monkeypatch, seen, stdout=None):
+ stdout = json.dumps({"usage": {"input_tokens": 100}}) if stdout is None else stdout
+
+ def run(cmd, **kw):
+ rec = {"cmd": cmd}
+ if "--settings" in cmd:
+ with open(cmd[cmd.index("--settings") + 1], encoding="utf-8") as fh:
+ rec["settings"] = json.load(fh)
+ seen.append(rec)
+ return subprocess.CompletedProcess(cmd, 0, stdout=stdout, stderr="")
+
+ monkeypatch.setattr(toolcheck.subprocess, "run", run)
+ monkeypatch.setattr(toolcheck, "claude_flags", lambda cli=None: [])
+
+
+def test_the_lean_probe_takes_the_deny_list_from_a_settings_file(monkeypatch):
+ # Hundreds of tool names on argv overflow cmd.exe's 8191-character line when `claude` is the
+ # npm `claude.cmd` shim, and the probe then reads no usage at all.
+ tools = [f"mcp__server_{i}__some_long_tool_name_{i}" for i in range(400)]
+ seen = []
+ _fake_claude(monkeypatch, seen)
+ toolcheck.measure(tools, cli="claude")
+ lean = seen[1]
+ assert sum(len(a) for a in lean["cmd"]) < 2000
+ assert lean["settings"] == {"permissions": {"deny": tools}}
+
+
[email protected]("stdout", [
+ "", # nothing started
+ "The command line is too long.", # what cmd.exe printed
+ json.dumps({"usage": {}}),
+ json.dumps({"is_error": True, "usage": {"input_tokens": 0}}),
+])
+def test_a_failed_probe_raises_instead_of_counting_zero(monkeypatch, stdout):
+ _fake_claude(monkeypatch, [], stdout=stdout)
+ with pytest.raises(RuntimeError):
+ toolcheck.measure(["mcp__x__y"], cli="claude")
diff --git a/tokeneconomizer/plugin/toolcheck.py b/tokeneconomizer/plugin/toolcheck.py
index 1d0de7d..b083107 100644
--- a/tokeneconomizer/plugin/toolcheck.py
+++ b/tokeneconomizer/plugin/toolcheck.py
@@ -218,14 +218,23 @@ def _prompt_tokens(extra_args, cli=None, timeout=180):
env = dict(os.environ, TE_SIDECALL="1")
cmd = [cli, "-p", "Reply with exactly: OK", "--output-format", "json", "--model", "haiku",
"--max-turns", "1"] + claude_flags(cli) + list(extra_args)
- with tempfile.TemporaryDirectory(prefix="te-measure-") as cwd:
+ # ignore_cleanup_errors: on Windows an MCP server the session started can still hold the cwd
+ # when `claude` exits (WinError 32), as in `_init_tools`.
+ with tempfile.TemporaryDirectory(prefix="te-measure-", ignore_cleanup_errors=True) as cwd:
out = subprocess.run(cmd, capture_output=True, encoding="utf-8", errors="replace", env=env,
cwd=cwd, timeout=timeout)
- data = json.loads(out.stdout or "{}")
+ try:
+ data = json.loads(out.stdout or "{}")
+ except ValueError:
+ data = {}
u = data.get("usage") or {}
- return (sum(int(u.get(k) or 0) for k in ("input_tokens", "cache_read_input_tokens",
- "cache_creation_input_tokens")),
- data.get("total_cost_usd"))
+ total = sum(int(u.get(k) or 0) for k in ("input_tokens", "cache_read_input_tokens",
+ "cache_creation_input_tokens"))
+ if data.get("is_error") or not total:
+ # A failed probe would otherwise count as 0 tokens, and the whole request as saved.
+ raise RuntimeError("`claude -p` reported no usage: "
+ + ((out.stderr or out.stdout or "").strip()[:300] or "no output"))
+ return total, data.get("total_cost_usd")
def _codex_prompt_chars(overrides, cli="codex", timeout=60):
@@ -271,7 +280,13 @@ def measure(tools, cli=None, host="claude"):
"probe_cost_usd": 0.0}
cli = cli or claude_cli()
full, c1 = _prompt_tokens([], cli=cli)
- lean, c2 = _prompt_tokens(["--disallowed-tools", ",".join(tools)], cli=cli)
+ # The deny list as a settings file, not `--disallowed-tools` on argv: hundreds of names
+ # overflow cmd.exe's 8191-character line when `claude` is the npm `claude.cmd` shim. Bare-name
+ # deny rules remove the same tools (see `denylist`), and are what `toolprune apply` writes.
+ with tempfile.TemporaryDirectory(prefix="te-deny-", ignore_cleanup_errors=True) as d:
+ settings = os.path.join(d, "settings.json")
+ _write_json(settings, {"permissions": {"deny": list(tools)}})
+ lean, c2 = _prompt_tokens(["--settings", settings], cli=cli)
return {"tools": list(tools), "prompt_tokens_full": full, "prompt_tokens_lean": lean,
"saved_per_request": full - lean,
"probe_cost_usd": round((c1 or 0) + (c2 or 0), 4)}