Experience-Driven Black-Box Cost Control for Coding Agents
Python agentic-aiagentic-codingai-codingai-coding-assistantanthropicartificial-intelligenceclaude-codecodexcoding-agentcoding-agentscost-optimizationcost-reductiondeveloper-toolsgithub-copilotllmopen-sourceopenaiproductivitytoken-optimization
Windows: te toolprune measure no longer reports a failed probe as savings
Useless00789d577dee6ee
diff --git a/tests/test_toolcheck_measure.py b/tests/test_toolcheck_measure.pynew file mode 100644index 0000000..d4fb1b2--- /dev/null+++ b/tests/test_toolcheck_measure.py@@ -0,0 +1,46 @@+"""`te toolprune measure` for Claude Code: what the two `claude -p` probes are started with."""+import json+import subprocess++import pytest++from tokeneconomizer.plugin import toolcheck+++def _fake_claude(monkeypatch, seen, stdout=None):+ stdout = json.dumps({"usage": {"input_tokens": 100}}) if stdout is None else stdout++ def run(cmd, **kw):+ rec = {"cmd": cmd}+ if "--settings" in cmd:+ with open(cmd[cmd.index("--settings") + 1], encoding="utf-8") as fh:+ rec["settings"] = json.load(fh)+ seen.append(rec)+ return subprocess.CompletedProcess(cmd, 0, stdout=stdout, stderr="")++ monkeypatch.setattr(toolcheck.subprocess, "run", run)+ monkeypatch.setattr(toolcheck, "claude_flags", lambda cli=None: [])+++def test_the_lean_probe_takes_the_deny_list_from_a_settings_file(monkeypatch):+ # Hundreds of tool names on argv overflow cmd.exe's 8191-character line when `claude` is the+ # npm `claude.cmd` shim, and the probe then reads no usage at all.+ tools = [f"mcp__server_{i}__some_long_tool_name_{i}" for i in range(400)]+ seen = []+ _fake_claude(monkeypatch, seen)+ toolcheck.measure(tools, cli="claude")+ lean = seen[1]+ assert sum(len(a) for a in lean["cmd"]) < 2000+ assert lean["settings"] == {"permissions": {"deny": tools}}++[email protected]("stdout", [+ "", # nothing started+ "The command line is too long.", # what cmd.exe printed+ json.dumps({"usage": {}}),+ json.dumps({"is_error": True, "usage": {"input_tokens": 0}}),+])+def test_a_failed_probe_raises_instead_of_counting_zero(monkeypatch, stdout):+ _fake_claude(monkeypatch, [], stdout=stdout)+ with pytest.raises(RuntimeError):+ toolcheck.measure(["mcp__x__y"], cli="claude")diff --git a/tokeneconomizer/plugin/toolcheck.py b/tokeneconomizer/plugin/toolcheck.pyindex 1d0de7d..b083107 100644--- a/tokeneconomizer/plugin/toolcheck.py+++ b/tokeneconomizer/plugin/toolcheck.py@@ -218,14 +218,23 @@ def _prompt_tokens(extra_args, cli=None, timeout=180):env = dict(os.environ, TE_SIDECALL="1")cmd = [cli, "-p", "Reply with exactly: OK", "--output-format", "json", "--model", "haiku","--max-turns", "1"] + claude_flags(cli) + list(extra_args)- with tempfile.TemporaryDirectory(prefix="te-measure-") as cwd:+ # ignore_cleanup_errors: on Windows an MCP server the session started can still hold the cwd+ # when `claude` exits (WinError 32), as in `_init_tools`.+ with tempfile.TemporaryDirectory(prefix="te-measure-", ignore_cleanup_errors=True) as cwd:out = subprocess.run(cmd, capture_output=True, encoding="utf-8", errors="replace", env=env,cwd=cwd, timeout=timeout)- data = json.loads(out.stdout or "{}")+ try:+ data = json.loads(out.stdout or "{}")+ except ValueError:+ data = {}u = data.get("usage") or {}- return (sum(int(u.get(k) or 0) for k in ("input_tokens", "cache_read_input_tokens",- "cache_creation_input_tokens")),- data.get("total_cost_usd"))+ total = sum(int(u.get(k) or 0) for k in ("input_tokens", "cache_read_input_tokens",+ "cache_creation_input_tokens"))+ if data.get("is_error") or not total:+ # A failed probe would otherwise count as 0 tokens, and the whole request as saved.+ raise RuntimeError("`claude -p` reported no usage: "+ + ((out.stderr or out.stdout or "").strip()[:300] or "no output"))+ return total, data.get("total_cost_usd")def _codex_prompt_chars(overrides, cli="codex", timeout=60):@@ -271,7 +280,13 @@ def measure(tools, cli=None, host="claude"):"probe_cost_usd": 0.0}cli = cli or claude_cli()full, c1 = _prompt_tokens([], cli=cli)- lean, c2 = _prompt_tokens(["--disallowed-tools", ",".join(tools)], cli=cli)+ # The deny list as a settings file, not `--disallowed-tools` on argv: hundreds of names+ # overflow cmd.exe's 8191-character line when `claude` is the npm `claude.cmd` shim. Bare-name+ # deny rules remove the same tools (see `denylist`), and are what `toolprune apply` writes.+ with tempfile.TemporaryDirectory(prefix="te-deny-", ignore_cleanup_errors=True) as d:+ settings = os.path.join(d, "settings.json")+ _write_json(settings, {"permissions": {"deny": list(tools)}})+ lean, c2 = _prompt_tokens(["--settings", settings], cli=cli)return {"tools": list(tools), "prompt_tokens_full": full, "prompt_tokens_lean": lean,"saved_per_request": full - lean,"probe_cost_usd": round((c1 or 0) + (c2 or 0), 4)}