Skip to content

Experience-Driven Black-Box Cost Control for Coding Agents

feat(toolprune): protect tools by name or pattern

Sathvik C5201ca3d9b18
diff --git a/plugin/README.md b/plugin/README.md
index fa1519f..9e86e60 100644
--- a/plugin/README.md
+++ b/plugin/README.md
@@ -425,6 +425,7 @@ and are shared by every agent working in it. The examples use Claude Code's form
| `toolprune_min_sessions` | `2` | sessions those prompts must span; `1` lets a single session's prompts decide |
| `toolprune_sessions` | `5` | or this many sessions, however few prompts they ran |
| `toolprune_quiet_minutes` | `60` | how long the repository's sessions must have been idle before `auto` writes a deny list, so that no conversation resumed while its prompt cache is warm loses it (a list is written at the latest a day after it formed); `0` writes at once |
+| `toolprune_protect` | none | tools never left out however long they go unused, by name or pattern (`mcp__headroom__*`), separated by commas; a deny rule the plugin added for one is removed at the next fold, and rules you wrote are never touched |
| `loopguard_aggr` | `default` | the guard's threshold preset: `conservative`, `default` or `aggressive` |
| `loopguard_nudges` | `1` | redirects per turn before the guard only tells you |
| `loopguard.<name>` | from the preset | one guard threshold, set by hand (below) |
diff --git a/tests/test_live_hooks.py b/tests/test_live_hooks.py
index d24c97c..37a0234 100644
--- a/tests/test_live_hooks.py
+++ b/tests/test_live_hooks.py
@@ -641,6 +641,27 @@ def test_a_high_risk_tool_needs_three_times_the_prompts(env, repo, tmp_path):
"NotebookEdit", "Workflow"]
+def test_a_protected_tool_is_never_left_out_and_a_rule_already_written_comes_back(
+ env, repo, tmp_path):
+ """`toolprune_protect` takes names and patterns. A deny rule the plugin wrote before the tool
+ was protected is removed at the next fold; the person's own rules stay."""
+ _catalog(env, ["Bash", "mcp__headroom__headroom_retrieve", "Workflow"])
+ path = denylist.settings_path(repo, "claude")
+ os.makedirs(os.path.dirname(path))
+ json.dump({"permissions": {"deny": ["Mine"]}}, open(path, "w"))
+ for i in range(5):
+ _prompts(env, repo, tmp_path, f"a{i}", 3)
+ deny = json.load(open(path))["permissions"]["deny"]
+ assert "mcp__headroom__headroom_retrieve" in deny and "Workflow" in deny
+ env = dict(env, TE_TOOLPRUNE_PROTECT="mcp__headroom__*, Nothing")
+ _prompts(env, repo, tmp_path, "b", 3)
+ deny = json.load(open(path))["permissions"]["deny"]
+ assert "mcp__headroom__headroom_retrieve" not in deny
+ assert "Workflow" in deny and "Mine" in deny
+ assert "mcp__headroom__headroom_retrieve" not in store_for(env, repo).read(
+ "untool.json")["proposal"]
+
+
def _pending_repo(env, repo, tmp_path, quiet="60"):
"""Five sessions that leave three tools unused, under the default wait before writing."""
env = dict(env, TE_TOOLPRUNE_QUIET_MINUTES=quiet)
diff --git a/tokeneconomizer/plugin/adapter.py b/tokeneconomizer/plugin/adapter.py
index 9c49ef9..b75b502 100644
--- a/tokeneconomizer/plugin/adapter.py
+++ b/tokeneconomizer/plugin/adapter.py
@@ -8,7 +8,8 @@ answers fit an interactive session instead of a sealed benchmark task:
keeps Bash/Read/Edit/...; `toolprune.tools.INAPPLICABLE_TOOLS` then treats AskUserQuestion, plan
mode and subagents as inapplicable "by construction" for a headless sealed task. A person
does use them interactively, so those are protected here, and so are the web tools, however
- long they go unused; only evidence (loaded but unused for N sessions) can propose anything
+ long they go unused, and whatever the person lists in `toolprune_protect` (names, or
+ patterns such as `mcp__headroom__*`); only evidence (loaded but unused for N sessions) can propose anything
else.
* `parse_catalog_with_source` reports what the session COULD call. An interactive transcript
does not record the tool list the way `stream-json`'s init event does, so the catalog is the
@@ -23,6 +24,7 @@ answers fit an interactive session instead of a sealed benchmark task:
* `problem_statements` is empty: there is no issue text to give the per-task tool judge.
"""
import dataclasses
+import fnmatch
import json
import os
@@ -58,6 +60,19 @@ COPILOT_PROTECTED = frozenset({
HIGH_RISK = frozenset({"NotebookEdit"})
HIGH_RISK_FACTOR = 3
+
+class _Protected(frozenset):
+ """The protected names, plus patterns (`mcp__headroom__*`) a name may match instead."""
+
+ def __new__(cls, names, patterns=()):
+ obj = super().__new__(cls, names)
+ obj.patterns = tuple(patterns)
+ return obj
+
+ def __contains__(self, name):
+ return super().__contains__(name) or any(fnmatch.fnmatchcase(str(name), p)
+ for p in self.patterns)
+
_PY_MARKERS = ("pyproject.toml", "setup.py", "setup.cfg", "requirements.txt", "Pipfile",
"tox.ini", "environment.yml")
_NODE_MARKERS = ("package.json",)
@@ -91,11 +106,12 @@ def load_catalog(state_root, host):
class LiveAdapter:
name = "live"
- def __init__(self, host, repo_root, vocabulary="auto", catalog=None):
+ def __init__(self, host, repo_root, vocabulary="auto", catalog=None, protect=()):
self.host = host
self.repo_root = repo_root
self.vocabulary = vocabulary
self.catalog = dict(catalog or {})
+ self.protect = tuple(protect)
# ---- trajectory reading ----
def load_trajectory(self, path):
@@ -127,7 +143,7 @@ class LiveAdapter:
# ---- tool pruning ----
def protected_tools(self):
extra = COPILOT_PROTECTED if self.host.name == "copilot" else frozenset()
- return PROTECTED_TOOLS | INTERACTIVE_PROTECTED | extra
+ return _Protected(PROTECTED_TOOLS | INTERACTIVE_PROTECTED | extra, self.protect)
def inapplicable_tools(self):
return () # nothing is inapplicable by construction here
diff --git a/tokeneconomizer/plugin/cli.py b/tokeneconomizer/plugin/cli.py
index 20df253..18587ef 100644
--- a/tokeneconomizer/plugin/cli.py
+++ b/tokeneconomizer/plugin/cli.py
@@ -440,7 +440,8 @@ def _config_view(cfg):
"toolprune_evidence": d.pop("toolprune_evidence"),
"toolprune_min_sessions": d.pop("toolprune_min_sessions"),
"toolprune_sessions": d.pop("toolprune_sessions"),
- "toolprune_quiet_minutes": d.pop("toolprune_quiet_minutes")},
+ "toolprune_quiet_minutes": d.pop("toolprune_quiet_minutes"),
+ "toolprune_protect": d.pop("toolprune_protect")},
"loopguard": {
"state": "off" if cfg.loopguard == "off" else "on",
"loopguard": d.pop("loopguard"), "loopguard_aggr": d.pop("loopguard_aggr"),
diff --git a/tokeneconomizer/plugin/config.py b/tokeneconomizer/plugin/config.py
index acb3246..877a1e8 100644
--- a/tokeneconomizer/plugin/config.py
+++ b/tokeneconomizer/plugin/config.py
@@ -35,6 +35,9 @@ person can tune by hand:
* toolprune_evidence -- prompts a never-used tool must have stayed unused through,
over at least toolprune_min_sessions sessions, before it is
proposed -- or over toolprune_sessions sessions, however short;
+ * toolprune_protect -- tools never proposed, by name or pattern (`mcp__headroom__*`),
+ comma-separated; a deny rule the plugin added for one is
+ removed at the next fold;
* toolprune_quiet_minutes -- how long every session of the repository must have been idle
before `auto` writes a deny list (0: at once), so that no
conversation resumed with a warm prompt cache loses it;
@@ -147,6 +150,7 @@ class LiveConfig:
toolprune_min_sessions: int = 2
toolprune_sessions: int = 5
toolprune_quiet_minutes: int = 60
+ toolprune_protect: list = field(default_factory=list)
loopguard: str = "nudge"
loopguard_aggr: str = "default"
loopguard_nudges: int = 1
@@ -198,7 +202,10 @@ class LiveConfig:
value = _VALUE_ALIASES.get(key, {}).get(word, value)
cur = getattr(self, key)
try:
- if isinstance(cur, bool):
+ if isinstance(cur, list):
+ items = value if isinstance(value, list) else str(value).split(",")
+ v = [str(x).strip() for x in items if str(x).strip()]
+ elif isinstance(cur, bool):
v = value if isinstance(value, bool) else str(value).strip().lower() in (
"1", "true", "yes", "on")
elif isinstance(cur, int):
@@ -256,6 +263,7 @@ _ENV = {"TE_MODE": "mode", "TE_SHADOW": "shadow",
"TE_TOOLPRUNE_QUIET_MINUTES": "toolprune_quiet_minutes",
"TE_TOOLPRUNE_MIN_SESSIONS": "toolprune_min_sessions",
"TE_TOOLPRUNE_SESSIONS": "toolprune_sessions",
+ "TE_TOOLPRUNE_PROTECT": "toolprune_protect",
"TE_LOOPGUARD": "loopguard", "TE_LOOPGUARD_AGGR": "loopguard_aggr",
"TE_LOOPGUARD_NUDGES": "loopguard_nudges",
"TE_REPOMAP": "repomap", "TE_REPOMAP_TOKENS": "repomap_tokens",
diff --git a/tokeneconomizer/plugin/fold.py b/tokeneconomizer/plugin/fold.py
index 2ba220c..1cb4785 100644
--- a/tokeneconomizer/plugin/fold.py
+++ b/tokeneconomizer/plugin/fold.py
@@ -383,7 +383,8 @@ def fold_session(ctx, session_id, transcript_path=None):
else:
run_dir = _run_dir(workdir, trace_path)
adapter = LiveAdapter(ctx.host, ctx.repo.root, vocabulary=ctx.config.vocabulary,
- catalog=load_catalog(ctx.root, ctx.host.name))
+ catalog=load_catalog(ctx.root, ctx.host.name),
+ protect=ctx.config.toolprune_protect)
with ctx.store.locked("mechanisms"):
inj = _fold_microskills(ctx, adapter, run_dir, session_id,
injected=sess.get("injected_ids"))