Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
"""Unit tests for the ``repowise distill`` / ``repowise expand`` commands."""
from __future__ import annotations
import json
import os
import shlex
import shutil
import subprocess
import sys
from pathlib import Path
import pytest
from click.testing import CliRunner
from repowise.cli.commands.distill_cmd import (
_posix_shell,
_render_command,
distill_command,
)
from repowise.cli.commands.expand_cmd import expand_command
from repowise.core.distill.markers import parse_marker_refs
from repowise.core.distill.store import OmissionStore
@pytest.fixture()
def repo_cwd(tmp_path: Path, monkeypatch) -> Path:
"""A scratch repo with .repowise/ so the store lands locally."""
(tmp_path / ".repowise").mkdir()
monkeypatch.chdir(tmp_path)
return tmp_path
def _py(code: str) -> list[str]:
return [sys.executable, "-c", code]
def test_distill_preserves_exit_code(repo_cwd: Path) -> None:
runner = CliRunner()
result = runner.invoke(distill_command, _py("import sys; sys.exit(7)"))
assert result.exit_code == 7
def test_distill_unmatched_command_passes_output_through(repo_cwd: Path) -> None:
runner = CliRunner()
result = runner.invoke(distill_command, _py("print('plain output')"))
assert result.exit_code == 0
assert "plain output" in result.output
assert "[repowise#" not in result.output
def test_distill_captures_stderr_too(repo_cwd: Path) -> None:
runner = CliRunner()
result = runner.invoke(
distill_command,
_py("import sys; print('out'); print('err line', file=sys.stderr)"),
)
assert "out" in result.output
assert "err line" in result.output
def test_distill_failing_git_command_keeps_exit_code(repo_cwd: Path) -> None:
# tmp dir is not a git repository: git fails, the engine passes the raw
# error output through, and the nonzero exit code survives.
result = CliRunner().invoke(distill_command, ["git", "log", "-40"])
assert result.exit_code != 0
@pytest.mark.skipif(os.name != "posix", reason="cmd.exe has no grep")
def test_distill_runs_a_pipeline_passed_as_one_token(repo_cwd: Path) -> None:
"""One argv token containing a pipe must execute AS a pipeline.
This is the execution model the rewrite hook's safe-pipeline shape
depends on: the hook quotes the whole pipeline so the pipe binds inside
distill's own shell. The producer prints a line only grep can remove, so
the assertion fails if the pipe silently did not run.
"""
script = (
"for i in range(40): "
"print('Requirement already satisfied: pkg%d in /venv/site-packages' % i)\n"
"print('ZZZDROPME')\n"
"print('Successfully installed flask-3.1.0')"
)
pipeline = f"{shlex.quote(sys.executable)} -c {shlex.quote(script)} | grep -v ZZZDROPME"
result = CliRunner().invoke(distill_command, [pipeline])
assert result.exit_code == 0
# Proof the pipe ran: only grep could have dropped this line.
assert "ZZZDROPME" not in result.output
# Proof distill still distilled what came back out of it.
assert "Successfully installed flask-3.1.0" in result.output
refs = parse_marker_refs(result.output)
assert refs, f"no recoverable marker in:\n{result.output}"
assert CliRunner().invoke(expand_command, [refs[0]]).exit_code == 0
def test_render_command_quotes_shell_metacharacters() -> None:
"""argv tokens must not turn into shell syntax when distill rejoins them.
The rewrite hook leans on this for every non-pipe command it forwards as
separate tokens: ``pytest -k "a|b"`` must run as one command, not a
pipeline. POSIX ``shlex.join`` gets it right. Windows
``list2cmdline`` quotes for the C runtime's argv parser rather than for
cmd.exe, so a metacharacter with no surrounding space rides through
unquoted; ``_render_command`` caret-escapes the rendered line to close
that.
"""
rendered = _render_command(("git", "log", "--grep=a&&b"))
if os.name == "posix":
assert rendered == "git log '--grep=a&&b'"
else:
assert rendered == "git log --grep=a^&^&b"
def test_render_command_passes_a_single_token_through() -> None:
"""One token means the user quoted the command themselves; that is intent."""
assert _render_command(("pytest -x | head -5",)) == "pytest -x | head -5"
@pytest.mark.parametrize(
"payload",
[
"--grep=a&whoami",
"--grep=a|whoami",
"--grep=a>rendered_should_not_write.txt",
"--grep=a;b",
"plain",
"has space",
'say "hi" now',
'a"b&c',
"a^b",
"a(b)c",
"trailing\\back\\slash\\",
"--format=%H",
"--format=%h%n%s",
"a!USERNAME!b",
"café中文",
"",
"^",
"a\tb",
],
)
def test_render_command_roundtrips_argv_through_the_shell(payload: str, tmp_path: Path) -> None:
"""The rendered string must hand the child exactly the token we started with.
This is the real contract: ``_render_command`` output goes straight to
``subprocess.run(..., shell=True)``, so anything the shell reinterprets
is a command the user never typed. Asserting on the rendered string
alone did not catch the cmd.exe metacharacter hole, so this one runs the
render and reads the child's ``sys.argv`` back.
"""
tokens = (sys.executable, "-c", "import sys,json;print(json.dumps(sys.argv[1:]))", payload)
proc = subprocess.run( # repowise-security-ignore: subprocess_shell_true
_render_command(tokens),
shell=True,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
cwd=tmp_path,
timeout=60,
)
assert proc.returncode == 0, proc.stderr
assert json.loads(proc.stdout.strip().splitlines()[-1]) == [payload]
# A redirect that reached the shell would have written a file instead.
assert list(tmp_path.iterdir()) == []
@pytest.mark.skipif(sys.platform != "win32", reason="cmd.exe program-name resolution")
def test_render_command_keeps_an_executable_path_with_spaces_intact(tmp_path: Path) -> None:
"""The program name needs real grouping quotes, not caret-escaped ones.
cmd.exe resolves the executable before caret processing, so escaping the
quotes around ``C:\\Program Files\\...`` splits the path at its first
space and the command does not run at all. This is the ``.cmd``-shim case
that ``shell=True`` exists to support.
"""
shim_dir = tmp_path / "with space"
shim_dir.mkdir()
shim = shim_dir / "my shim.cmd"
# Echoes a fixed marker rather than %*: a batch file re-parses whatever %*
# expands to, which is the shim's own hazard and would not be testing the
# renderer. Argument fidelity is covered by the parametrized test above.
shim.write_text("@echo SHIM_OK\n", encoding="utf-8")
proc = subprocess.run( # repowise-security-ignore: subprocess_shell_true
_render_command((str(shim), "arg one", "a&b")),
shell=True,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
cwd=tmp_path,
timeout=60,
)
assert proc.returncode == 0, proc.stderr or proc.stdout
assert "SHIM_OK" in proc.stdout
@pytest.mark.skipif(sys.platform != "win32", reason="cmd.exe program-name resolution")
def test_render_command_runs_a_forward_slash_program_path(tmp_path: Path) -> None:
"""A Git Bash spelling like ``./bin/tool.cmd`` must reach cmd as one program.
Unquoted, cmd reads the leading ``.`` as the program and ``/bin`` as a
switch, so a rewritten ``./.venv/Scripts/python.exe -m pytest`` never ran.
"""
(tmp_path / "bin").mkdir()
(tmp_path / "bin" / "tool.cmd").write_text("@echo TOOL_OK\n", encoding="utf-8")
proc = subprocess.run( # repowise-security-ignore: subprocess_shell_true
_render_command(("./bin/tool.cmd", "arg")),
shell=True,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
cwd=tmp_path,
timeout=60,
)
assert proc.returncode == 0, proc.stderr or proc.stdout
assert "TOOL_OK" in proc.stdout
@pytest.mark.skipif(sys.platform != "win32", reason="cmd.exe rendering rules")
def test_render_command_refuses_a_defined_env_var_expansion(monkeypatch) -> None:
"""cmd expands %VAR% and re-parses the result, so a metacharacter in the
value is command execution. There is no escape for it, so refuse."""
monkeypatch.setenv("REPOWISE_TEST_EVIL", "x&echo pwned")
with pytest.raises(Exception, match="REPOWISE_TEST_EVIL"):
_render_command(("git", "log", "--grep=%REPOWISE_TEST_EVIL%"))
@pytest.mark.skipif(sys.platform != "win32", reason="cmd.exe rendering rules")
def test_render_command_allows_an_undefined_percent_pair(monkeypatch) -> None:
"""`git log --format=%h%n%s` reads as %h% to cmd but expands to nothing."""
monkeypatch.delenv("h", raising=False)
monkeypatch.delenv("n", raising=False)
assert "%h%n%s" in _render_command(("git", "log", "--format=%h%n%s"))
@pytest.mark.skipif(sys.platform != "win32", reason="cmd.exe rendering rules")
@pytest.mark.parametrize("payload", ["a\nb", "a\r\nb", "a\rb"])
def test_render_command_refuses_newlines(payload: str) -> None:
"""cmd truncates the command line at a newline and drops a bare CR, both
silently, so the child would get a quietly different argv."""
with pytest.raises(Exception, match="newline"):
_render_command(("git", "log", f"--grep={payload}"))
# ---------------------------------------------------------------------------
# --source also names a dialect: the command runs in the shell it was
# written for
#
# The rewrite hook declined every compound command on a Windows host because
# distill handed everything to cmd.exe, where a POSIX command line means
# something else. These pin the replacement: the dialect travels with the
# command, and when the shell it names cannot be found distill refuses rather
# than running a different command than the one it was given.
# ---------------------------------------------------------------------------
@pytest.mark.skipif(sys.platform != "win32", reason="cmd.exe is only the default on Windows")
@pytest.mark.parametrize(
"shell_value",
[
pytest.param(None, id="unset"),
pytest.param("powershell.exe", id="powershell-also-takes--c"),
pytest.param("system32-bash", id="wsl-is-a-posix-shell-and-still-wrong"),
],
)
def test_only_a_real_posix_shell_is_accepted(monkeypatch, shell_value) -> None:
"""Breaking what the guard protects, one substitute at a time.
``-c`` is how ``powershell.exe`` takes a command too, so an
existence check would hand a POSIX pipeline to PowerShell and get a
plausible answer to a different question. WSL's ``System32\bash.exe``
passes a name test and is still another machine.
"""
if shell_value is None:
monkeypatch.delenv("SHELL", raising=False)
elif shell_value == "system32-bash":
root = os.environ.get("SYSTEMROOT", r"C:\Windows")
wsl = os.path.join(root, "System32", "bash.exe")
if not os.path.isfile(wsl):
pytest.skip("WSL bash.exe not installed on this host")
monkeypatch.setenv("SHELL", wsl)
else:
found = shutil.which(shell_value)
if found is None:
pytest.skip(f"{shell_value} not on this host")
monkeypatch.setenv("SHELL", found)
assert _posix_shell() is None
@pytest.mark.skipif(sys.platform != "win32", reason="cmd.exe is only the default on Windows")
def test_a_plain_command_never_needs_a_posix_shell(repo_cwd: Path, monkeypatch) -> None:
"""The refusal must not reach commands that were always fine in cmd.exe.
A multi-token argv is the same command in either dialect. Only the
single-token shape the hook wraps a chain in carries POSIX syntax of its
own, so only that shape may ever refuse to run.
"""
monkeypatch.delenv("SHELL", raising=False)
result = CliRunner().invoke(
distill_command, ["--source", "hook-bash", *_py("print('still runs')")]
)
assert result.exit_code == 0, result.output
assert "still runs" in result.output
@pytest.mark.skipif(sys.platform != "win32", reason="cmd.exe is only the default on Windows")
def test_posix_dialect_runs_in_a_posix_shell(repo_cwd: Path) -> None:
"""``echo "a b" | tr " " "_"`` is the cheapest proof the shell changed.
cmd.exe keeps the quotes literal and answers ``"a_b"_``; bash answers
``a_b``. Both exit 0, so a wrong shell here is silent corruption rather
than an error — which is why the hook used to decline the command
outright.
"""
if _posix_shell() is None:
pytest.skip("no POSIX shell on this host")
command = ['echo "a b" | tr " " "_"']
posix = CliRunner().invoke(distill_command, ["--source", "hook-bash", *command])
assert posix.exit_code == 0, posix.output
assert posix.output.strip() == "a_b"
host = CliRunner().invoke(distill_command, command)
assert host.output.strip() != "a_b"
@pytest.mark.skipif(sys.platform != "win32", reason="cmd.exe is only the default on Windows")
def test_posix_dialect_refuses_rather_than_falling_back_to_cmd(
repo_cwd: Path, monkeypatch
) -> None:
"""Breaking what the guard protects: with no POSIX shell it must not run.
Falling back to the host shell is the exact bug this path exists to
close, and it fails silently, so the refusal is the safer half of the
trade. A command that did not run is recoverable; a command that ran and
meant something else is not.
"""
monkeypatch.delenv("SHELL", raising=False)
result = CliRunner().invoke(distill_command, ["--source", "hook-bash", "echo ran-anyway"])
assert result.exit_code != 0
assert "ran-anyway" not in result.output
assert "POSIX shell" in result.output
def test_no_shell_flag_keeps_the_host_default(repo_cwd: Path) -> None:
"""Plain ``repowise distill ...`` from a terminal is untouched by any of
this: no dialect named means the shell this host would have used."""
result = CliRunner().invoke(distill_command, _py("print('host default')"))
assert result.exit_code == 0
assert "host default" in result.output
def test_distill_and_expand_roundtrip(repo_cwd: Path, fixtures_dir: Path) -> None:
raw = (fixtures_dir / "distill" / "git_log_full.txt").read_text(encoding="utf-8-sig")
from repowise.core.distill import distill_output
store = OmissionStore(repo_cwd / ".repowise" / "omissions" / "omissions.db")
distilled = distill_output(raw, command="git log -40", store=store)
store.close()
assert distilled.distilled
(ref,) = parse_marker_refs(distilled.text)
result = CliRunner().invoke(expand_command, [ref])
assert result.exit_code == 0
assert result.output.rstrip("\n") == raw.rstrip("\n")
def test_expand_accepts_pasted_marker(repo_cwd: Path) -> None:
store = OmissionStore(repo_cwd / ".repowise" / "omissions" / "omissions.db")
ref = store.put("stashed content", source="cli:logs", original_tokens=10, kept_tokens=2)
store.close()
marker = f"[repowise#{ref}: 3 lines omitted (~30 tokens); restore: repowise expand {ref}]"
result = CliRunner().invoke(expand_command, [marker])
assert result.exit_code == 0
assert "stashed content" in result.output
def test_expand_with_query_filters_lines(repo_cwd: Path) -> None:
store = OmissionStore(repo_cwd / ".repowise" / "omissions" / "omissions.db")
ref = store.put(
"keep FAILED a\ndrop ok b\nkeep FAILED c", source="cli:t", original_tokens=9, kept_tokens=1
)
store.close()
result = CliRunner().invoke(expand_command, [ref, "--query", "FAILED"])
assert result.exit_code == 0
assert "drop ok b" not in result.output
assert "keep FAILED a" in result.output
def test_expand_unknown_ref_fails_cleanly(repo_cwd: Path) -> None:
result = CliRunner().invoke(expand_command, ["0" * 12])
assert result.exit_code == 1
def test_expand_invalid_ref_fails_cleanly(repo_cwd: Path) -> None:
result = CliRunner().invoke(expand_command, ["zzz"])
assert result.exit_code == 2
def test_distill_source_flag_tags_the_ledger(repo_cwd: Path) -> None:
"""The hook's --source tag flows into the ledger for `saved --by source`."""
runner = CliRunner()
noisy = "import sys; sys.stdout.write('x.py:1:1: E501 line too long\\n' * 80)"
result = runner.invoke(distill_command, ["--source", "hook-powershell", *_py(noisy)])
assert result.exit_code == 0
store = OmissionStore(repo_cwd / ".repowise" / "omissions" / "omissions.db")
rows = store.savings_rollup(by="source")
store.close()
assert [r["group"] for r in rows] == ["hook-powershell"]
def test_distill_expand_roundtrip_from_codex_surface(repo_cwd: Path) -> None:
"""The omission store and `expand` are agent-agnostic — a marker produced
under the Codex hook's source tag restores exactly like any other."""
runner = CliRunner()
noisy = "import sys; sys.stdout.write('x.py:1:1: E501 line too long\\n' * 80)"
result = runner.invoke(distill_command, ["--source", "hook-codex", *_py(noisy)])
assert result.exit_code == 0
(ref,) = parse_marker_refs(result.output)
expanded = CliRunner().invoke(expand_command, [ref])
assert expanded.exit_code == 0
assert expanded.output.count("E501 line too long") == 80
store = OmissionStore(repo_cwd / ".repowise" / "omissions" / "omissions.db")
rows = store.savings_rollup(by="source")
store.close()
assert [r["group"] for r in rows] == ["hook-codex"]
def test_distill_does_not_eat_the_wrapped_commands_source_flag(repo_cwd: Path) -> None:
"""--source after the command belongs to the command, not to distill."""
result = CliRunner().invoke(
distill_command,
[*_py("import sys; print(sys.argv[1:])"), "--source", "mine"],
)
assert result.exit_code == 0
assert "['--source', 'mine']" in result.output
def test_distill_records_savings_ledger(repo_cwd: Path, fixtures_dir: Path) -> None:
fixture = fixtures_dir / "distill" / "find_paths.txt"
raw = fixture.read_text(encoding="utf-8-sig")
from repowise.core.distill import distill_output
store = OmissionStore(repo_cwd / ".repowise" / "omissions" / "omissions.db")
result = distill_output(raw, command="find packages -name *.py", source="cli", store=store)
assert result.distilled
summary = store.savings_summary()
store.close()
assert summary["per_filter"]["file_listing"]["events"] == 1