Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

11.1 KB
Raw Blame
"""Fail-closed coverage for the whitelist gate's PR and push collection modes.

``.github/scripts/file-whitelist-check.sh`` collects the files to check with
``git diff -z`` (PR mode: ``origin/$GITHUB_BASE_REF..HEAD``; push mode:
``HEAD~1..HEAD``). Reading that output through ``while read ... done <
<(git diff ...)`` discards git's exit status, so an unresolvable range (a bad
or unfetched base ref, a depth-1 checkout with no ``HEAD~1``) collected zero
files and the gate passed having checked nothing. The script now captures
each listing to a temp file and fails closed when git fails.

Each case runs the real script in a throwaway repository; the normal-run
cases are the no-false-positive controls.
"""

# allow: no-sut-import — the subject is a CI shell script
# (.github/scripts/file-whitelist-check.sh), not importable Python; it is run
# as a subprocess so its own pipeline mechanics are what is tested.

from __future__ import annotations

import os
import subprocess
from pathlib import Path
from typing import Final

import pytest

REPO_ROOT: Final = Path(__file__).resolve().parents[2]
GATE_SCRIPT: Final = REPO_ROOT / ".github/scripts/file-whitelist-check.sh"

_IDENT: Final = {
    "GIT_AUTHOR_NAME": "t",
    "GIT_AUTHOR_EMAIL": "[email protected]",
    "GIT_COMMITTER_NAME": "t",
    "GIT_COMMITTER_EMAIL": "[email protected]",
}


def _git(repo: Path, *args: str) -> str:
    return subprocess.run(
        ["git", *args],
        cwd=repo,
        check=True,
        capture_output=True,
        text=True,
        env={**os.environ, **_IDENT},
    ).stdout.strip()


def _commit_file(repo: Path, name: str, content: str, message: str) -> str:
    """Stage ``name`` and commit it with git plumbing (no hooks, no config)."""
    (repo / name).write_text(content, encoding="utf-8")
    _git(repo, "add", "--", name)
    tree = _git(repo, "write-tree")
    parent = subprocess.run(
        ["git", "rev-parse", "-q", "--verify", "HEAD"],
        cwd=repo,
        capture_output=True,
        text=True,
        check=False,
    ).stdout.strip()
    parent_args = ["-p", parent] if parent else []
    commit = _git(repo, "commit-tree", tree, *parent_args, "-m", message)
    _git(repo, "update-ref", "HEAD", commit)
    return commit


def _write_whitelist(repo: Path) -> None:
    # Untracked on purpose: the gate reads it from the working tree, and
    # leaving it out of the commits keeps it out of the PR/push file lists.
    (repo / ".file-whitelist.txt").write_text(r"\.py$" + "\n", encoding="utf-8")


def _two_commit_repo(tmp_path: Path) -> Path:
    """Repo with a base commit (mirrored as origin/main) and one PR commit."""
    repo = tmp_path / "repo"
    repo.mkdir()
    _git(repo, "init", "-q")
    _commit_file(repo, "base.py", "a = 1\n", "base")
    _git(repo, "update-ref", "refs/remotes/origin/main", "HEAD")
    _commit_file(repo, "added.py", "b = 2\n", "change")
    _write_whitelist(repo)
    return repo


def _run_gate(
    repo: Path, event: str, base_ref: str | None = None
) -> subprocess.CompletedProcess[str]:
    env = {k: v for k, v in os.environ.items() if k != "CHECK_ALL_FILES"}
    env.pop("GITHUB_BASE_REF", None)
    env["GITHUB_EVENT_NAME"] = event
    if base_ref is not None:
        env["GITHUB_BASE_REF"] = base_ref
    return subprocess.run(
        ["bash", str(GATE_SCRIPT)],
        cwd=repo,
        env=env,
        capture_output=True,
        text=True,
        check=False,
    )


def test_pr_mode_normal_run_passes(tmp_path: Path) -> None:
    repo = _two_commit_repo(tmp_path)
    result = _run_gate(repo, "pull_request", "main")
    assert result.returncode == 0, (
        f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
    )
    assert "Checked 1 files" in result.stdout, result.stdout


def test_pr_mode_unresolvable_base_ref_fails_closed(tmp_path: Path) -> None:
    """Revert caught: reading ``git diff`` via ``< <(...)`` again (rc=0 -> red)."""
    repo = _two_commit_repo(tmp_path)
    result = _run_gate(repo, "pull_request", "no-such-base-branch")
    assert result.returncode != 0, (
        "PR mode passed with an unresolvable base ref (0 files collected).\n"
        f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
    )
    # Step-specific: the files-changed listing is what must fail, so a later
    # step failing closed cannot mask a revert of this one.
    assert "Could not list files changed in the PR" in result.stdout, (
        result.stdout
    )


def test_pr_mode_unset_base_ref_fails_closed(tmp_path: Path) -> None:
    repo = _two_commit_repo(tmp_path)
    result = _run_gate(repo, "pull_request", "")
    assert result.returncode != 0, (
        f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
    )
    assert "GITHUB_BASE_REF is not set" in result.stdout, result.stdout


def test_push_mode_normal_run_passes(tmp_path: Path) -> None:
    repo = _two_commit_repo(tmp_path)
    result = _run_gate(repo, "push")
    assert result.returncode == 0, (
        f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
    )
    assert "Found 1 files in direct push" in result.stdout, result.stdout


def test_push_mode_depth_one_checkout_fails_closed(tmp_path: Path) -> None:
    """A depth-1 checkout has no ``HEAD~1``; the gate must not pass on 0 files.

    Revert caught: reading ``git diff HEAD~1..HEAD`` via ``< <(...)`` again
    (rc=0 -> red).
    """
    origin = _two_commit_repo(tmp_path)
    shallow = tmp_path / "shallow"
    subprocess.run(
        [
            "git",
            "clone",
            "-q",
            "--depth",
            "1",
            origin.resolve().as_uri(),
            str(shallow),
        ],
        check=True,
        capture_output=True,
        text=True,
    )
    probe = subprocess.run(
        ["git", "rev-parse", "-q", "--verify", "HEAD~1"],
        cwd=shallow,
        capture_output=True,
        text=True,
        check=False,
    )
    assert probe.returncode != 0, "fixture is not shallow: HEAD~1 resolves"
    _write_whitelist(shallow)

    result = _run_gate(shallow, "push")
    assert result.returncode != 0, (
        "push mode passed on a depth-1 checkout (0 files collected).\n"
        f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
    )
    assert "Could not list files in the latest commit" in result.stdout, (
        result.stdout
    )


def _pr_repo_with_change(tmp_path: Path, change: str) -> Path:
    """Base commit with ``tool.py`` (mirrored as origin/main), then ``change``.

    ``rename``: ``git mv tool.py tool.sqlite`` (a non-whitelisted name).
    ``typechange``: ``tool.py`` becomes a symlink to a missing path.
    """
    repo = tmp_path / "repo"
    repo.mkdir()
    _git(repo, "init", "-q")
    _commit_file(repo, "tool.py", "x = 1\n", "base")
    _git(repo, "update-ref", "refs/remotes/origin/main", "HEAD")
    if change == "rename":
        _git(repo, "mv", "tool.py", "tool.sqlite")
    else:
        (repo / "tool.py").unlink()
        os.symlink("missing-target.py", repo / "tool.py")
        _git(repo, "add", "--", "tool.py")
    tree = _git(repo, "write-tree")
    commit = _git(repo, "commit-tree", tree, "-p", "HEAD", "-m", change)
    _git(repo, "update-ref", "HEAD", commit)
    _write_whitelist(repo)
    return repo


@pytest.mark.parametrize("event", ["pull_request", "push"])
@pytest.mark.parametrize("change", ["rename", "typechange"])
def test_rename_and_typechange_are_checked(
    tmp_path: Path, event: str, change: str
) -> None:
    """Renames and type changes must reach the whitelist loop.

    With rename detection a ``git mv`` is an R entry and a file->symlink
    change is a T entry; an ``A``/``AM`` filter drops both, so the
    renamed-to ``tool.sqlite`` (not whitelisted) and the dangling symlink
    were never checked. Here the change is an ordinary PR commit, so in PR
    mode BOTH listings (the range ``git diff`` and the per-commit ``git
    log``) see it: reverting only one of them stays green in PR mode.
    Revert caught: push mode pins the ``HEAD~1..HEAD`` diff; PR mode pins
    reverting both PR listings together. The PR-mode ``git diff`` on its own
    is pinned by ``test_merge_resolution_change_checked_by_pr_diff``.
    """
    repo = _pr_repo_with_change(tmp_path, change)
    result = _run_gate(repo, event, "main" if event == "pull_request" else None)
    assert result.returncode != 0, (
        f"{change} passed in {event} mode.\n"
        f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
    )
    expected = "tool.sqlite" if change == "rename" else "NON-REGULAR"
    assert expected in result.stdout, result.stdout


def _pr_repo_with_merge_resolution_change(tmp_path: Path, change: str) -> Path:
    """PR whose only rename/type change is made inside a merge resolution.

    History: base (tool.py, = origin/main) -> side (adds other.py) and
    pr (adds pr.py); HEAD merges side into pr, and the merge tree ALSO
    renames tool.py to tool.sqlite (``rename``) or turns tool.py into a
    dangling symlink (``typechange``). ``git log`` without ``-m`` lists no
    files for a merge commit, so only the range ``git diff`` sees the change.
    """
    repo = tmp_path / "repo"
    repo.mkdir()
    _git(repo, "init", "-q")
    base = _commit_file(repo, "tool.py", "x = 1\n", "base")
    _git(repo, "update-ref", "refs/remotes/origin/main", base)
    side = _commit_file(repo, "other.py", "y = 1\n", "side")
    # Branch "pr" from base: reset index and worktree to base, add pr.py.
    _git(repo, "read-tree", base)
    (repo / "other.py").unlink()
    (repo / "pr.py").write_text("z = 1\n", encoding="utf-8")
    _git(repo, "add", "--", "pr.py")
    pr = _git(
        repo, "commit-tree", _git(repo, "write-tree"), "-p", base, "-m", "pr"
    )
    # Merge resolution: pr's tree + side's other.py + the hidden change.
    (repo / "other.py").write_text("y = 1\n", encoding="utf-8")
    _git(repo, "add", "--", "other.py")
    if change == "rename":
        _git(repo, "mv", "tool.py", "tool.sqlite")
    else:
        (repo / "tool.py").unlink()
        os.symlink("missing-target.py", repo / "tool.py")
        _git(repo, "add", "--", "tool.py")
    merge = _git(
        repo,
        "commit-tree",
        _git(repo, "write-tree"),
        "-p",
        pr,
        "-p",
        side,
        "-m",
        "merge",
    )
    _git(repo, "update-ref", "HEAD", merge)
    _write_whitelist(repo)
    return repo


@pytest.mark.parametrize("change", ["rename", "typechange"])
def test_merge_resolution_change_checked_by_pr_diff(
    tmp_path: Path, change: str
) -> None:
    """A rename/type change made only in a merge resolution must be checked.

    The per-commit ``git log`` listing cannot see it (no ``-m``), so this
    pins the PR-mode ``git diff --no-renames --diff-filter=AMT`` listing on
    its own. Revert caught: restoring ``--diff-filter=AM`` with rename
    detection on that diff lets the PR pass (rc=0 -> red).
    """
    repo = _pr_repo_with_merge_resolution_change(tmp_path, change)
    result = _run_gate(repo, "pull_request", "main")
    assert result.returncode != 0, (
        f"{change} made in a merge resolution passed.\n"
        f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
    )
    expected = "tool.sqlite" if change == "rename" else "NON-REGULAR"
    assert expected in result.stdout, result.stdout