Skip to content

πŸš€ The fast, Pythonic way to build MCP servers and clients.

16.0 KB
Raw Blame
"""Public analytics preserve uncertainty and do not manufacture review verdicts."""

import importlib.util
import json
import subprocess
import sys
from datetime import UTC, datetime
from pathlib import Path
from types import SimpleNamespace

import pytest

NOW = datetime(2026, 11, 3, 18, tzinfo=UTC)


@pytest.fixture
def triage():
    spec = importlib.util.spec_from_file_location(
        "public_triage", Path(__file__).parents[1] / "scripts/public_triage.py"
    )
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


def item(number=1, **overrides):
    return {
        "number": number,
        "title": "Fix supported behavior",
        "created_at": "2026-10-01T00:00:00Z",
        "state": "open",
        "pull_request": {},
        "draft": False,
        "author_association": "NONE",
        "user": {"login": "contributor", "type": "User"},
        **overrides,
    }


def pr(**overrides):
    return {
        **item(),
        "head": {"sha": "a" * 40},
        "base": {"sha": "b" * 40, "ref": "main"},
        "updated_at": "2026-11-03T00:00:00Z",
        **overrides,
    }


def activity(login, when, *, bot=False, association="MEMBER", **fields):
    return {
        "user": {"login": login, "type": "Bot" if bot else "User"},
        "author_association": association,
        "created_at": when,
        **fields,
    }


def test_discussion_after_first_page_and_historical_review(triage, monkeypatch):
    # Paginated REST pages include a maintainer after 100 earlier comments.
    comments = [activity("contributor", "2026-10-02T00:00:00Z")] * 100
    last = activity("maintainer", "2026-11-02T00:00:00Z")
    review = activity(
        "fastmcp-triage-reviewer[bot]",
        "2026-10-03T00:00:00Z",
        bot=True,
        commit_id="c" * 40,
    )
    calls = []

    def run(command, **kwargs):
        calls.append(command)
        value = [[*comments], [last]] if "comments" in command[2] else [[review]]
        if "--paginate" not in command:
            value = {"commit": {"sha": "b" * 40}} if "branches/" in command[2] else pr()
        return SimpleNamespace(stdout=json.dumps(value))

    monkeypatch.setattr(triage.subprocess, "run", run)
    report = triage.build([item()], NOW)
    lead = report["review_leads"][0]
    assert lead["activity"]["comments"] == 101
    assert lead["activity"]["last_maintainer_account_activity_at"] == last["created_at"]
    assert lead["activity"]["triage_app_review_seen"] is True
    assert lead["activity"]["triage_app_review_on_snapshot_revision"] is False
    assert lead["head_sha"] == "a" * 40
    assert all(
        "--slurp" in command
        for command in calls
        if "comments" in command[2] or "reviews" in command[2]
    )


def test_failed_activity_is_unknown_not_unanswered(triage, monkeypatch):
    def failed(*args, **kwargs):
        raise subprocess.TimeoutExpired("gh", 20)

    monkeypatch.setattr(triage, "github", failed)
    report = triage.build([item()], NOW)
    lead = report["review_leads"][0]
    assert lead["activity"] == {"status": "unavailable"}
    assert "first_maintainer_account_response_at" not in lead["activity"]
    assert "collection_gap" in lead
    assert report["open_age_buckets"]["prs"]["30_90d"] == 1


def test_bot_and_author_are_not_maintainer_response(triage, monkeypatch):
    comments = [
        activity("contributor", "2026-10-02T00:00:00Z"),
        activity("bot", "2026-10-03T00:00:00Z", bot=True),
    ]
    monkeypatch.setattr(
        triage,
        "github",
        lambda path, **kw: pr() if not kw else comments if "comments" in path else [],
    )
    lead = triage.build([item()], NOW)["review_leads"][0]
    assert lead["activity"]["status"] == "collected"
    assert lead["activity"]["first_maintainer_account_response_at"] is None


@pytest.mark.parametrize(
    "change",
    [
        {"draft": True},
        {"title": "[DNM] Fix"},
        {"body": "DO NOT MERGE"},
        {"labels": [{"name": "dont-merge"}]},
        {"author_association": "MEMBER"},
        {"user": {"login": "bot", "type": "Bot"}},
    ],
)
def test_holds_drafts_bots_and_maintainers_are_not_leads(triage, monkeypatch, change):
    monkeypatch.setattr(
        triage, "github", lambda *a, **kw: pytest.fail("must not inspect excluded PR")
    )
    report = triage.build([item(**change)], NOW)
    assert not report["review_leads"]
    assert sum(report["open_age_buckets"]["prs"].values()) == 1


def test_hold_after_inventory_prevents_nomination(triage, monkeypatch):
    monkeypatch.setattr(triage, "github", lambda *a, **kw: pr(title="[DRAFT] held"))
    report = triage.build([item()], NOW)
    assert report["selection"]["eligible_in_inventory"] == 1
    assert not report["review_leads"]


def test_age_boundaries_and_empty_success_are_explicit(triage):
    dates = ["2026-11-03", "2026-10-27", "2026-10-04", "2026-08-05"]
    items = [
        item(i, pull_request=None, created_at=d + "T18:00:00Z")
        for i, d in enumerate(dates)
    ]
    report = triage.build(items, NOW)
    assert report["open_age_buckets"]["issues"] == {
        "lt_7d": 1,
        "7_30d": 1,
        "30_90d": 1,
        "gte_90d": 1,
    }
    empty = triage.build([], NOW)
    assert empty["inventory_items"] == 0
    assert empty["collection_started_at"] == NOW.isoformat()
    assert empty["collection_finished_at"]
    assert empty["selection"]["returned"] == 0


def test_bounded_queue_keeps_counts_and_does_not_export_bodies(triage, monkeypatch):
    calls = []

    def inspect(number, now):
        calls.append(number)
        return {"activity": {"status": "unavailable"}}

    monkeypatch.setattr(triage, "inspect_pr", inspect)
    report = triage.build(
        [item(2, body="unpublished-sentinel input"), item(1)], NOW, limit=1
    )
    assert calls == [1]
    assert report["selection"]["eligible_in_inventory"] == 2
    assert sum(report["open_age_buckets"]["prs"].values()) == 2
    assert "unpublished-sentinel" not in json.dumps(report)


def test_counts_use_collection_start_like_backlog_chart(triage):
    report = triage.build(
        [
            item(
                1, pull_request=None, state="closed", closed_at="2026-11-03T18:01:00Z"
            ),
            item(
                2, pull_request=None, state="closed", closed_at="2026-11-03T17:59:00Z"
            ),
            item(3, pull_request=None, created_at="2026-11-03T18:01:00Z"),
        ],
        NOW,
    )
    assert sum(report["open_age_buckets"]["issues"].values()) == 1


def test_discussion_failure_preserves_successful_head_observation(triage, monkeypatch):
    def fetch(path, **kwargs):
        if kwargs:
            raise subprocess.TimeoutExpired("gh", 20)
        return pr()

    monkeypatch.setattr(triage, "github", fetch)
    lead = triage.build([item()], NOW)["review_leads"][0]
    assert lead["head_sha"] == "a" * 40
    assert lead["pr_collected_at"]
    assert lead["activity"] == {"status": "unavailable"}
    assert "collection_gap" in lead


@pytest.mark.parametrize(
    "failure,expected_kind,status,expected_calls",
    [
        (
            subprocess.TimeoutExpired("gh", 15, stderr="secret-sentinel"),
            "timeout",
            None,
            2,
        ),
        (
            subprocess.CalledProcessError(1, "gh", stderr="secret-sentinel (HTTP 503)"),
            "http_error",
            503,
            2,
        ),
        (
            subprocess.CalledProcessError(1, "gh", stderr="secret-sentinel (HTTP 403)"),
            "http_error",
            403,
            1,
        ),
        (
            subprocess.CalledProcessError(
                1,
                "gh",
                stderr="invalid character 'S' looking for beginning of value: secret-sentinel",
            ),
            "invalid_json",
            None,
            2,
        ),
    ],
)
def test_sanitized_diagnostics_and_bounded_retry(
    triage, monkeypatch, failure, expected_kind, status, expected_calls
):
    calls = []

    def run(*args, **kwargs):
        calls.append(args)
        raise failure

    monkeypatch.setattr(triage.subprocess, "run", run)
    monkeypatch.setattr(triage.time, "sleep", lambda seconds: None)
    report = triage.build([item()], NOW)
    gap = report["review_leads"][0]["collection_diagnostics"][0]
    assert gap == {
        "stage": "pr_metadata",
        "kind": expected_kind,
        "attempts": expected_calls,
        "http_status": status,
    }
    assert len(calls) == expected_calls
    assert "secret-sentinel" not in json.dumps(report)
    assert report["review_leads"][0]["activity"]["status"] == "unavailable"


def test_transient_non_json_response_recovers_without_retaining_body(
    triage, monkeypatch
):
    results = iter(
        [
            SimpleNamespace(stdout="Server Error secret-sentinel"),
            SimpleNamespace(stdout=json.dumps(pr())),
        ]
    )
    calls = []

    def run(command, **kwargs):
        calls.append(command)
        return next(results)

    monkeypatch.setattr(triage.subprocess, "run", run)
    monkeypatch.setattr(triage.time, "sleep", lambda seconds: None)
    assert triage.github("pulls/1")["head"]["sha"] == "a" * 40
    assert len(calls) == 2


def marked_review(*, head="a" * 40, baseline="b" * 40, number=1, **overrides):
    return (
        activity(
            "fastmcp-triage-reviewer[bot]",
            "2026-10-03T00:00:00Z",
            bot=True,
            id=7,
            state="COMMENTED",
            commit_id=head,
            body=f"<!-- fastmcp-triage-review:v1 pr:{number} head:{head} base:{baseline} -->\nPublic finding",
        )
        | overrides
    )


@pytest.mark.parametrize(
    "review,expected",
    [
        (marked_review(), True),
        (marked_review(baseline="c" * 40), False),
        (marked_review(head="c" * 40), False),
        (marked_review(number=2), False),
        (marked_review(commit_id="c" * 40), False),
        (marked_review(body="unbound review"), False),
        (marked_review(state="DISMISSED"), False),
    ],
)
def test_review_provenance_requires_pr_head_and_current_target_baseline(
    triage, monkeypatch, review, expected
):
    def fetch(path, **kwargs):
        if path.startswith("branches/"):
            return {"commit": {"sha": "b" * 40}}
        if path.endswith("/reviews?per_page=100"):
            return [review]
        return [] if kwargs else pr(base={"sha": "d" * 40, "ref": "main"})

    monkeypatch.setattr(triage, "github", fetch)
    lead = triage.build([item()], NOW)["review_leads"][0]
    assert (
        lead["base_sha"] == "d" * 40
    )  # PR metadata isn't the authoritative target tip.
    assert lead["baseline_sha"] == "b" * 40
    assert lead["activity"]["triage_app_review_on_snapshot_revision"] is expected
    assert lead["activity"]["triage_app_review_seen"] is True


def test_missing_baseline_stays_unknown_while_discussion_is_retained(
    triage, monkeypatch
):
    def fetch(path, **kwargs):
        if path.startswith("branches/"):
            raise triage.GitHubReadError("http_error", 1, 403)
        if "/reviews?" in path:
            return [marked_review()]
        return [] if kwargs else pr()

    monkeypatch.setattr(triage, "github", fetch)
    lead = triage.build([item()], NOW)["review_leads"][0]
    assert lead["head_sha"] == "a" * 40
    assert lead["baseline_sha"] is None
    assert lead["activity"]["status"] == "collected"
    assert lead["activity"]["triage_app_review_on_snapshot_revision"] is None
    assert lead["collection_diagnostics"] == [
        {
            "stage": "target_baseline",
            "kind": "http_error",
            "attempts": 1,
            "http_status": 403,
        }
    ]


def test_untrusted_account_cannot_supply_revision_marker(triage, monkeypatch):
    review = marked_review(
        user={"login": "fastmcp-triage-reviewer[bot]", "type": "User"}
    )
    monkeypatch.setattr(
        triage,
        "github",
        lambda path, **kw: (
            {"commit": {"sha": "b" * 40}}
            if path.startswith("branches/")
            else [review]
            if "/reviews?" in path
            else []
            if kw
            else pr()
        ),
    )
    activity_result = triage.build([item()], NOW)["review_leads"][0]["activity"]
    assert activity_result["triage_app_review_seen"] is False
    assert activity_result["triage_app_review_bindings"] == []


def test_optional_triage_failure_cannot_block_backlog_publication():
    import yaml

    workflow = yaml.safe_load(
        (Path(__file__).parents[1] / ".github/workflows/backlog-status.yml").read_text()
    )
    jobs = workflow["jobs"]
    assert jobs["publish"]["needs"] == "build"
    assert jobs["publish-triage"]["needs"] == "triage"
    assert "needs" not in jobs["build"] and "needs" not in jobs["triage"]
    maintenance = yaml.safe_load(
        (
            Path(__file__).parents[1] / ".github/workflows/maintenance-status.yml"
        ).read_text()
    )
    for publisher in (
        jobs["publish"],
        jobs["publish-triage"],
        maintenance["jobs"]["publish"],
    ):
        assert publisher["concurrency"] == {
            "group": "maintenance-status-publish",
            "cancel-in-progress": False,
            "queue": "max",
        }
    commands = "\n".join(step.get("run", "") for step in jobs["build"]["steps"])
    assert "--triage-output" not in commands and "public_triage.py" not in commands
    assert (
        "pull_request" in jobs["publish"]["if"]
        and "pull_request" in jobs["publish-triage"]["if"]
    )


def test_malformed_discussion_retains_metadata_and_stage(triage, monkeypatch):
    def fetch(path, **kwargs):
        if path.startswith("branches/"):
            return {"commit": {"sha": "b" * 40}}
        return ["secret-sentinel"] if kwargs else pr()

    monkeypatch.setattr(triage, "github", fetch)
    report = triage.build([item()], NOW)
    lead = report["review_leads"][0]
    assert lead["head_sha"] == "a" * 40
    assert lead["activity"]["status"] == "unavailable"
    assert lead["collection_diagnostics"] == [
        {
            "stage": "discussion",
            "kind": "invalid_response",
            "attempts": 1,
            "http_status": None,
        }
    ]
    assert report["lead_collection_status"] == "partial"
    assert "secret-sentinel" not in json.dumps(report)


def test_standalone_cli_generates_provenance_without_cloud_access(tmp_path):
    # Intercept only the external gh boundary in the child interpreter. A fake
    # shebang executable on PATH cannot run on Windows.
    wrapper = tmp_path / "run_collector.py"
    wrapper.write_text(
        "import json, runpy, subprocess, sys\nfrom pathlib import Path\n"
        "original_run = subprocess.run\n"
        "def fake_run(command, *args, **kwargs):\n"
        "    if command[0] != 'gh':\n"
        "        return original_run(command, *args, **kwargs)\n"
        "    endpoint = command[2]\n"
        "    if 'per_page=1&' in endpoint: value = 1\n"
        f"    elif '/issues?' in endpoint: value = {item()!r}\n"
        f"    elif '/branches/' in endpoint: value = {{'commit': {{'sha': {'b' * 40!r}}}}}\n"
        "    elif '--slurp' in command: value = [[]]\n"
        f"    else: value = {pr()!r}\n"
        "    return subprocess.CompletedProcess(command, 0, json.dumps(value), '')\n"
        "subprocess.run = fake_run\n"
        "sys.argv = sys.argv[1:]\n"
        "sys.path.insert(0, str(Path(sys.argv[0]).parent))\n"
        "runpy.run_path(sys.argv[0], run_name='__main__')\n"
    )
    output = tmp_path / "triage.json"
    root = Path(__file__).parents[1]
    subprocess.run(
        [
            sys.executable,
            str(wrapper),
            str(root / "scripts/public_triage.py"),
            "--output",
            str(output),
        ],
        cwd=root,
        check=True,
        capture_output=True,
        text=True,
    )
    report = json.loads(output.read_text())
    assert len(report["source_sha"]) == 40
    assert report["inventory_items"] == 1
    assert report["lead_collection_status"] == "collected"
    assert report["review_leads"][0]["baseline_sha"] == "b" * 40
    assert report["review_leads"][0]["activity"]["status"] == "collected"