Skip to content

Self-hosted AI workspace.

Python
10.9 KB
Raw Blame
"""Regression coverage for PR description and validation-state checks."""

import json
import shutil
import subprocess
from pathlib import Path

import pytest


_REPO = Path(__file__).resolve().parent.parent
_CHECKER = _REPO / ".github" / "scripts" / "check-pr-description.js"
_WORKFLOW = _REPO / ".github" / "workflows" / "pr-description-check.yml"
pytestmark = pytest.mark.skipif(not shutil.which("node"), reason="node not on PATH")


def _body(
    *,
    app_ran=False,
    app_not_run=False,
    screenshot=False,
    media="",
    linked_issue="Fixes #5934",
):
    return f"""## Summary

This focused change has enough concrete summary detail for the checker.

## Linked Issue

{linked_issue}

## Type of Change

- [x] CI / tooling / configuration

## Checklist

- [x] I searched open issues and open PRs.
- [{'x' if app_ran else ' '}] I actually ran the app and verified the change works end-to-end.
- [{'x' if app_not_run else ' '}] I did not run the app/runtime validation and stated that gap in How to Test.

## How to Test

Run the focused checker regression tests and inspect their exact assertions.

## Visual / UI changes

- [{'x' if screenshot else ' '}] **Screenshot or short clip** of the running change.

### Screenshots / clips

{media}
"""


def _run_checker(
    files,
    body,
    *,
    missing_labels=(),
    draft=False,
    owner="odysseus-dev",
    repo="odysseus",
):
    harness = r"""
const checkPrDescription = require(process.argv[1]);
const input = JSON.parse(process.argv[2]);
const calls = [];
const listFiles = async () => {};
const listComments = async () => {};

const github = {
  paginate: async (method) => {
    if (method === listFiles) return input.files.map(filename => ({ filename }));
    if (method === listComments) return [];
    throw new Error('unexpected paginated method');
  },
  rest: {
    pulls: { listFiles },
    issues: {
      listComments,
      getLabel: async ({ name }) => {
        if (input.missingLabels.includes(name)) {
          const error = new Error(`missing label: ${name}`);
          error.status = 404;
          throw error;
        }
        return { data: { name } };
      },
      addLabels: async (params) => calls.push({ method: 'addLabels', params }),
      removeLabel: async (params) => calls.push({ method: 'removeLabel', params }),
      createComment: async (params) => calls.push({ method: 'createComment', params }),
      updateComment: async (params) => calls.push({ method: 'updateComment', params }),
      deleteComment: async (params) => calls.push({ method: 'deleteComment', params }),
    },
  },
};
const context = {
  payload: {
    pull_request: {
      number: 42,
      body: input.body,
      draft: input.draft,
    },
  },
  repo: { owner: input.owner, repo: input.repo },
};
const core = {
  warning: (message) => calls.push({ method: 'warning', message }),
  setFailed: (message) => calls.push({ method: 'setFailed', message }),
};

checkPrDescription({ github, context, core })
  .then(() => process.stdout.write(JSON.stringify(calls)))
  .catch((error) => {
    console.error(error);
    process.exitCode = 1;
  });
"""
    payload = json.dumps(
        {
            "files": files,
            "body": body,
            "missingLabels": list(missing_labels),
            "draft": draft,
            "owner": owner,
            "repo": repo,
        }
    )
    proc = subprocess.run(
        ["node", "-e", harness, str(_CHECKER), payload],
        capture_output=True,
        text=True,
        cwd=str(_REPO),
        timeout=30,
    )
    assert proc.returncode == 0, proc.stderr
    return json.loads(proc.stdout)


def _added_labels(calls):
    return {
        call["params"]["labels"][0]
        for call in calls
        if call["method"] == "addLabels"
    }


def _comment(calls):
    comments = [call for call in calls if call["method"] == "createComment"]
    return comments[0]["params"]["body"] if comments else ""


@pytest.mark.parametrize(
    ("files", "body"),
    [
        (["README.md"], _body()),
        (["routes/example.py"], _body(app_ran=True)),
        (
            ["static/js/example.js"],
            _body(
                app_ran=True,
                screenshot=True,
                media="https://github.com/user-attachments/assets/example",
            ),
        ),
    ],
)
def test_complete_expected_state_is_ready(files, body):
    calls = _run_checker(files, body)

    assert _added_labels(calls) == {"ready for review"}
    assert not _comment(calls)
    assert not any(call["method"] == "setFailed" for call in calls)


def test_public_repo_still_requires_linked_issue():
    calls = _run_checker(
        ["README.md"],
        _body(linked_issue="N/A — maintainer integration work"),
    )

    assert any(call["method"] == "setFailed" for call in calls)
    assert "**Linked Issue**" in _comment(calls)
    assert "ready for review" not in _added_labels(calls)


def test_maintainer_preview_accepts_explicit_na_linked_issue():
    calls = _run_checker(
        ["README.md"],
        _body(linked_issue="N/A — maintainer integration work"),
        owner="pewdiepie-archdaemon",
        repo="odysseus-maintainer-preview",
    )

    assert _added_labels(calls) == {"ready for review"}
    assert not _comment(calls)
    assert not any(call["method"] == "setFailed" for call in calls)


def test_maintainer_preview_rejects_ambiguous_non_issue_text():
    calls = _run_checker(
        ["README.md"],
        _body(linked_issue="No tracking needed"),
        owner="pewdiepie-archdaemon",
        repo="odysseus-maintainer-preview",
    )

    assert any(call["method"] == "setFailed" for call in calls)
    assert "**Linked Issue**" in _comment(calls)
    assert "ready for review" not in _added_labels(calls)


def test_ui_checkbox_without_media_still_needs_visual_evidence():
    calls = _run_checker(
        ["static/js/example.js"],
        _body(app_ran=True, screenshot=True),
    )

    assert _added_labels(calls) == {"needs visual evidence"}
    assert "does not contain an actual attachment or link" in _comment(calls)
    assert not any(call["method"] == "setFailed" for call in calls)


def test_ticked_screenshot_box_counts_without_the_template_bolding():
    body = _body(
        app_ran=True,
        screenshot=True,
        media="https://github.com/user-attachments/assets/example",
    ).replace("**Screenshot or short clip**", "Screenshot or short clip")

    calls = _run_checker(["static/js/example.js"], body)

    assert _added_labels(calls) == {"ready for review"}
    assert not _comment(calls)
    assert not any(call["method"] == "setFailed" for call in calls)


def test_explicit_not_run_is_honest_but_not_ready():
    calls = _run_checker(
        ["services/example.py"],
        _body(app_not_run=True),
    )

    assert _added_labels(calls) == {"needs runtime validation"}
    assert "explicitly reports that app/runtime validation was not performed" in _comment(calls)
    assert "does not prove that a local command ran" in _comment(calls)
    assert not any(call["method"] == "setFailed" for call in calls)


def test_missing_validation_label_fails_soft():
    calls = _run_checker(
        ["services/example.py"],
        _body(app_not_run=True),
        missing_labels=("needs runtime validation",),
    )

    assert "needs runtime validation" not in _added_labels(calls)
    assert any(
        call["method"] == "warning"
        and 'Label "needs runtime validation" does not exist' in call["message"]
        for call in calls
    )
    assert not any(call["method"] == "setFailed" for call in calls)


@pytest.mark.parametrize(
    "filename",
    ("requirements.txt", "requirements-optional.txt"),
)
def test_requirement_manifests_require_runtime_validation(filename):
    calls = _run_checker(
        [filename],
        _body(app_not_run=True),
    )

    assert _added_labels(calls) == {"needs runtime validation"}
    assert not any(
        call["method"] == "addLabels"
        and call["params"]["labels"] == ["ready for review"]
        for call in calls
    )
    assert any(
        call["method"] == "removeLabel"
        and call["params"]["name"] == "ready for review"
        for call in calls
    )
    assert "Changed-file classification: **backend/runtime**." in _comment(calls)


def test_old_template_runtime_pr_is_not_ready_without_attestation():
    calls = _run_checker(
        ["routes/example.py"],
        _body(),
    )

    assert _added_labels(calls) == {"needs runtime validation"}
    assert "App/runtime validation is not author-attested" in _comment(calls)
    assert not any(call["method"] == "setFailed" for call in calls)


def test_conflicting_runtime_attestations_are_not_ready():
    calls = _run_checker(
        ["services/example.py"],
        _body(app_ran=True, app_not_run=True),
    )

    assert _added_labels(calls) == {"needs runtime validation"}
    assert "both checked" in _comment(calls)
    assert not any(
        call["method"] == "addLabels"
        and call["params"]["labels"] == ["ready for review"]
        for call in calls
    )


def test_structurally_invalid_description_still_fails_hard_gate():
    calls = _run_checker(
        ["README.md"],
        "## Summary\nshort\n",
    )

    assert any(call["method"] == "setFailed" for call in calls)
    assert "PR description" in _comment(calls)
    assert "needs work" in _added_labels(calls)
    assert "ready for review" not in _added_labels(calls)


def test_github_workflow_only_change_does_not_require_app_runtime():
    calls = _run_checker(
        [".github/workflows/example.yml"],
        _body(),
    )

    assert _added_labels(calls) == {"ready for review"}
    assert not _comment(calls)
    assert not any(call["method"] == "setFailed" for call in calls)


def test_draft_pr_never_receives_ready_for_review():
    calls = _run_checker(
        ["README.md"],
        _body(),
        draft=True,
    )

    assert "ready for review" not in _added_labels(calls)
    assert any(
        call["method"] == "removeLabel"
        and call["params"]["name"] == "ready for review"
        for call in calls
    )
    assert not any(call["method"] == "setFailed" for call in calls)


def test_workflow_serializes_readiness_before_mergeability():
    workflow = _WORKFLOW.read_text()

    assert (
        "types: [opened, edited, synchronize, reopened, ready_for_review, "
        "converted_to_draft]"
    ) in workflow
    assert "group: pr-description-${{ github.event.pull_request.number }}" in workflow
    assert "cancel-in-progress: true" in workflow

    mergeable = workflow.split("  check-mergeable:", 1)[1]
    assert "needs: check-description" in mergeable
    assert (
        "if: ${{ !cancelled() && github.event.pull_request.user.type != 'Bot' }}"
        in mergeable
    )


def test_privileged_pr_workflow_executes_only_base_code():
    workflow = _WORKFLOW.read_text()

    assert "pull_request_target:" in workflow
    assert "ref: ${{ github.base_ref }}" in workflow
    assert "persist-credentials: false" in workflow
    assert "github.event.pull_request.head" not in workflow