Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

14.1 KB
Raw Blame
"""Execute the actual workflow selector before it can propose a release PR.

Also pins the breaking-release guard shared by version_check.yml and
release.yml (``scripts/ci/check_breaking_release.py``): breaking-fragment
detection must agree with what towncrier renders, and a non-major version
must never be proposed or published while breaking fragments are pending.
"""

import os
from pathlib import Path
import subprocess
import sys

import pytest
import yaml


ROOT = Path(__file__).resolve().parents[2]
WORKFLOW = ROOT / ".github/workflows/version_check.yml"
RELEASE_WORKFLOW = ROOT / ".github/workflows/release.yml"
SCRIPT = ROOT / "scripts/ci/check_breaking_release.py"
STALE_STEP = "Close stale non-major bump PR"

sys.path.insert(0, str(ROOT / "scripts/ci"))
sys.path.insert(0, str(ROOT / ".pre-commit-hooks"))

import check_breaking_release  # noqa: E402
from _changelog_fragments import towncrier_fragments  # noqa: E402


def _steps():
    return yaml.safe_load(WORKFLOW.read_text())["jobs"]["version-bump"]["steps"]


def _step(steps, **match):
    return next(
        step
        for step in steps
        if all(step.get(key) == value for key, value in match.items())
    )


def _repo_tree(tmp_path, fragments):
    """Lay out the files the workflow scripts read, relative to the cwd."""
    for name in ("scripts", ".pre-commit-hooks", "pyproject.toml"):
        (tmp_path / name).symlink_to(ROOT / name)
    if fragments:
        directory = tmp_path / "changelog.d"
        directory.mkdir()
        for name in fragments:
            (directory / name).write_text("Release change\n")


def _bash(script, cwd, env):
    # GitHub's `shell: bash` runs `bash --noprofile --norc -eo pipefail`.
    return subprocess.run(
        ["bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", script],
        cwd=cwd,
        env={**os.environ, **env},
        capture_output=True,
        text=True,
        check=False,
    )


def _run_selector(tmp_path, event, requested):
    output = tmp_path / "outputs"
    output.touch()
    selector = _step(_steps(), id="release")
    result = _bash(
        selector["run"],
        tmp_path,
        {
            "EVENT_NAME": event,
            "REQUESTED_TYPE": requested,
            "GITHUB_OUTPUT": str(output),
        },
    )
    outputs = dict(
        line.split("=", 1) for line in output.read_text().splitlines()
    )
    return result, outputs


@pytest.mark.parametrize(
    "fragments,event,requested,expected_type,allowed,exit_code",
    [
        ([], "push", "", "patch", True, 0),
        (["123.bugfix.md"], "push", "", "patch", True, 0),
        (["123.feature.md"], "push", "", "patch", True, 0),
        (["123.breaking.md"], "push", "", None, False, 0),
        (["123.bugfix.md", "456.breaking.md"], "push", "", None, False, 0),
        ([], "workflow_dispatch", "patch", "patch", True, 0),
        ([], "workflow_dispatch", "minor", "minor", True, 0),
        (["123.breaking.md"], "workflow_dispatch", "patch", None, False, 1),
        (["123.breaking.md"], "workflow_dispatch", "minor", "minor", True, 0),
        (["123.breaking.md"], "workflow_dispatch", "major", "major", True, 0),
        ([], "workflow_dispatch", "invalid", None, False, 1),
        # Counter-suffixed breaking fragments are sanctioned by the
        # changelog-fragment grammar (pre-commit hook: <id>.<category>.<n>.md,
        # rendered as a separate breaking bullet) and MUST trip the gate too.
        (["456.breaking.2.md"], "push", "", None, False, 0),
        (["456.breaking.2.md"], "workflow_dispatch", "patch", None, False, 1),
        (["+two-breaks.breaking.3.md"], "push", "", None, False, 0),
        # towncrier renders a dot-prefixed fragment (issue ".123") under
        # Breaking Changes even though shell globs skip it.
        ([".123.breaking.md"], "push", "", None, False, 0),
        ([".123.breaking.md"], "workflow_dispatch", "patch", None, False, 1),
        # towncrier takes the LAST declared category, so this is a bugfix.
        (["123.breaking.bugfix.md"], "push", "", "patch", True, 0),
        # A slug containing "breaking" is not the breaking category.
        (
            ["+breaking-release-selection.bugfix.md"],
            "push",
            "",
            "patch",
            True,
            0,
        ),
    ],
)
def test_release_selection(
    tmp_path, fragments, event, requested, expected_type, allowed, exit_code
):
    _repo_tree(tmp_path, fragments)
    result, outputs = _run_selector(tmp_path, event, requested)
    assert result.returncode == exit_code, result.stdout + result.stderr
    assert (outputs.get("allowed") == "true") is allowed
    assert outputs.get("type") == expected_type
    if not allowed and exit_code == 0:
        assert "Breaking changes" in result.stdout


def test_selector_fails_closed_when_detection_errors(tmp_path):
    # Without pyproject.toml the detector cannot know the categories. That
    # must fail the step, never read as "no breaking fragments pending".
    (tmp_path / "scripts").symlink_to(ROOT / "scripts")
    (tmp_path / ".pre-commit-hooks").symlink_to(ROOT / ".pre-commit-hooks")
    (tmp_path / "changelog.d").mkdir()
    (tmp_path / "changelog.d" / "123.breaking.md").write_text("x\n")
    result, outputs = _run_selector(tmp_path, "push", "")
    assert result.returncode != 0
    assert "allowed" not in outputs


def test_selector_env_maps_the_real_event_and_dispatch_input():
    # The selector tests above inject EVENT_NAME / REQUESTED_TYPE directly,
    # so pin the workflow's own mapping from the GitHub context.
    workflow = yaml.safe_load(WORKFLOW.read_text())
    selector = _step(_steps(), id="release")
    assert selector["shell"] == "bash"
    assert selector["env"] == {
        "EVENT_NAME": "${{ github.event_name }}",
        "REQUESTED_TYPE": "${{ inputs.release_type }}",
    }
    # Only the mapped variables reach the script; no inline expressions.
    assert "${{" not in selector["run"]
    assert '"$EVENT_NAME"' in selector["run"]
    assert '"$REQUESTED_TYPE"' in selector["run"]
    # PyYAML parses the bare `on:` key as boolean True.
    triggers = workflow.get("on", workflow.get(True))
    release_input = triggers["workflow_dispatch"]["inputs"]["release_type"]
    assert release_input["options"] == ["patch", "minor", "major"]


def test_every_downstream_step_requires_an_allowed_release():
    # A correct selector alone is insufficient if PR creation can bypass it.
    steps = _steps()
    selector_index = next(
        index for index, step in enumerate(steps) if step.get("id") == "release"
    )
    downstream = steps[selector_index + 1 :]
    assert any(step["name"] == "Create Pull Request" for step in downstream)
    for step in downstream:
        if step["name"] == STALE_STEP:
            # The only step for the blocked case; it closes, never creates.
            assert step["if"] == "steps.release.outputs.allowed == 'false'"
            assert "gh pr create" not in step["run"]
            continue
        condition = step.get("if", "")
        assert condition.startswith(
            "steps.release.outputs.allowed == 'true' && "
        ), step["name"]
        assert "||" not in condition, step["name"]


def test_bump_is_rechecked_against_pending_breaking_fragments():
    steps = _steps()
    names = [step["name"] for step in steps]
    guard = _step(steps, name="Refuse a non-major version for breaking changes")
    assert names.index("Bump version") < names.index(guard["name"])
    assert names.index(guard["name"]) < names.index("Create Pull Request")
    assert guard["env"] == {
        "NEW_VERSION": "${{ steps.bump.outputs.new_version }}"
    }
    assert guard["run"].strip() == (
        'python scripts/ci/check_breaking_release.py version "$NEW_VERSION"'
    )
    assert not guard.get("continue-on-error", False)
    assert "new_version=" in _step(steps, id="bump")["run"]


def test_release_workflow_refuses_non_major_breaking_release():
    # A stale patch bump PR merged by hand reaches release.yml directly.
    build = yaml.safe_load(RELEASE_WORKFLOW.read_text())["jobs"]["build"]
    steps = build["steps"]
    names = [step.get("name") for step in steps]
    guard = _step(
        steps, name="Refuse a non-major release with pending breaking changes"
    )
    assert guard["env"] == {
        "RELEASE_VERSION": "${{ steps.version.outputs.version }}"
    }
    assert guard["run"].strip() == (
        'python3 scripts/ci/check_breaking_release.py version "$RELEASE_VERSION"'
    )
    assert guard["if"] == "steps.check_release.outputs.exists == 'false'"
    assert not guard.get("continue-on-error", False)
    # Only pre-flight checks run before the guard: nothing is downloaded,
    # attested, signed or uploaded for a version the guard refuses.
    preflight = {
        "Harden the runner (Audit all outbound calls)",
        "Checkout code",
        "Verify build commit is on main",
        "Determine version",
        "Check if release already exists",
        "Verify version matches __version__.py",
    }
    guard_index = names.index(guard["name"])
    assert set(names[:guard_index]) <= preflight
    for later in (
        "Download tested Python distributions",
        "Verify Python artifacts before attestation",
        "Attest tested Python artifacts at the release commit",
        "Sign release artifacts with Sigstore",
        "Upload artifacts for release",
    ):
        assert names.index(later) > guard_index, later
    for step in steps[:guard_index]:
        uses = step.get("uses", "")
        for action in ("attest", "download-artifact", "upload-artifact"):
            assert action not in uses, uses
    checkout = _step(steps, name="Checkout code")
    assert "sparse-checkout" not in checkout.get("with", {})


@pytest.mark.parametrize(
    "version,major",
    [
        ("2.0.0", True),
        ("v2.0.0", True),
        ("2.0.0rc1", True),
        ("0.5.0", True),
        ("1.10.8", False),
        ("1.11.0", False),
        ("2.0.1", False),
        ("0.5.1", False),
        ("0.0.0", False),
        ("2.0", False),
        ("garbage", False),
        ("", False),
    ],
)
def test_is_major_release(version, major):
    assert check_breaking_release.is_major_release(version) is major


@pytest.mark.parametrize(
    "fragments,version,exit_code",
    [
        (["3299.breaking.md"], "1.10.8", 1),
        (["3299.breaking.md"], "1.11.0", 1),
        (["3299.breaking.md"], "2.0.0", 0),
        ([".3299.breaking.md"], "1.10.8", 1),
        (["3299.breaking.2.md"], "1.10.8", 1),
        (["3299.bugfix.md"], "1.10.8", 0),
        ([], "1.10.8", 0),
    ],
)
def test_version_subcommand(tmp_path, fragments, version, exit_code):
    _repo_tree(tmp_path, fragments)
    result = subprocess.run(
        [sys.executable, str(SCRIPT), "version", version],
        cwd=tmp_path,
        capture_output=True,
        text=True,
        check=False,
    )
    assert result.returncode == exit_code, result.stdout + result.stderr
    if exit_code:
        assert "::error::" in result.stdout


def test_close_stale_bump_pr_closes_only_non_major(tmp_path):
    # Execute the real step with a fake `gh`: #6083-style patch PRs close,
    # a manually dispatched major PR stays open.
    _repo_tree(tmp_path, ["3299.breaking.md"])
    bin_dir = tmp_path / "bin"
    bin_dir.mkdir()
    log = tmp_path / "gh.log"
    fake_gh = bin_dir / "gh"
    fake_gh.write_text(
        "#!/bin/bash\n"
        f'echo "$*" >> "{log}"\n'
        'if [ "$1 $2" = "pr list" ]; then\n'
        "  printf '6083\\tchore: bump patch version to 1.10.8\\n'\n"
        "  printf '6100\\tchore: bump minor version to 1.11.0\\n'\n"
        "  printf '7000\\tchore: bump major version to 2.0.0\\n'\n"
        "fi\n"
    )
    fake_gh.chmod(0o755)
    step = _step(_steps(), name=STALE_STEP)
    assert step["env"]["GH_TOKEN"] == "${{ secrets.GITHUB_TOKEN }}"
    runner_temp = tmp_path / "runner"
    runner_temp.mkdir()
    result = _bash(
        step["run"],
        tmp_path,
        {
            "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
            "RUNNER_TEMP": str(runner_temp),
        },
    )
    assert result.returncode == 0, result.stdout + result.stderr
    calls = log.read_text().splitlines()
    assert "--head chore/auto-version-bump" in calls[0]
    assert "isCrossRepository" in calls[0]
    closed = [call.split()[2] for call in calls if call.startswith("pr close")]
    assert closed == ["6083", "6100"]


def _towncrier_rendered(tmp_path, names):
    towncrier_builder = pytest.importorskip("towncrier._builder")
    towncrier_load = pytest.importorskip("towncrier._settings.load")
    (tmp_path / "pyproject.toml").write_text(
        (ROOT / "pyproject.toml").read_text()
    )
    directory = tmp_path / "changelog.d"
    directory.mkdir()
    for name in names:
        (directory / name).write_text("x\n")
    config = towncrier_load.load_config_from_file(
        str(tmp_path), str(tmp_path / "pyproject.toml")
    )
    _, files = towncrier_builder.find_fragments(
        str(tmp_path), config, strict=False
    )
    return {Path(path).name: category for path, category in files}


def test_detection_matches_towncrier_find_fragments(tmp_path):
    # Distinct (issue, category, counter) keys: towncrier rejects duplicates.
    names = [
        "1.breaking.md",
        "2.breaking.3.md",
        ".3.breaking.md",
        ".breaking.md",
        "4.breaking.txt",
        "5.breaking",
        "6.BREAKING.md",
        "7.breaking.bugfix.md",
        "8.bugfix.breaking.md",
        "x.y.breaking.md",
        "breaking.md",
        "9.feature.md",
        "notes.md",
        "README.md",
        "readme.rst",
        "Readme",
        ".gitkeep",
        ".keep",
        ".DS_Store",
    ]
    expected = _towncrier_rendered(tmp_path, names)
    assert expected["1.breaking.md"] == "breaking"
    assert expected[".3.breaking.md"] == "breaking"
    actual = towncrier_fragments(
        tmp_path / "changelog.d", tmp_path / "pyproject.toml"
    )
    assert actual == expected


def test_repo_changelog_detection_matches_towncrier(tmp_path):
    names = [p.name for p in (ROOT / "changelog.d").iterdir() if p.is_file()]
    expected = _towncrier_rendered(tmp_path, names)
    actual = towncrier_fragments(
        tmp_path / "changelog.d", tmp_path / "pyproject.toml"
    )
    assert actual == expected