Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

32.9 KB
Raw Blame
"""Invariants that keep `CI Summary` fail-closed.

`CI Summary` is a required status check on `main`. GitHub reports a *skipped*
job to branch protection as a success, so the summary can only stay fail-closed
while two properties hold:

1. every job it gathers is also named in its ``check_job`` list, and
2. none of those jobs is ever skipped, which is why their conditions live on
   their steps rather than on the job.

Both are prose in ``docs/branch-protection.md``. This module turns them into
something that fails. See that document's "Required contexts must be summary
jobs" and "Gate at the step, not at the job" sections for the reasoning.

``ci.yml`` has a second contract test, ``frontend/src/ci/frontend-test-manifest.test.ts``.
It freezes the summary script by exact text, pins both ``jobs.changes`` filter
rule sets, and checks the six required frontend-build steps semantically --
command, working directory, shell, and an ``if:`` that is either absent or
exactly the path-filter gate. A change to either
region has to update both files or CI fails in the frontend lane.
"""

from __future__ import annotations

import copy
import re
import tomllib
from pathlib import Path

import pytest
import yaml

REPO_ROOT = Path(__file__).resolve().parents[1]
CI_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "ci.yml"
MAIN_PUSH_CHECK = REPO_ROOT / ".github" / "workflows" / "main-push-check.yml"
PYPROJECT = REPO_ROOT / "pyproject.toml"

# This suite validates the `changes` outputs, so a job gated on them cannot be
# the only place it runs.
CONTRACT_TEST_PATH = "tests/test_ci_summary_contract.py"

# The no-op step that keeps a gated job reporting success instead of skipping.
# Matched by name, because it is the one step whose guard is legitimately inverted.
SKIP_SENTINEL_PREFIX = "Skip"

DRAFT_GUARD = (
    "github.event_name != 'pull_request' || github.event.pull_request.draft == false"
)

# Jobs whose steps are gated on the `changes` job, and the output each one
# reads. Everything else CI Summary gathers runs unconditionally.
GATED_JOBS = {
    "pytest-fast": "code",
    "pytest-fast-deepdoc": "code",
    "pytest-slow": "code",
    "pytest-milvus": "code",
    "e2e": "code",
    "frontend-build": "frontend",
}

# Widening this set means docs-only pull requests stop running some part of the
# suite. Changing it here as well as in the workflow is the point: it has to be
# a deliberate act, not a one-line edit that reads as harmless in review.
#
# `!frontend/src/**` rather than `!frontend/**` because Python tests read
# frontend/package.json and frontend/public, and paths-filter cannot re-include a
# file an earlier pattern excluded. See docs/branch-protection.md.
CODE_FILTER_EXCLUDES = frozenset(
    {"!docs/**", "!assets/**", "!*.md", "!frontend/src/**"}
)

# The wheel-build inputs frontend-build owns. A rule dropped or negated here
# stops the frontend lane running for changes that need it.
FRONTEND_FILTER_RULES = (
    "frontend/**",
    "pyproject.toml",
    # Hatchling honours .gitignore when it selects the files that go into the
    # wheel, so an edit there can silently drop one.
    ".gitignore",
    "README.md",
    # The whole package tree, not just the entrypoint: hatchling packages all
    # of src/xagent and drops any file .gitignore matches, so a backend-only
    # change can break the wheel (PR #1848 review).
    "src/xagent/**",
    ".github/workflows/ci.yml",
)

# This action decides whether the test suite runs at all, so it is pinned by
# commit. Bumping it must be deliberate -- update this constant in the same
# change.
PATHS_FILTER_PIN = "dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d"

# pulls.listFiles caps its response at this many files, and paths-filter does
# not compare the rows it received against the pull request's changed_files.
LIST_FILES_MAX = 3000

# The step each gated job exists to run, and the command it must execute. Named
# rather than searched: a substring scan over every step's text lets a decoy in
# the sentinel stand in for a deleted workload. frontend-build is absent -- the
# TS contract pins its steps by exact command.
GATED_JOB_WORK_STEPS = {
    "pytest-fast": ("Run tests", "python -m pytest"),
    "pytest-fast-deepdoc": ("Run tests", "python -m pytest"),
    "pytest-slow": ("Run slow tests", "python -m pytest"),
    "pytest-milvus": ("Run Milvus tests", "python -m pytest"),
    "e2e": ("Run e2e tests", "python -m pytest"),
}

_CACHE_HIT = "(needs.prepare-deepdoc-cache.outputs.cache-hit == 'true')"
_CACHE_MISS = "(needs.prepare-deepdoc-cache.outputs.cache-hit != 'true')"

# Every step whose gate may carry more than the `changes` output, and the exact
# extra it carries. Keyed by (job, step), not allowlisted globally: each of these
# predicates is false on some reachable run, so one that is right on a setup step
# silently skips a workload step (PR #1848 review).
STEP_GUARD_EXTRAS = {
    ("pytest-fast", "Free GitHub-hosted runner disk space"): ("(matrix.name == 'web')"),
    ("pytest-fast", "Pre-pull the sandbox image"): "(matrix.name == 'web')",
    ("pytest-fast", "Run privileged writable sandbox tests"): (
        "(matrix.name == 'web')"
    ),
    ("pytest-fast-deepdoc", "Pre-pull the sandbox image"): "(matrix.name == 'core')",
    ("pytest-fast-deepdoc", "Restore Deepdoc cache"): _CACHE_HIT,
    ("pytest-fast-deepdoc", "Download Deepdoc cache artifact"): _CACHE_MISS,
    ("pytest-slow", "Restore Deepdoc cache"): _CACHE_HIT,
    ("pytest-slow", "Download Deepdoc cache artifact"): _CACHE_MISS,
    ("pytest-milvus", "Show Milvus logs"): "failure()",
}

# Applied to the value above, not to the workflow: renaming a leg in `strategy`
# alone leaves a guard that is false on every leg.
MATRIX_CONJUNCT = re.compile(r"^\(matrix\.name == '([a-z0-9-]+)'\)$")


class _NoDuplicateKeyLoader(yaml.SafeLoader):
    """A loader that rejects duplicate mapping keys.

    PyYAML accepts them and keeps the last one, which silently drops a step's
    guard when an edit adds a second ``if:`` to the same step. GitHub rejects
    the workflow outright, so the permissive parse is the wrong default here.
    """


def _no_duplicates(loader: yaml.Loader, node: yaml.MappingNode, deep: bool = False):
    seen = set()
    for key_node, _ in node.value:
        key = loader.construct_object(key_node, deep=deep)
        if key in seen:
            raise AssertionError(
                f"duplicate key {key!r} at line {key_node.start_mark.line + 1}"
            )
        seen.add(key)
    return yaml.SafeLoader.construct_mapping(loader, node, deep=deep)


_NoDuplicateKeyLoader.add_constructor(
    yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _no_duplicates
)


@pytest.fixture(scope="module")
def workflow() -> dict:
    return yaml.load(
        CI_WORKFLOW.read_text(encoding="utf-8"), Loader=_NoDuplicateKeyLoader
    )


@pytest.fixture(scope="module")
def jobs(workflow: dict) -> dict:
    return workflow["jobs"]


@pytest.fixture(scope="module")
def summary(jobs: dict) -> dict:
    return jobs["ci-summary"]


def _filter_step(jobs: dict) -> dict:
    return next(step for step in jobs["changes"]["steps"] if step.get("id") == "filter")


def _matrix_leg_names(job: dict) -> list[str]:
    include = (job.get("strategy") or {}).get("matrix", {}).get("include") or []
    return [leg["name"] for leg in include if "name" in leg]


def _normalise(expression: str) -> str:
    return re.sub(r"\s+", " ", expression).strip()


def test_summary_gathers_every_job_it_checks(jobs: dict, summary: dict) -> None:
    """`needs` and the `check_job` calls must name the same set of jobs.

    A job missing from either list is advisory only: it can fail without
    failing the required check.
    """
    script = summary["steps"][0]["run"]
    checked = set(re.findall(r'check_job "([^"]+)"', script))
    gathered = set(summary["needs"])

    assert checked == gathered, (
        "ci-summary's needs and its check_job list have drifted apart; "
        f"only in needs: {sorted(gathered - checked)}, "
        f"only in check_job: {sorted(checked - gathered)}"
    )

    # Nothing may be left out of the summary altogether.
    everything_else = set(jobs) - {"ci-summary"}
    assert gathered == everything_else, (
        "every job in ci.yml must be gathered by ci-summary; "
        f"missing: {sorted(everything_else - gathered)}"
    )


def test_gathered_jobs_are_never_skipped(summary: dict, jobs: dict) -> None:
    """No gathered job may carry a condition beyond the draft guard.

    Any other job-level `if:` makes the job skippable, and a skipped required
    job reports success to branch protection. Conditions belong on the steps.
    """
    for name in summary["needs"]:
        condition = _normalise(jobs[name].get("if", DRAFT_GUARD))
        assert condition == _normalise(DRAFT_GUARD), (
            f"job '{name}' carries a job-level condition: {condition!r}. "
            "A gathered job that can be skipped reports success to branch "
            "protection -- gate its steps instead. See "
            "docs/branch-protection.md 'Gate at the step, not at the job'."
        )


def test_summary_rejects_a_non_literal_changes_output(summary: dict) -> None:
    """An empty output leaves only the sentinel running, and the job goes green.

    `check_job` cannot see that, so the summary has to reject the value itself.
    """
    script = summary["steps"][0]["run"]
    checked = set(re.findall(r'check_flag "([^"]+)"', script))
    assert checked == set(GATED_JOBS.values()), (
        "every output the gated jobs read must be asserted to be a literal "
        f"true/false; missing: {sorted(set(GATED_JOBS.values()) - checked)}"
    )


@pytest.mark.parametrize("job_name", sorted(GATED_JOBS))
def test_every_step_of_a_gated_job_is_guarded(jobs: dict, job_name: str) -> None:
    """Guards must have the right polarity, not merely name the output.

    Referencing the output is not enough: flipping a work step to ``!= 'true'``
    skips the real tests on exactly the pull requests that need them while the
    job, and so the summary, stays green.
    """
    output = GATED_JOBS[job_name]
    run_guard = f"needs.changes.outputs.{output} == 'true'"
    skip_guard = f"needs.changes.outputs.{output} != 'true'"

    sentinels = []
    for step in jobs[job_name]["steps"]:
        label = step.get("name") or step.get("uses") or "<unnamed>"
        condition = _normalise(step.get("if") or "")
        assert condition, (
            f"step '{label}' in job '{job_name}' has no condition; every step "
            f"of a gated job must be guarded on {run_guard}"
        )

        if label.startswith(SKIP_SENTINEL_PREFIX):
            sentinels.append(label)
            assert condition == skip_guard, (
                f"sentinel '{label}' in job '{job_name}' is conditional on "
                f"{condition!r}; it must be exactly {skip_guard!r} so the job "
                "reports success on a change it does not apply to"
            )
            continue

        # The gate has to lead and cannot be OR-ed away.
        assert condition == run_guard or condition.startswith(f"{run_guard} && "), (
            f"step '{label}' in job '{job_name}' is conditional on "
            f"{condition!r}; a work step must start with {run_guard!r} and may "
            "only add '&&' conjunctions"
        )

        remainder = condition[len(run_guard) :].removeprefix(" && ")
        expected = STEP_GUARD_EXTRAS.get((job_name, label), "")
        assert remainder == expected, (
            f"step '{label}' in job '{job_name}' is gated on "
            f"{run_guard!r} && {remainder!r}, but the only extra this step may "
            f"carry is {expected!r}. Every permitted predicate is false on some "
            "reachable run, so one on the wrong step skips it there while the "
            "job still reports success."
        )

        leg = MATRIX_CONJUNCT.match(expected)
        if leg:
            declared = _matrix_leg_names(jobs[job_name])
            assert leg.group(1) in declared, (
                f"step '{label}' in job '{job_name}' narrows itself to matrix "
                f"leg {leg.group(1)!r}, but the job declares "
                f"{declared or 'no matrix at all'}. The predicate is false on "
                "every leg, so the step never runs while the job still reports "
                "success."
            )

    assert len(sentinels) == 1, (
        f"job '{job_name}' must have exactly one '{SKIP_SENTINEL_PREFIX}...' "
        f"sentinel step so it never reports an empty run; found {sentinels}"
    )


# Words that open or close a compound statement, and their effect on nesting.
# `if false; then <workload>; fi` is not proof that the workload runs.
_BLOCK_WORDS = {
    "if": 1,
    "while": 1,
    "until": 1,
    "for": 1,
    "case": 1,
    "select": 1,
    "{": 1,
    "fi": -1,
    "done": -1,
    "esac": -1,
    "}": -1,
    "then": 0,
    "else": 0,
    "elif": 0,
    "do": 0,
    "in": 0,
    "(": 0,
    ")": 0,
}

_SEPARATOR = re.compile(r"\|\||&&|[;\n|&]")
_ASSIGNMENT = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=")

# The one wrapper a workload is legitimately launched through, so the check
# cannot be satisfied by another command that merely mentions pytest.
WORKLOAD_RUNNER_PREFIXES = ("python3 -m uv run ",)


def _top_level_commands(script: str) -> list[str]:
    """The commands a `run:` script reaches unconditionally, in command position.

    Substring membership is not execution: `echo python -m pytest`,
    `if false; then python -m pytest; fi`, `true || python -m pytest` and a
    function body that is never called all contain the workload and none of them
    runs it. So quoted literals and comments go first, then a command counts only
    at nesting depth zero and only from the start of a command.

    This is a command-shape contract, not a shell parser: it models nesting,
    command position and `||`, and deliberately accepts the right-hand side of
    `&&`, which the leading `set -e` already makes load-bearing in these steps.
    """
    text = re.sub(r"'[^']*'|\"[^\"]*\"", "", script)
    text = re.sub(r"#[^\n]*", "", text)
    text = re.sub(r"\\\n", " ", text)

    commands: list[str] = []
    depth = 0
    preceded_by = ""
    cursor = 0
    for match in [*_SEPARATOR.finditer(text), None]:
        segment = text[cursor : match.start()] if match else text[cursor:]
        words = segment.split()
        while words and words[0] in _BLOCK_WORDS:
            depth = max(depth + _BLOCK_WORDS[words.pop(0)], 0)
        if words and words[-1] == "{":
            # `run_tests() { ... }` defines the workload, it does not call it.
            depth += 1
            words.pop()
        if words and depth == 0 and preceded_by != "||":
            commands.append(" ".join(words))
        if match:
            preceded_by = match.group(0)
            cursor = match.end()
    return commands


def _invokes(segment: str, command: str) -> bool:
    words = segment.split()
    while words and _ASSIGNMENT.match(words[0]):
        words.pop(0)
    text = " ".join(words)
    for prefix in WORKLOAD_RUNNER_PREFIXES:
        text = text.removeprefix(prefix)
    return text == command or text.startswith(f"{command} ")


@pytest.mark.parametrize("job_name", sorted(GATED_JOB_WORK_STEPS))
def test_a_gated_job_still_runs_its_test_command(jobs: dict, job_name: str) -> None:
    """A correctly guarded job that no longer runs anything is still green."""
    step_name, command = GATED_JOB_WORK_STEPS[job_name]
    matching = [
        step for step in jobs[job_name]["steps"] if step.get("name") == step_name
    ]

    assert len(matching) == 1, (
        f"job '{job_name}' must have exactly one '{step_name}' step, the one it "
        f"exists to run; found {len(matching)}"
    )
    commands = _top_level_commands(matching[0].get("run") or "")
    assert any(_invokes(c, command) for c in commands), (
        f"step '{step_name}' in job '{job_name}' never reaches {command!r} in "
        f"command position; the commands it does run are {commands}. Every step "
        "of a gated job is skippable by design, so without this the job reports "
        "success having executed no tests at all."
    )


@pytest.mark.parametrize("job_name", sorted(GATED_JOBS))
def test_gated_jobs_depend_on_changes(jobs: dict, job_name: str) -> None:
    needs = jobs[job_name]["needs"]
    needs = [needs] if isinstance(needs, str) else needs
    assert "changes" in needs, (
        f"job '{job_name}' reads needs.changes.outputs but does not list "
        "'changes' in its needs"
    )


def test_changes_job_can_read_the_pull_request(jobs: dict) -> None:
    """paths-filter uses the pulls.listFiles API whenever a token is present.

    Without this scope the call is a 403 and every pull request fails here.
    """
    permissions = jobs["changes"]["permissions"]
    assert permissions.get("pull-requests") == "read"
    assert permissions.get("contents") == "read"


def test_paths_filter_is_pinned_by_commit(jobs: dict) -> None:
    """The pin has to bind to the step that produces the outputs.

    Looking for the SHA anywhere in the job would also accept a dead pinned
    step sitting beside a real gate that uses a mutable tag.
    """
    assert _filter_step(jobs)["uses"] == PATHS_FILTER_PIN, (
        f"the 'filter' step must use exactly {PATHS_FILTER_PIN}, found "
        f"{_filter_step(jobs)['uses']!r}. Bumping the action is a supply-chain "
        "decision: it gates whether the test suite runs at all."
    )

    invocations = [
        step["uses"]
        for step in jobs["changes"]["steps"]
        if "dorny/paths-filter" in step.get("uses", "")
    ]
    assert invocations == [PATHS_FILTER_PIN], (
        "the changes job must invoke paths-filter exactly once, pinned; found "
        f"{invocations}"
    )


def test_code_filter_exclusions_are_exactly_as_declared(jobs: dict) -> None:
    step = _filter_step(jobs)
    filters = yaml.safe_load(step["with"]["filters"])

    excludes = {p for p in filters["code"] if p.startswith("!")}
    assert excludes == CODE_FILTER_EXCLUDES, (
        "the set of paths treated as 'not code' changed; anything listed here "
        "stops triggering the test suite on its own. "
        f"added: {sorted(excludes - CODE_FILTER_EXCLUDES)}, "
        f"removed: {sorted(CODE_FILTER_EXCLUDES - excludes)}"
    )

    # The include side has to stay open: everything is code until excluded, so
    # a newly added top-level directory runs the suite rather than skipping it.
    assert "**" in filters["code"]

    # Without this quantifier the exclusions above do not mean what they read
    # as -- the default `some` does not combine includes with excludes.
    assert step["with"]["predicate-quantifier"] == "some-with-excludes"


def test_frontend_filter_rules_are_exactly_as_declared(jobs: dict) -> None:
    """Both suites pin these rules, which is overlap rather than duplication.

    This suite's load-bearing run is the ungated `pre-commit` job, so no filter
    edit can skip it; the frontend suite is still `frontend`-gated. Dropping or
    negating a rule has to get past both.
    """
    filters = yaml.safe_load(_filter_step(jobs)["with"]["filters"])

    assert tuple(filters["frontend"]) == FRONTEND_FILTER_RULES, (
        "the frontend filter rules changed; anything removed here stops the "
        f"frontend lane running for that path. expected "
        f"{list(FRONTEND_FILTER_RULES)}, found {filters['frontend']}"
    )

    negated = [rule for rule in filters["frontend"] if rule.startswith("!")]
    assert not negated, (
        f"the frontend filter must not exclude anything; found {negated}. An "
        "exclusion cannot be undone by a later rule, so it silently narrows the "
        "lane."
    )


def test_non_pull_request_events_never_filter(jobs: dict) -> None:
    """The merge queue must behave exactly as it did before this job existed."""
    for output in GATED_JOBS.values():
        expression = jobs["changes"]["outputs"][output]
        assert "github.event_name != 'pull_request' ||" in expression, (
            f"output '{output}' must short-circuit to true on non-pull_request "
            "events so merge_group and workflow_dispatch run everything"
        )
        assert f"steps.filter.outputs.{output} != 'false'" in expression, (
            f"output '{output}' must use != 'false' so a missing filter output "
            "runs everything instead of silently skipping"
        )


def test_a_truncated_file_list_runs_everything(jobs: dict) -> None:
    """paths-filter cannot see past the 3000th file, and does not notice.

    It paginates pulls.listFiles without comparing the rows it got back to
    changed_files, so a larger pull request whose visible portion is all
    excluded paths yields a genuine `false` with code after the cutoff.
    """
    for output in sorted(set(GATED_JOBS.values())):
        expression = _normalise(jobs["changes"]["outputs"][output])
        assert (
            f"github.event.pull_request.changed_files > {LIST_FILES_MAX}" in expression
        ), (
            f"output '{output}' must fall back to true once the pull request "
            f"exceeds {LIST_FILES_MAX} files, which truncates the file list "
            f"paths-filter reads; got {expression!r}"
        )
        assert "github.event.pull_request.changed_files == null" in expression, (
            f"output '{output}' must also fall back to true when the file count "
            "is unavailable -- an absent count compares as 0 and would pass the "
            "size check silently"
        )


def _expected_output_expression(output: str) -> str:
    return (
        "${{ github.event_name != 'pull_request'"
        " || github.event.pull_request.changed_files == null"
        f" || github.event.pull_request.changed_files > {LIST_FILES_MAX}"
        f" || steps.filter.outputs.{output} != 'false' }}}}"
    )


def test_the_changes_outputs_are_exactly_as_declared(jobs: dict) -> None:
    """The two tests above assert substrings, which precedence can defeat.

    `&&` binds tighter than `||`, so a chain rewritten to use it keeps every
    substring they look for while forcing the output to false on an ordinary
    pull request -- skipping the whole suite.
    """
    for output in sorted(set(GATED_JOBS.values())):
        expression = _normalise(jobs["changes"]["outputs"][output])
        assert expression == _expected_output_expression(output), (
            f"the '{output}' output expression changed. Each clause is a "
            "fallback to running everything, and they have to stay a flat "
            f"'||' chain. expected {_expected_output_expression(output)!r}, "
            f"found {expression!r}"
        )


def test_frontend_filter_covers_the_readme_the_wheel_needs(jobs: dict) -> None:
    """frontend-build is the only job that builds a wheel.

    pyproject points `readme` at a root markdown file, which `code` excludes, so
    unless the frontend filter names it too, renaming it skips that build.
    """
    readme = tomllib.loads(PYPROJECT.read_text(encoding="utf-8"))["project"]["readme"]
    filters = yaml.safe_load(_filter_step(jobs)["with"]["filters"])

    assert readme in filters["frontend"], (
        f"pyproject declares readme = {readme!r}, so it is a wheel build input, "
        f"but the frontend filter is {filters['frontend']}. A pull request "
        "touching only that file would skip 'Verify package bundles the "
        "frontend', the one step that builds the wheel."
    )


def _repo_path_for_wheel_member(member: str, packages: list[str]) -> str | None:
    """Where a wheel member lives in the repository, per hatch's `packages`."""
    head, _, rest = member.partition("/")
    for package in packages:
        if package.rsplit("/", 1)[-1] == head:
            return f"{package}/{rest}" if rest else package
    return None


def _covered_by_filter(path: str, rules: list[str]) -> bool:
    return any(
        path.startswith(rule[:-2]) if rule.endswith("/**") else path == rule
        for rule in rules
    )


def test_wheel_members_are_covered_by_the_frontend_filter(jobs: dict) -> None:
    """The grep targets and the filter rules are literals in two namespaces.

    The archive holds `xagent/...` while the filter names `src/xagent/...`, so
    nothing textual links them and either side can be repointed alone. Hatch's
    `packages` is the mapping, which makes the relationship assertable.
    """
    step = _step(jobs, "frontend-build", "Verify package bundles the frontend")
    members = re.findall(r'grep -q "([^"]+)"', step["run"])
    assert members, (
        "the wheel step no longer asserts any package member is in the archive"
    )

    wheel = tomllib.loads(PYPROJECT.read_text(encoding="utf-8"))["tool"]["hatch"][
        "build"
    ]["targets"]["wheel"]
    rules = yaml.safe_load(_filter_step(jobs)["with"]["filters"])["frontend"]

    for member in members:
        path = _repo_path_for_wheel_member(member, wheel["packages"])
        assert path is not None, (
            f"the wheel step asserts {member!r}, which is under none of hatch's "
            f"packages {wheel['packages']}. Either the grep target or the "
            "packaging config was repointed without the other."
        )
        assert _covered_by_filter(path, rules), (
            f"the wheel step asserts {member!r}, which lives at {path!r}, but "
            f"the frontend filter {rules} does not cover it. A change to that "
            "file would skip the only job that inspects a built wheel."
        )


def _mutated(jobs: dict) -> dict:
    return copy.deepcopy(jobs)


def _step(jobs: dict, job_name: str, step_name: str) -> dict:
    return next(
        step for step in jobs[job_name]["steps"] if step.get("name") == step_name
    )


def test_a_work_guard_widened_with_an_event_check_is_rejected(jobs: dict) -> None:
    mutated = _mutated(jobs)
    _step(mutated, "pytest-fast", "Run tests")["if"] = (
        "needs.changes.outputs.code == 'true' && github.event_name == 'push'"
    )

    with pytest.raises(AssertionError):
        test_every_step_of_a_gated_job_is_guarded(mutated, "pytest-fast")


def test_a_gated_job_stripped_to_its_sentinel_is_rejected(jobs: dict) -> None:
    mutated = _mutated(jobs)
    mutated["pytest-fast"]["steps"] = [
        step
        for step in mutated["pytest-fast"]["steps"]
        if (step.get("name") or "").startswith(SKIP_SENTINEL_PREFIX)
    ]

    with pytest.raises(AssertionError):
        test_a_gated_job_still_runs_its_test_command(mutated, "pytest-fast")


def test_an_and_chained_output_expression_is_rejected(jobs: dict) -> None:
    """`&&` binds tighter than `||`, so this skips CI on every ordinary PR."""
    mutated = _mutated(jobs)
    mutated["changes"]["outputs"]["code"] = (
        _normalise(mutated["changes"]["outputs"]["code"])
        .replace("== null ||", "== null &&")
        .replace("> 3000 ||", "> 3000 &&")
    )

    test_non_pull_request_events_never_filter(mutated)
    test_a_truncated_file_list_runs_everything(mutated)

    with pytest.raises(AssertionError):
        test_the_changes_outputs_are_exactly_as_declared(mutated)


def test_the_contract_runs_in_a_job_the_filter_cannot_gate(
    jobs: dict, summary: dict
) -> None:
    """This suite is the thing that catches a neutered `changes` job.

    Behind a gate it validates nothing: an output pinned to a literal `false`
    skips the jobs, skips this suite with them, and `check_flag` accepts
    `false` by design, so the summary stays green having run no tests at all.
    """
    ungated = [name for name in summary["needs"] if name not in GATED_JOBS]
    running = [
        name
        for name in ungated
        for step in jobs[name]["steps"]
        if CONTRACT_TEST_PATH in (step.get("run") or "") and not step.get("if")
    ]

    assert running, (
        f"no unconditional step in any of {sorted(ungated)} runs "
        f"{CONTRACT_TEST_PATH}. Every job that does run it is gated on the "
        "very outputs it exists to validate."
    )


def test_a_workload_replaced_with_an_echo_decoy_is_rejected(jobs: dict) -> None:
    mutated = _mutated(jobs)
    _step(mutated, "pytest-fast", "Run tests")["run"] = "echo 'python -m pytest'\n"

    with pytest.raises(AssertionError):
        test_a_gated_job_still_runs_its_test_command(mutated, "pytest-fast")


def test_a_matrix_leg_renamed_out_from_under_a_guard_is_rejected(
    jobs: dict,
) -> None:
    """The rename is on the `strategy` side; the guard still names the old leg."""
    mutated = _mutated(jobs)
    for leg in mutated["pytest-fast"]["strategy"]["matrix"]["include"]:
        if leg["name"] == "web":
            leg["name"] = "webapp"

    with pytest.raises(AssertionError):
        test_every_step_of_a_gated_job_is_guarded(mutated, "pytest-fast")


def test_a_matrix_guard_on_a_job_without_a_matrix_is_rejected(jobs: dict) -> None:
    mutated = _mutated(jobs)
    _step(mutated, "e2e", "Run e2e tests")["if"] = (
        "needs.changes.outputs.code == 'true' && (matrix.name == 'web')"
    )

    with pytest.raises(AssertionError):
        test_every_step_of_a_gated_job_is_guarded(mutated, "e2e")


def test_a_wheel_member_outside_the_frontend_filter_is_rejected(jobs: dict) -> None:
    mutated = _mutated(jobs)
    step = _step(mutated, "frontend-build", "Verify package bundles the frontend")
    step["run"] = step["run"].replace(
        "xagent/web/__main__.py", "somewhere/else/__main__.py"
    )

    with pytest.raises(AssertionError):
        test_wheel_members_are_covered_by_the_frontend_filter(mutated)


def test_an_unquoted_echo_decoy_is_rejected(jobs: dict) -> None:
    mutated = _mutated(jobs)
    _step(mutated, "pytest-fast", "Run tests")["run"] = (
        "set -e\necho python -m pytest tests\n"
    )

    with pytest.raises(AssertionError):
        test_a_gated_job_still_runs_its_test_command(mutated, "pytest-fast")


def test_a_workload_in_an_unreachable_branch_is_rejected(jobs: dict) -> None:
    mutated = _mutated(jobs)
    _step(mutated, "pytest-fast", "Run tests")["run"] = (
        "set -e\nif false; then python3 -m uv run python -m pytest; fi\n"
    )

    with pytest.raises(AssertionError):
        test_a_gated_job_still_runs_its_test_command(mutated, "pytest-fast")


def test_a_declared_matrix_leg_is_rejected_on_a_workload_step(jobs: dict) -> None:
    mutated = _mutated(jobs)
    _step(mutated, "pytest-fast", "Run tests")["if"] = (
        "needs.changes.outputs.code == 'true' && (matrix.name == 'web')"
    )

    with pytest.raises(AssertionError):
        test_every_step_of_a_gated_job_is_guarded(mutated, "pytest-fast")


def test_a_cache_predicate_is_rejected_on_a_workload_step(jobs: dict) -> None:
    mutated = _mutated(jobs)
    _step(mutated, "pytest-slow", "Run slow tests")["if"] = (
        "needs.changes.outputs.code == 'true' && "
        "(needs.prepare-deepdoc-cache.outputs.cache-hit != 'true')"
    )

    with pytest.raises(AssertionError):
        test_every_step_of_a_gated_job_is_guarded(mutated, "pytest-slow")


def test_a_pre_pull_leg_swapped_for_another_declared_leg_is_rejected(
    jobs: dict,
) -> None:
    mutated = _mutated(jobs)
    _step(mutated, "pytest-fast", "Pre-pull the sandbox image")["if"] = (
        "needs.changes.outputs.code == 'true' && (matrix.name == 'integration')"
    )

    with pytest.raises(AssertionError):
        test_every_step_of_a_gated_job_is_guarded(mutated, "pytest-fast")


def _triggers(workflow: dict) -> dict:
    # YAML 1.1 reads a bare `on` key as the boolean True.
    return workflow.get("on", workflow.get(True))


def test_ci_does_not_rerun_on_push(workflow: dict) -> None:
    """The merge queue already ran ci.yml on the commit a merge puts on main.

    A push trigger repeats that run on every merge and holds runners queued
    pull requests are waiting for.
    """
    assert "push" not in _triggers(workflow), (
        "ci.yml must not run on push; main-push-check.yml calls it for a push "
        "the merge queue did not test"
    )
    assert "workflow_call" in _triggers(workflow), (
        "main-push-check.yml calls ci.yml, which needs a workflow_call trigger"
    )


def test_a_push_the_queue_did_not_test_still_runs_ci() -> None:
    """Administrators can push to main directly, bypassing the queue.

    main-push-check.yml is the only thing that runs ci.yml on such a commit, so
    it has to fire on every push, look for a successful merge_group run of this
    exact commit, treat a failed lookup as "not tested", and otherwise call
    ci.yml.
    """
    check = yaml.load(
        MAIN_PUSH_CHECK.read_text(encoding="utf-8"), Loader=_NoDuplicateKeyLoader
    )
    assert _triggers(check) == {"push": {"branches": ["main"]}}

    lookup = check["jobs"]["check"]
    assert "if" not in lookup
    # Key presence, not truthiness: `if: false` loads as a falsy False.
    assert not [step for step in lookup["steps"] if "if" in step]
    script = "\n".join(step.get("run", "") for step in lookup["steps"])
    assert (
        "actions/workflows/ci.yml/runs?head_sha=$SHA&event=merge_group&status=success"
        in script
    )
    assert "|| count=0" in script, "a failed lookup must run the suite"

    ci = check["jobs"]["ci"]
    assert ci["uses"] == "./.github/workflows/ci.yml"
    assert ci["needs"] == "check"
    assert _normalise(ci["if"]) == (
        "${{ !cancelled() && needs.check.outputs.tested != 'true' }}"
    ), "only a confirmed merge_group success may skip the suite"