Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

12.3 KB
Raw Blame
"""Exercise the actual workflow conditions without app dependencies.

Policy under test (revert of the #7101 label gate): the shared test image and
the full pytest suite run unconditionally on every pull request and every
main push. No label can request, defer, or suppress them; a red
"All Pytest Tests + Coverage" check therefore means the suite failed, not
that validation was never requested. Merge enforcement for a failed image
build lives in the repository ruleset, which lists "Build Test Image" as a
required check; this file can only verify the workflow side of that contract.

Actionlint validates YAML and Actions syntax separately. The small expression
interpreter below accepts only the operators/functions used by these conditions
and rejects unknown syntax or context. These are policy regressions, not a
simulation of GitHub's scheduler or proof of a hosted run's final check
conclusion.
"""

import ast
import json
import re
import textwrap
import unittest
from pathlib import Path


ROOT = Path(__file__).resolve().parents[3]
WORKFLOW = (ROOT / ".github/workflows/docker-tests.yml").read_text()
TOKEN = re.compile(
    r"\s*('(?:[^']|'')*'|&&|\|\||==|!=|!|[(),]|"
    r"[A-Za-z_][A-Za-z0-9_.*-]*|[0-9]+)"
)
# Every #7101 gating mechanism that must stay out of the workflow: the
# request labels, the label-event special cases, and the failing guard.
GATE_ARTIFACTS = (
    "test:pytest",
    "code-ready",
    "code-ready-preliminary",
    "auto-merge",
    "github.event.label",
    "github.event.action",
    "PYTEST_REQUESTED",
    "Full pytest is required before merging",
)
REQUIRED = {
    "detect-changes": "detect-changes",
    "build-test-image": "Build Test Image",
    "pytest-tests": "All Pytest Tests + Coverage",
}
FOCUSED_JOBS = (
    ("llm-unit-tests", "llm"),
    ("llm-example-tests", "llm"),
    ("accessibility-tests", "accessibility"),
    ("ldr-production-smoke-test", "docker"),
)


def job(name):
    match = re.search(
        rf"^  {re.escape(name)}:\n(.*?)(?=^  [\w-]+:|\Z)",
        WORKFLOW.split("\njobs:\n", 1)[1],
        re.MULTILINE | re.DOTALL,
    )
    if match is None:
        raise AssertionError(f"Missing job: {name}")
    return match[1]


def field(text, key, indent):
    """Read a scalar at a known YAML indentation; not a general YAML parser."""
    lines = text.splitlines()
    prefix = " " * indent + key + ":"
    for index, line in enumerate(lines):
        if not line.startswith(prefix):
            continue
        value = line[len(prefix) :].strip()
        if value not in {"|", ">", ">-", "|-"}:
            return value
        body = []
        for following in lines[index + 1 :]:
            if following.strip() and not following.startswith(
                " " * (indent + 1)
            ):
                break
            body.append(following)
        return textwrap.dedent("\n".join(body)).strip()
    raise AssertionError(f"Missing field: {key}")


def evaluate(expression, context, *, cancelled=False, success=True):
    expression = expression.strip().removeprefix("${{").removesuffix("}}")
    translated = []
    functions = {
        "always",
        "cancelled",
        "success",
        "contains",
        "format",
        "fromJSON",
        "hashFiles",
    }
    operators = {"&&": "and", "||": "or", "!": "not"}
    position = 0
    while position < len(expression.rstrip()):
        match = TOKEN.match(expression, position)
        if match is None:
            raise AssertionError(
                f"Unsupported expression: {expression[position:]}"
            )
        token = match[1]
        position = match.end()
        if token.startswith("'"):
            translated.append(repr(token[1:-1].replace("''", "'")))
        elif token in {"true", "false", "null"}:
            translated.append(
                {"true": "True", "false": "False", "null": "None"}[token]
            )
        elif re.match(r"[A-Za-z_]", token) and token not in functions:
            translated.append(f"lookup({token!r})")
        else:
            translated.append(operators.get(token, token))

    def lookup(path):
        # Every referenced field must be modelled; missing fields never silently
        # become false and conceal a new bypass in a workflow expression.
        if path not in context:
            raise AssertionError(f"Unmodelled context: {path}")
        return context[path]

    calls = {
        "lookup": lookup,
        "fromJSON": json.loads,
        "always": lambda: True,
        "cancelled": lambda: cancelled,
        "success": lambda: success,
        "contains": lambda values, value: (
            value.lower() in [item.lower() for item in (values or [])]
        ),
        "format": lambda template, value: template.format(value),
        "hashFiles": lambda pattern: "test-coverage-hash",
    }

    def visit(node):
        if isinstance(node, ast.Constant):
            return node.value
        if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
            return calls[node.func.id](*(visit(arg) for arg in node.args))
        if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not):
            return not visit(node.operand)
        if isinstance(node, ast.BoolOp):
            result = visit(node.values[0])
            for value in node.values[1:]:
                if isinstance(node.op, ast.And) and not result:
                    break
                if isinstance(node.op, ast.Or) and result:
                    break
                result = visit(value)
            return result
        if isinstance(node, ast.Compare) and len(node.ops) == 1:
            left, right = visit(node.left), visit(node.comparators[0])
            if isinstance(left, str) and isinstance(right, str):
                left, right = left.lower(), right.lower()
            if isinstance(node.ops[0], ast.Eq):
                return left == right
            if isinstance(node.ops[0], ast.NotEq):
                return left != right
        raise AssertionError(f"Unsupported expression node: {ast.dump(node)}")

    return visit(ast.parse(" ".join(translated).strip(), mode="eval").body)


def context(
    event="pull_request",
    action="synchronize",
    *,
    ref=None,
    reusable=False,
    strict=None,
):
    if ref is None:
        ref = (
            "refs/pull/42/merge"
            if event == "pull_request"
            else "refs/heads/main"
        )
    if strict is None:
        strict = reusable  # workflow_call defaults strict-mode to true
    return {
        "github.event_name": event,
        "github.event.action": action,
        "github.event.pull_request.number": 42,
        "github.ref": ref,
        "github.run_id": 123,
        "inputs.reusable-invocation": reusable,
        "inputs.strict-mode": strict,
        "needs.pytest-tests.result": "success",
        "needs.build-test-image.result": "success",
        "needs.detect-changes.outputs.llm": "false",
        "needs.detect-changes.outputs.infrastructure": "false",
        "needs.detect-changes.outputs.docker": "false",
        "needs.detect-changes.outputs.accessibility": "false",
    }


class PytestSchedulingTests(unittest.TestCase):
    def test_pr_trigger_excludes_label_events_and_daily_schedule(self):
        self.assertEqual(
            field(WORKFLOW, "types", 4),
            "[opened, synchronize, reopened]",
        )
        self.assertNotRegex(WORKFLOW, r"(?m)^\s*schedule:")

    def test_no_label_gate_remains_in_the_workflow(self):
        for artifact in GATE_ARTIFACTS:
            with self.subTest(artifact=artifact):
                self.assertNotIn(artifact, WORKFLOW)

    def test_required_checks_run_on_every_pr_and_main_push(self):
        contexts = [
            context(action=a) for a in ("opened", "synchronize", "reopened")
        ]
        contexts += [
            context("push"),
            context(reusable=True),
            context("workflow_dispatch"),
        ]
        for ctx in contexts:
            with self.subTest(
                event=ctx["github.event_name"],
                action=ctx["github.event.action"],
                reusable=ctx["inputs.reusable-invocation"],
            ):
                for job_id, check in REQUIRED.items():
                    self.assertEqual(field(job(job_id), "name", 4), check)
                self.assertTrue(
                    evaluate(field(job("build-test-image"), "if", 4), ctx)
                )
        # A manual dispatch on a non-main branch also builds the image, so its
        # pytest runs instead of skipping (a skipped job reports success); a
        # push to any other branch does not trigger the workflow.
        self.assertTrue(
            evaluate(
                field(job("build-test-image"), "if", 4),
                context("workflow_dispatch", ref="refs/heads/feature"),
            )
        )

    def test_detect_changes_and_pytest_carry_no_condition(self):
        # The pre-#7101 shape: these jobs run whenever their dependencies
        # allow. A reintroduced `if:` here is how the #7101 gate came back.
        for job_id in ("detect-changes", "pytest-tests"):
            with self.subTest(job=job_id), self.assertRaises(AssertionError):
                field(job(job_id), "if", 4)

    def test_pytest_depends_on_the_image_build_without_a_request_guard(self):
        self.assertEqual(
            field(job("pytest-tests"), "needs", 4), "build-test-image"
        )
        self.assertNotIn("pytest_request", job("pytest-tests"))
        self.assertNotIn("PYTEST_REQUESTED", job("pytest-tests"))

    def test_image_build_is_unconditional_and_path_filter_free(self):
        build = job("build-test-image")
        with self.assertRaises(AssertionError):
            field(build, "needs", 4)
        self.assertNotIn("detect-changes", build)
        self.assertNotIn("labels", build)

    def test_focused_checks_keep_their_path_filters(self):
        # Path filters still gate the focused suites themselves; only the
        # shared image build stopped consulting them.
        for job_id, output in FOCUSED_JOBS:
            with self.subTest(job=job_id):
                condition = field(job(job_id), "if", 4)
                self.assertIn(
                    f"needs.detect-changes.outputs.{output}", condition
                )

    def test_concurrency_cancels_superseded_direct_pr_runs_only(self):
        group = field(WORKFLOW, "group", 2)
        cancel = field(WORKFLOW, "cancel-in-progress", 2)
        direct = [
            context(action=a) for a in ("opened", "synchronize", "reopened")
        ]
        self.assertEqual(len({evaluate(group, ctx) for ctx in direct}), 1)
        for ctx in direct:
            self.assertTrue(evaluate(cancel, ctx))
        push, newer = context("push"), {**context("push"), "github.run_id": 456}
        self.assertEqual(evaluate(group, push), evaluate(group, newer))
        self.assertFalse(evaluate(cancel, push))
        for ctx in (
            context("workflow_dispatch", ref="refs/heads/feature"),
            context(reusable=True),
            context("push", reusable=True),
        ):
            self.assertFalse(evaluate(cancel, ctx))
            self.assertNotEqual(
                evaluate(group, ctx), evaluate(group, direct[0])
            )
            self.assertNotEqual(evaluate(group, ctx), evaluate(group, push))
        # Main keeps GitHub's default single-pending-slot behavior; queueing
        # every push would restore the main-push backlog.
        concurrency = WORKFLOW.split("\nconcurrency:\n", 1)[1].split(
            "\npermissions:", 1
        )[0]
        self.assertNotRegex(concurrency, r"\bqueue:")

    def test_publish_coverage_only_from_a_successful_direct_main_push(self):
        condition = field(job("publish-coverage"), "if", 4)
        self.assertNotIn("schedule", condition)
        self.assertTrue(evaluate(condition, context("push")))
        for ctx in (
            {**context("push"), "needs.pytest-tests.result": "failure"},
            context("pull_request"),
            context(reusable=True),
            context("workflow_dispatch", ref="refs/heads/feature"),
        ):
            with self.subTest(context=ctx):
                self.assertFalse(evaluate(condition, ctx))

    def test_release_pipeline_still_calls_the_strict_suite(self):
        release_caller = (ROOT / ".github/workflows/ci-gate.yml").read_text()
        self.assertRegex(
            release_caller,
            r"uses: \$/\.github/workflows/docker-tests.yml\s+with:\s+strict-mode: true",
        )


if __name__ == "__main__":
    unittest.main()