Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

6.5 KB
Raw Blame
"""The notes concurrency probes must execute and block a broken release."""

from __future__ import annotations

import os
from pathlib import Path
import shlex
import subprocess
import sys

import pytest
import yaml

REPO_ROOT = Path(__file__).resolve().parents[2]
WORKFLOWS = REPO_ROOT / ".github" / "workflows"


def _workflow(name):
    parsed = yaml.safe_load((WORKFLOWS / name).read_text(encoding="utf-8"))
    # PyYAML uses YAML 1.1, which interprets the GitHub Actions `on` key as True.
    if True in parsed and "on" not in parsed:
        parsed["on"] = parsed.pop(True)
    return parsed


@pytest.fixture(scope="module")
def concurrency_workflow():
    return _workflow("notes-concurrency.yml")


def test_release_requires_successful_concurrency_gate():
    jobs = _workflow("release.yml")["jobs"]
    gate = jobs["notes-concurrency-gate"]
    assert gate["uses"] == "$/.github/workflows/notes-concurrency.yml"
    assert gate["needs"] == ["version-check"]
    assert gate["if"] == (
        "needs.version-check.outputs.should_release == 'true'"
    )
    assert not gate.get("continue-on-error", False)

    # Existing required gates must stay required alongside the new one. An
    # advisory job in build.needs alone would not prevent publication because
    # build has an explicit status expression that permits advisory failures.
    required = {
        "release-gate",
        "ci-gate",
        "e2e-test-gate",
        "notes-test-gate",
        "notes-concurrency-gate",
        "compat-test-gate",
        "compose-integration-gate",
    }
    build = jobs["build"]
    assert required <= set(build["needs"])
    condition = build["if"].removeprefix("${{").removesuffix("}}")
    assert "||" not in condition, (
        "A successful advisory job must not bypass a gate"
    )
    operands = {operand.strip() for operand in condition.split("&&")}
    assert "!cancelled()" in operands
    for name in required:
        assert f"needs.{name}.result == 'success'" in operands


def test_probes_run_serially_only_for_releases_or_manual_requests(
    concurrency_workflow,
):
    assert set(concurrency_workflow["on"]) == {
        "workflow_call",
        "workflow_dispatch",
    }
    job = concurrency_workflow["jobs"]["notes-concurrency"]
    assert "if" not in job, "A release call must not silently skip the probes"
    assert not job.get("continue-on-error", False)
    assert job["env"]["LDR_TESTING_WITH_MOCKS"] == "false"
    assert job["env"]["LDR_DISABLE_RATE_LIMITING"] == "false"
    steps = {step.get("id"): step for step in job["steps"] if "id" in step}
    run = steps["tests"]
    assert not run.get("continue-on-error", False)
    assert "if" not in run
    assert run["shell"] == "bash"  # Actions runs bash with -e -o pipefail.
    command = shlex.split(run["run"])
    command = command[command.index("pytest") + 1 :]
    assert "tests/integration/test_notes_release_concurrency.py" in command
    assert (
        "tests/notes/test_note_ai_service.py::"
        "TestNoteAIServiceAsyncTeardownAndOffload::"
        "test_cancellation_during_cleanup_waits_for_worker"
    ) in command
    assert command[command.index("-n") + 1] == "0"
    assert command[command.index("-p") + 1] == "no:cov"
    assert "--junitxml=test-results/notes-concurrency.xml" in command
    assert steps["validate-results"]["if"] == "always()"
    assert not steps["validate-results"].get("continue-on-error", False)

    uploads = [
        step
        for step in job["steps"]
        if step.get("uses", "").startswith("actions/upload-artifact@")
    ]
    assert uploads, (
        "Keep diagnostics available when the watchdog detects a stall"
    )
    assert all(step["if"] == "always()" for step in uploads)


@pytest.mark.parametrize(
    ("report", "passes"),
    [
        (
            '<testsuites><testsuite><testcase name="probe"/></testsuite></testsuites>',
            True,
        ),
        (None, False),
        ("<testsuites><testsuite tests='0'/></testsuites>", False),
        ("<testsuites>", False),
        *[
            (
                "<testsuites><testsuite>"
                '<testcase name="passed"/>'
                f'<testcase name="probe"><{outcome}/></testcase>'
                "</testsuite></testsuites>",
                False,
            )
            for outcome in ("skipped", "failure", "error")
        ],
    ],
    ids=[
        "pass",
        "missing",
        "empty",
        "malformed",
        "skipped",
        "failure",
        "error",
    ],
)
def test_report_validation_rejects_missing_or_unsuccessful_probes(
    concurrency_workflow, tmp_path, report, passes
):
    """Execute the workflow's real check, including a mixed pass/skip report."""
    if report is not None:
        reports = tmp_path / "test-results"
        reports.mkdir()
        (reports / "notes-concurrency.xml").write_text(report, encoding="utf-8")
    steps = concurrency_workflow["jobs"]["notes-concurrency"]["steps"]
    validator = next(
        step for step in steps if step.get("id") == "validate-results"
    )
    result = subprocess.run(
        ["bash", "-e", "-o", "pipefail", "-c", validator["run"]],
        cwd=tmp_path,
        env={
            **os.environ,
            "PATH": f"{Path(sys.executable).parent}{os.pathsep}{os.environ.get('PATH', '')}",
        },
        capture_output=True,
        text=True,
        timeout=10,
        check=False,
    )
    assert (result.returncode == 0) is passes, result.stdout + result.stderr


@pytest.mark.parametrize(
    "pytest_status", [0, 1, 5], ids=["pass", "failure", "no-tests"]
)
def test_pytest_exit_status_survives_diagnostic_tee(
    concurrency_workflow, tmp_path, pytest_status
):
    """The artifact-producing pipeline must preserve pytest failure/no-tests."""
    binary_dir = tmp_path / "bin"
    binary_dir.mkdir()
    pdm = binary_dir / "pdm"
    pdm.write_text(
        f"#!/bin/sh\nprintf 'probe output\\n'\nexit {pytest_status}\n",
        encoding="utf-8",
    )
    pdm.chmod(0o755)
    steps = concurrency_workflow["jobs"]["notes-concurrency"]["steps"]
    test_step = next(step for step in steps if step.get("id") == "tests")
    result = subprocess.run(
        ["bash", "-e", "-o", "pipefail", "-c", test_step["run"]],
        cwd=tmp_path,
        env={
            **os.environ,
            "PATH": f"{binary_dir}{os.pathsep}{os.environ.get('PATH', '')}",
        },
        capture_output=True,
        text=True,
        timeout=10,
        check=False,
    )
    assert result.returncode == pytest_status, result.stdout + result.stderr
    assert (
        tmp_path / "test-results" / "notes-concurrency.log"
    ).read_text() == ("probe output\n")