Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

12.6 KB
Raw Blame
"""
CRITICAL — DO NOT MODIFY OR DELETE THIS FILE

These tests verify that the release gate workflow contains the pip install
verification job and all its required wiring. This job prevents publishing
broken wheels to PyPI.

Background (v1.4.0 incident):
  PDM resolution overrides (pyproject.toml [tool.pdm.resolution.overrides])
  leak into wheel metadata when `pdm build` runs. This creates dependency
  constraints that pip cannot resolve, making `pip install local-deep-research`
  fail for ALL users. PDM installs work fine because PDM handles overrides
  differently, so the problem is invisible during development and CI — it
  only surfaces after publishing to PyPI, when it's too late.

  Version 1.4.0 shipped with requests>=2.33 in its metadata (from a PDM
  override for a CVE fix), but arxiv~=2.4 requires requests~=2.32.0.
  pip could not resolve this conflict. Every user who ran
  `pip install local-deep-research` got an error.

Why guardian tests:
  The pip-install-check job in release-gate.yml is the ONLY thing that
  catches this class of bug before it reaches users. If someone removes
  it (e.g. to "simplify" the workflow, reduce CI time, or fix an
  unrelated failure), the safety net disappears silently. These tests
  make that removal loud and obvious.

DO NOT:
  - Delete this file
  - Mark these tests as xfail or skip
  - Weaken the assertions (e.g. checking substring instead of exact job name)
  - Move the pip-install-check job to a non-blocking position in the workflow
"""

from pathlib import Path
import re

import yaml
import pytest

RELEASE_GATE = (
    Path(__file__).resolve().parents[2]
    / ".github"
    / "workflows"
    / "release-gate.yml"
)


@pytest.fixture(scope="module")
def workflow():
    """Load the release-gate workflow YAML.

    CRITICAL — DO NOT MODIFY: this fixture is used by all guardian tests below.
    """
    assert RELEASE_GATE.exists(), (
        f"release-gate.yml not found at {RELEASE_GATE}. "
        "If you moved it, update the path — do NOT delete the tests."
    )
    return yaml.safe_load(RELEASE_GATE.read_text())


class TestPipInstallCheckJobExists:
    """
    CRITICAL — DO NOT DELETE OR WEAKEN THESE TESTS.

    They verify that the pip-install-check job exists in the release gate
    and cannot be published without catching PDM override leaks.
    See module docstring for the full incident report.
    """

    def test_pip_install_check_job_is_defined(self, workflow):
        """The pip-install-check job must exist in release-gate.yml.

        CRITICAL — DO NOT REMOVE: without this job, broken wheels reach PyPI.
        """
        jobs = workflow.get("jobs", {})
        assert "pip-install-check" in jobs, (
            "CRITICAL: 'pip-install-check' job is missing from release-gate.yml! "
            "This job prevents publishing wheels with broken dependencies to PyPI. "
            "It was removed — this MUST be restored before any release. "
            "See tests/ci/test_release_gate_integrity.py docstring for background."
        )

    def test_pip_install_check_is_in_summary_needs(self, workflow):
        """The summary job must depend on pip-install-check so failures block releases.

        CRITICAL — DO NOT REMOVE from needs: removing the dependency means the
        release gate passes even when pip install fails.
        """
        summary_job = workflow.get("jobs", {}).get("release-gate-summary", {})
        needs = summary_job.get("needs", [])
        assert "pip-install-check" in needs, (
            "CRITICAL: 'pip-install-check' is missing from release-gate-summary.needs! "
            "The release gate will pass even if pip install fails. "
            "Add it back to the needs list."
        )

    def test_pip_install_check_has_no_continue_on_error(self, workflow):
        """The pip-install-check job must NOT have continue-on-error.

        CRITICAL — DO NOT ADD continue-on-error: it would make pip install
        failures non-blocking, allowing broken wheels to be published.
        """
        job = workflow.get("jobs", {}).get("pip-install-check", {})
        assert job.get("continue-on-error") is not True, (
            "CRITICAL: pip-install-check has continue-on-error: true! "
            "This makes dependency failures non-blocking — broken wheels "
            "will be published to PyPI. Remove continue-on-error."
        )

    def test_pip_install_check_uses_pip_not_pdm(self, workflow):
        """The verification step must use pip, not pdm.

        CRITICAL — DO NOT CHANGE to pdm install: PDM resolves overrides
        differently and will NOT catch the dependency conflicts that break
        pip users. The whole point is to test what pip users experience.
        """
        job = workflow.get("jobs", {}).get("pip-install-check", {})
        steps = job.get("steps", [])

        # Find the verification step
        verify_step = None
        for step in steps:
            run = step.get("run", "")
            if "pip install" in run and "no-cache-dir" in run:
                verify_step = step
                break

        assert verify_step is not None, (
            "CRITICAL: Cannot find the pip install verification step in pip-install-check! "
            "The step must run 'pip install --no-cache-dir' on the built wheel."
        )

        # Make sure it doesn't use pdm install
        run_content = verify_step.get("run", "")
        assert "pdm install" not in run_content, (
            "CRITICAL: Verification step uses 'pdm install' instead of 'pip install'! "
            "PDM handles overrides differently and won't catch dependency conflicts. "
            "Use 'pip install' to simulate what end users experience."
        )

    def test_pip_install_check_does_not_use_no_deps(self, workflow):
        """The pip install must NOT use --no-deps.

        CRITICAL — DO NOT ADD --no-deps: it skips dependency resolution
        entirely, which is the exact thing we're trying to verify.
        """
        job = workflow.get("jobs", {}).get("pip-install-check", {})
        steps = job.get("steps", [])

        for step in steps:
            run = step.get("run", "")
            if "pip install" in run and ".whl" in run:
                # Check actual pip install commands, not comments
                for line in run.splitlines():
                    stripped = line.strip()
                    if stripped.startswith("#"):
                        continue
                    if "pip install" in stripped and "--no-deps" in stripped:
                        raise AssertionError(
                            "CRITICAL: pip install command uses --no-deps! "
                            "This skips dependency resolution — the exact check we need. "
                            "Remove --no-deps."
                        )

    def test_pip_install_check_runs_pip_check(self, workflow):
        """The verification must run `pip check` to catch inconsistencies.

        CRITICAL — DO NOT REMOVE: pip check catches dependency version
        mismatches that pip install might not flag during resolution.
        """
        job = workflow.get("jobs", {}).get("pip-install-check", {})
        steps = job.get("steps", [])

        has_pip_check = False
        for step in steps:
            if "pip check" in step.get("run", ""):
                has_pip_check = True
                break

        assert has_pip_check, (
            "CRITICAL: pip-install-check job does not run 'pip check'! "
            "This catches dependency inconsistencies after installation. "
            "Add 'pip check' to the verification step."
        )


AUDIT_SCRIPT = ".github/scripts/audit_installed_dependencies.py"

# Every audit invocation must keep these bindings: --python pins the scan
# to the interpreter that resolved the dependencies, --scanner to the
# hash-verified osv-scanner binary, --manifest to the recorded release
# package, --policy to the reviewed osv-scanner.toml and --output to the
# evidence directory the retention step uploads. Dropping any one of
# them silently changes what gets audited.
AUDIT_BINDING_ARGS = (
    "--python",
    "--scanner",
    "--manifest",
    "--policy",
    "--output",
)


def _audit_invocations(steps):
    """Return each audit script call in ``steps`` as one joined command."""
    calls = []
    for run in (step.get("run", "") for step in steps):
        lines = run.splitlines()
        for start, line in enumerate(lines):
            if AUDIT_SCRIPT not in line:
                continue
            end = start
            while end < len(lines) and lines[end].rstrip().endswith("\\"):
                end += 1
            calls.append(
                " ".join(
                    part.strip().rstrip("\\").strip()
                    for part in lines[start : end + 1]
                )
            )
    return calls


def _assert_audit_bindings(calls):
    """Fail unless every audit call keeps the arguments that bind the scan."""
    for call in calls:
        for argument in AUDIT_BINDING_ARGS:
            assert re.search(rf"{argument}\s+\S", call), (
                f"CRITICAL: installed-dependency audit invocation lost its "
                f"{argument} binding: {call}"
            )


class TestInstalledDependencyAuditWiring:
    """
    CRITICAL — DO NOT DELETE OR WEAKEN THESE TESTS.

    The installed-dependency audit is what makes the exact versions pip
    resolves a blocking part of the release gate: vulnerabilities,
    collection failures and incomplete scans fail the package jobs. The
    base and [mcp] audits live inside the shared pip-verification run
    block with no step names of their own, so deleting the invocations
    would otherwise leave every test green. These tests make that
    removal loud, the same way the pip-resolution tests above guard
    their job.
    """

    def test_pip_install_check_audits_base_and_mcp_installs(self, workflow):
        """Both the base wheel install and the [mcp] extra install must be audited.

        CRITICAL — DO NOT REMOVE the invocations: releases would ship with
        the resolved dependency set unaudited.
        """
        job = workflow.get("jobs", {}).get("pip-install-check", {})
        calls = _audit_invocations(job.get("steps", []))

        assert len(calls) == 2, (
            f"CRITICAL: pip-install-check must invoke {AUDIT_SCRIPT} exactly "
            "twice — after the base wheel install and again after the [mcp] "
            f"extra install. Found {len(calls)}."
        )
        _assert_audit_bindings(calls)

    def test_package_install_matrix_audits_each_resolution(self, workflow):
        """Every matrix leg must audit the dependencies it just resolved.

        CRITICAL — DO NOT REMOVE the audit step: wheel and sdist installs
        would pass the gate without their resolved dependencies being
        scanned.
        """
        job = workflow.get("jobs", {}).get("package-install-matrix", {})
        calls = _audit_invocations(job.get("steps", []))

        assert len(calls) == 1, (
            f"CRITICAL: package-install-matrix must invoke {AUDIT_SCRIPT} "
            f"once per matrix leg. Found {len(calls)}."
        )
        _assert_audit_bindings(calls)

    @pytest.mark.parametrize(
        "job_id", ["pip-install-check", "package-install-matrix"]
    )
    def test_audit_evidence_is_retained_from_blocked_jobs(
        self, workflow, job_id
    ):
        """Audit evidence must upload even when the audit fails the job.

        CRITICAL — DO NOT REMOVE the retention step: the audit is
        fail-closed, so without if: always() its evidence disappears
        exactly when it blocks a release.
        """
        steps = workflow.get("jobs", {}).get(job_id, {}).get("steps", [])
        uploads = [
            step
            for step in steps
            if step.get("name") == "Retain installed dependency audit evidence"
        ]

        assert len(uploads) == 1, (
            f"CRITICAL: {job_id} must have exactly one 'Retain installed "
            f"dependency audit evidence' step. Found {len(uploads)}."
        )
        upload = uploads[0]
        assert str(upload.get("uses", "")).startswith(
            "actions/upload-artifact@"
        ), (
            "CRITICAL: audit evidence must be uploaded with actions/upload-artifact."
        )
        assert upload.get("if") == "always()", (
            "CRITICAL: the audit evidence upload needs if: always() — "
            "otherwise the evidence is lost whenever the audit blocks the job."
        )
        assert "installed-dependency-audit" in (upload.get("with") or {}).get(
            "path", ""
        ), (
            "CRITICAL: the evidence upload must target the audit output "
            "directory (installed-dependency-audit)."
        )