~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
CRITICAL — DO NOT MODIFY OR DELETE THIS FILE
These tests verify that the release gate workflow contains the pip install
verification job and all its required wiring. This job prevents publishing
broken wheels to PyPI.
Background (v1.4.0 incident):
PDM resolution overrides (pyproject.toml [tool.pdm.resolution.overrides])
leak into wheel metadata when `pdm build` runs. This creates dependency
constraints that pip cannot resolve, making `pip install local-deep-research`
fail for ALL users. PDM installs work fine because PDM handles overrides
differently, so the problem is invisible during development and CI — it
only surfaces after publishing to PyPI, when it's too late.
Version 1.4.0 shipped with requests>=2.33 in its metadata (from a PDM
override for a CVE fix), but arxiv~=2.4 requires requests~=2.32.0.
pip could not resolve this conflict. Every user who ran
`pip install local-deep-research` got an error.
Why guardian tests:
The pip-install-check job in release-gate.yml is the ONLY thing that
catches this class of bug before it reaches users. If someone removes
it (e.g. to "simplify" the workflow, reduce CI time, or fix an
unrelated failure), the safety net disappears silently. These tests
make that removal loud and obvious.
DO NOT:
- Delete this file
- Mark these tests as xfail or skip
- Weaken the assertions (e.g. checking substring instead of exact job name)
- Move the pip-install-check job to a non-blocking position in the workflow
"""
from pathlib import Path
import re
import yaml
import pytest
RELEASE_GATE = (
Path(__file__).resolve().parents[2]
/ ".github"
/ "workflows"
/ "release-gate.yml"
)
@pytest.fixture(scope="module")
def workflow():
"""Load the release-gate workflow YAML.
CRITICAL — DO NOT MODIFY: this fixture is used by all guardian tests below.
"""
assert RELEASE_GATE.exists(), (
f"release-gate.yml not found at {RELEASE_GATE}. "
"If you moved it, update the path — do NOT delete the tests."
)
return yaml.safe_load(RELEASE_GATE.read_text())
class TestPipInstallCheckJobExists:
"""
CRITICAL — DO NOT DELETE OR WEAKEN THESE TESTS.
They verify that the pip-install-check job exists in the release gate
and cannot be published without catching PDM override leaks.
See module docstring for the full incident report.
"""
def test_pip_install_check_job_is_defined(self, workflow):
"""The pip-install-check job must exist in release-gate.yml.
CRITICAL — DO NOT REMOVE: without this job, broken wheels reach PyPI.
"""
jobs = workflow.get("jobs", {})
assert "pip-install-check" in jobs, (
"CRITICAL: 'pip-install-check' job is missing from release-gate.yml! "
"This job prevents publishing wheels with broken dependencies to PyPI. "
"It was removed — this MUST be restored before any release. "
"See tests/ci/test_release_gate_integrity.py docstring for background."
)
def test_pip_install_check_is_in_summary_needs(self, workflow):
"""The summary job must depend on pip-install-check so failures block releases.
CRITICAL — DO NOT REMOVE from needs: removing the dependency means the
release gate passes even when pip install fails.
"""
summary_job = workflow.get("jobs", {}).get("release-gate-summary", {})
needs = summary_job.get("needs", [])
assert "pip-install-check" in needs, (
"CRITICAL: 'pip-install-check' is missing from release-gate-summary.needs! "
"The release gate will pass even if pip install fails. "
"Add it back to the needs list."
)
def test_pip_install_check_has_no_continue_on_error(self, workflow):
"""The pip-install-check job must NOT have continue-on-error.
CRITICAL — DO NOT ADD continue-on-error: it would make pip install
failures non-blocking, allowing broken wheels to be published.
"""
job = workflow.get("jobs", {}).get("pip-install-check", {})
assert job.get("continue-on-error") is not True, (
"CRITICAL: pip-install-check has continue-on-error: true! "
"This makes dependency failures non-blocking — broken wheels "
"will be published to PyPI. Remove continue-on-error."
)
def test_pip_install_check_uses_pip_not_pdm(self, workflow):
"""The verification step must use pip, not pdm.
CRITICAL — DO NOT CHANGE to pdm install: PDM resolves overrides
differently and will NOT catch the dependency conflicts that break
pip users. The whole point is to test what pip users experience.
"""
job = workflow.get("jobs", {}).get("pip-install-check", {})
steps = job.get("steps", [])
# Find the verification step
verify_step = None
for step in steps:
run = step.get("run", "")
if "pip install" in run and "no-cache-dir" in run:
verify_step = step
break
assert verify_step is not None, (
"CRITICAL: Cannot find the pip install verification step in pip-install-check! "
"The step must run 'pip install --no-cache-dir' on the built wheel."
)
# Make sure it doesn't use pdm install
run_content = verify_step.get("run", "")
assert "pdm install" not in run_content, (
"CRITICAL: Verification step uses 'pdm install' instead of 'pip install'! "
"PDM handles overrides differently and won't catch dependency conflicts. "
"Use 'pip install' to simulate what end users experience."
)
def test_pip_install_check_does_not_use_no_deps(self, workflow):
"""The pip install must NOT use --no-deps.
CRITICAL — DO NOT ADD --no-deps: it skips dependency resolution
entirely, which is the exact thing we're trying to verify.
"""
job = workflow.get("jobs", {}).get("pip-install-check", {})
steps = job.get("steps", [])
for step in steps:
run = step.get("run", "")
if "pip install" in run and ".whl" in run:
# Check actual pip install commands, not comments
for line in run.splitlines():
stripped = line.strip()
if stripped.startswith("#"):
continue
if "pip install" in stripped and "--no-deps" in stripped:
raise AssertionError(
"CRITICAL: pip install command uses --no-deps! "
"This skips dependency resolution — the exact check we need. "
"Remove --no-deps."
)
def test_pip_install_check_runs_pip_check(self, workflow):
"""The verification must run `pip check` to catch inconsistencies.
CRITICAL — DO NOT REMOVE: pip check catches dependency version
mismatches that pip install might not flag during resolution.
"""
job = workflow.get("jobs", {}).get("pip-install-check", {})
steps = job.get("steps", [])
has_pip_check = False
for step in steps:
if "pip check" in step.get("run", ""):
has_pip_check = True
break
assert has_pip_check, (
"CRITICAL: pip-install-check job does not run 'pip check'! "
"This catches dependency inconsistencies after installation. "
"Add 'pip check' to the verification step."
)
AUDIT_SCRIPT = ".github/scripts/audit_installed_dependencies.py"
# Every audit invocation must keep these bindings: --python pins the scan
# to the interpreter that resolved the dependencies, --scanner to the
# hash-verified osv-scanner binary, --manifest to the recorded release
# package, --policy to the reviewed osv-scanner.toml and --output to the
# evidence directory the retention step uploads. Dropping any one of
# them silently changes what gets audited.
AUDIT_BINDING_ARGS = (
"--python",
"--scanner",
"--manifest",
"--policy",
"--output",
)
def _audit_invocations(steps):
"""Return each audit script call in ``steps`` as one joined command."""
calls = []
for run in (step.get("run", "") for step in steps):
lines = run.splitlines()
for start, line in enumerate(lines):
if AUDIT_SCRIPT not in line:
continue
end = start
while end < len(lines) and lines[end].rstrip().endswith("\\"):
end += 1
calls.append(
" ".join(
part.strip().rstrip("\\").strip()
for part in lines[start : end + 1]
)
)
return calls
def _assert_audit_bindings(calls):
"""Fail unless every audit call keeps the arguments that bind the scan."""
for call in calls:
for argument in AUDIT_BINDING_ARGS:
assert re.search(rf"{argument}\s+\S", call), (
f"CRITICAL: installed-dependency audit invocation lost its "
f"{argument} binding: {call}"
)
class TestInstalledDependencyAuditWiring:
"""
CRITICAL — DO NOT DELETE OR WEAKEN THESE TESTS.
The installed-dependency audit is what makes the exact versions pip
resolves a blocking part of the release gate: vulnerabilities,
collection failures and incomplete scans fail the package jobs. The
base and [mcp] audits live inside the shared pip-verification run
block with no step names of their own, so deleting the invocations
would otherwise leave every test green. These tests make that
removal loud, the same way the pip-resolution tests above guard
their job.
"""
def test_pip_install_check_audits_base_and_mcp_installs(self, workflow):
"""Both the base wheel install and the [mcp] extra install must be audited.
CRITICAL — DO NOT REMOVE the invocations: releases would ship with
the resolved dependency set unaudited.
"""
job = workflow.get("jobs", {}).get("pip-install-check", {})
calls = _audit_invocations(job.get("steps", []))
assert len(calls) == 2, (
f"CRITICAL: pip-install-check must invoke {AUDIT_SCRIPT} exactly "
"twice — after the base wheel install and again after the [mcp] "
f"extra install. Found {len(calls)}."
)
_assert_audit_bindings(calls)
def test_package_install_matrix_audits_each_resolution(self, workflow):
"""Every matrix leg must audit the dependencies it just resolved.
CRITICAL — DO NOT REMOVE the audit step: wheel and sdist installs
would pass the gate without their resolved dependencies being
scanned.
"""
job = workflow.get("jobs", {}).get("package-install-matrix", {})
calls = _audit_invocations(job.get("steps", []))
assert len(calls) == 1, (
f"CRITICAL: package-install-matrix must invoke {AUDIT_SCRIPT} "
f"once per matrix leg. Found {len(calls)}."
)
_assert_audit_bindings(calls)
@pytest.mark.parametrize(
"job_id", ["pip-install-check", "package-install-matrix"]
)
def test_audit_evidence_is_retained_from_blocked_jobs(
self, workflow, job_id
):
"""Audit evidence must upload even when the audit fails the job.
CRITICAL — DO NOT REMOVE the retention step: the audit is
fail-closed, so without if: always() its evidence disappears
exactly when it blocks a release.
"""
steps = workflow.get("jobs", {}).get(job_id, {}).get("steps", [])
uploads = [
step
for step in steps
if step.get("name") == "Retain installed dependency audit evidence"
]
assert len(uploads) == 1, (
f"CRITICAL: {job_id} must have exactly one 'Retain installed "
f"dependency audit evidence' step. Found {len(uploads)}."
)
upload = uploads[0]
assert str(upload.get("uses", "")).startswith(
"actions/upload-artifact@"
), (
"CRITICAL: audit evidence must be uploaded with actions/upload-artifact."
)
assert upload.get("if") == "always()", (
"CRITICAL: the audit evidence upload needs if: always() — "
"otherwise the evidence is lost whenever the audit blocks the job."
)
assert "installed-dependency-audit" in (upload.get("with") or {}).get(
"path", ""
), (
"CRITICAL: the evidence upload must target the audit output "
"directory (installed-dependency-audit)."
)