Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

ci: audit dependencies resolved by release package installs (#7622)

LearningCircuit541fc0120e36
diff --git a/.github/scripts/audit_installed_dependencies.py b/.github/scripts/audit_installed_dependencies.py
new file mode 100644
index 0000000000..005dc03c01
--- /dev/null
+++ b/.github/scripts/audit_installed_dependencies.py
@@ -0,0 +1,151 @@
+"""Audit the exact distributions in a clean release-install environment."""
+
+import argparse
+import importlib.util
+import json
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import sys
+
+
+# Isolated mode excludes the checkout and user site. Reading distribution
+# metadata imports no application modules and discloses no direct_url values.
+INVENTORY_PROGRAM = """
+import importlib.metadata as metadata
+import json
+import sys
+print(json.dumps({
+ "isolated_environment": sys.prefix != sys.base_prefix,
+ "python_version": ".".join(map(str, sys.version_info[:3])),
+ "packages": [
+ {"name": distribution.metadata["Name"], "version": distribution.version}
+ for distribution in metadata.distributions()
+ ],
+}))
+"""
+
+
+def inventory(python, expected_version):
+ result = subprocess.run(
+ [str(python), "-I", "-c", INVENTORY_PROGRAM],
+ capture_output=True,
+ text=True,
+ check=True,
+ timeout=30,
+ )
+ data = json.loads(result.stdout)
+ if (
+ not isinstance(data, dict)
+ or data.get("isolated_environment") is not True
+ ):
+ raise ValueError("Expected a clean virtual environment to audit")
+ packages = data.get("packages")
+ if not isinstance(packages, list) or not packages:
+ raise ValueError("Installed dependency inventory is empty or malformed")
+ by_name = {}
+ for package in packages:
+ if not isinstance(package, dict):
+ raise ValueError("Installed distribution metadata is malformed")
+ name, version = package.get("name"), package.get("version")
+ if not isinstance(name, str) or not re.fullmatch(
+ r"[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?", name
+ ):
+ raise ValueError("Installed distribution has an invalid name")
+ if not isinstance(version, str) or not re.fullmatch(
+ r"[A-Za-z0-9][A-Za-z0-9.!+_-]*", version
+ ):
+ raise ValueError("Installed distribution has an invalid version")
+ normalized = re.sub(r"[-_.]+", "-", name).lower()
+ if normalized in by_name:
+ raise ValueError("Installed distribution names are duplicated")
+ by_name[normalized] = version
+ if by_name.get("local-deep-research") != expected_version:
+ raise ValueError(
+ "Installed project does not match the tested package manifest"
+ )
+ if not isinstance(data.get("python_version"), str) or not re.fullmatch(
+ r"[0-9]+\.[0-9]+\.[0-9]+", data["python_version"]
+ ):
+ raise ValueError("Installed inventory has no recognized Python version")
+ return {
+ "schema": 1,
+ "python_version": data["python_version"],
+ "project_version": expected_version,
+ "packages": [
+ {"name": name, "version": version}
+ for name, version in sorted(by_name.items())
+ ],
+ }
+
+
+def scan(scanner, output):
+ # Reuse the repository's native exit-status, diagnostic and report policy.
+ path = Path(__file__).with_name("run_osv_scan.py")
+ spec = importlib.util.spec_from_file_location("installed_osv_policy", path)
+ policy = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(policy)
+ return policy.scan(scanner, output, output / "results.sarif", resolve=False)
+
+
+def audit(python, scanner, manifest, policy, output):
+ expected_version = json.loads(manifest.read_text(encoding="utf-8"))[
+ "version"
+ ]
+ if not isinstance(expected_version, str) or not expected_version:
+ raise ValueError("The tested package manifest has no version")
+ if not policy.is_file():
+ raise ValueError("The reviewed OSV exception policy is missing")
+ data = inventory(python, expected_version)
+ output = output.resolve()
+ output.mkdir(parents=True, exist_ok=True)
+ (output / "inventory.json").write_text(
+ json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+ )
+ (output / "requirements.txt").write_text(
+ "# Exact installed distributions; no dependency resolution is performed.\n"
+ + "".join(
+ f"{row['name']}=={row['version']}\n" for row in data["packages"]
+ ),
+ encoding="utf-8",
+ )
+ # Native OSV discovers this policy beside the generated requirements.
+ # Keep the existing narrow, expiring exceptions; add none automatically.
+ shutil.copyfile(policy, output / "osv-scanner.toml")
+ status = scan(scanner, output)
+ print(
+ f"Audited {len(data['packages'])} installed distributions "
+ f"on Python {data['python_version']} (scanner status {status})"
+ )
+ return status
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--python", required=True, type=Path)
+ parser.add_argument("--scanner", required=True, type=Path)
+ parser.add_argument("--manifest", required=True, type=Path)
+ parser.add_argument("--policy", required=True, type=Path)
+ parser.add_argument("--output", required=True, type=Path)
+ args = parser.parse_args()
+ try:
+ return audit(
+ args.python, args.scanner, args.manifest, args.policy, args.output
+ )
+ except (
+ OSError,
+ ValueError,
+ KeyError,
+ TypeError,
+ subprocess.SubprocessError,
+ ):
+ # Parser/collector output may contain private index or filesystem data.
+ print(
+ "::error::Installed dependency audit could not produce complete evidence"
+ )
+ return 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/.github/scripts/run_osv_scan.py b/.github/scripts/run_osv_scan.py
index 5a700052b4..ff1ecfcd75 100644
--- a/.github/scripts/run_osv_scan.py
+++ b/.github/scripts/run_osv_scan.py
@@ -128,7 +128,9 @@ def _printable(text: str) -> str:
return re.sub(r"[^A-Za-z0-9._/@+-]", "?", text)
-def run_scanner(scanner: Path, root: Path, pending: Path) -> tuple[int, int]:
+def run_scanner(
+ scanner: Path, root: Path, pending: Path, *, resolve: bool = True
+) -> tuple[int, int]:
"""Run one complete scan of ``root`` into ``pending``.
Returns the scanner's status and the number of findings.
@@ -143,6 +145,9 @@ def run_scanner(scanner: Path, root: Path, pending: Path) -> tuple[int, int]:
"-r",
"./",
]
+ if not resolve:
+ # An installed inventory is already resolved, including unpublished wheels.
+ command.insert(3, "--no-resolve")
# Scanner logs include repository-controlled package names and paths.
token = secrets.token_hex(16)
print(f"::stop-commands::{token}", flush=True)
@@ -177,7 +182,12 @@ def run_scanner(scanner: Path, root: Path, pending: Path) -> tuple[int, int]:
def scan(
- scanner: Path, root: Path, output: Path, baseline: Path | None = None
+ scanner: Path,
+ root: Path,
+ output: Path,
+ baseline: Path | None = None,
+ *,
+ resolve: bool = True,
) -> int:
# An earlier invocation or a checked-in file must never qualify this scan.
output.unlink(missing_ok=True)
@@ -185,7 +195,9 @@ def scan(
prefix=".osv-scan-", dir=output.parent
) as tmp:
pending = Path(tmp) / "results.sarif"
- returncode, findings = run_scanner(scanner, root, pending)
+ returncode, findings = run_scanner(
+ scanner, root, pending, resolve=resolve
+ )
keys = finding_keys(pending, root)
pending.replace(output)
github_output = os.environ.get("GITHUB_OUTPUT")
@@ -195,7 +207,9 @@ def scan(
print(f"OSV scan completed: {findings} SARIF findings", flush=True)
if returncode == 0 or baseline is None:
return returncode
- return compare_with_baseline(scanner, baseline, output.parent, keys)
+ return compare_with_baseline(
+ scanner, baseline, output.parent, keys, resolve=resolve
+ )
def compare_with_baseline(
@@ -203,6 +217,8 @@ def compare_with_baseline(
baseline: Path,
workdir: Path,
keys: set[tuple[str, str, str]],
+ *,
+ resolve: bool = True,
) -> int:
"""Fail only on findings absent from the base commit's own scan."""
with tempfile.TemporaryDirectory(
@@ -210,7 +226,7 @@ def compare_with_baseline(
) as tmp:
pending = Path(tmp) / "results.sarif"
try:
- run_scanner(scanner, baseline, pending)
+ run_scanner(scanner, baseline, pending, resolve=resolve)
base_keys = finding_keys(pending, baseline)
except (OSError, ScanError, ValueError) as exc:
# Without a complete baseline nothing can be called pre-existing.
diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml
index d3806a92a7..15a71af3db 100644
--- a/.github/workflows/release-gate.yml
+++ b/.github/workflows/release-gate.yml
@@ -428,6 +428,18 @@ jobs:
- name: Enforce package artifact policy
run: python .github/scripts/package_artifact_policy.py dist/
+ - name: Install verified OSV-Scanner for installed dependency audits
+ env:
+ OSV_VERSION: '2.6.0'
+ # Official v2.6.0 osv-scanner_SHA256SUMS, Linux amd64 binary.
+ OSV_SHA256: ca69b3d3cd08f889a49dc0a383122f71cc528b83803671df5fd874d97485b108
+ run: |
+ curl --fail --silent --show-error --location --retry 3 --proto '=https' \
+ --output "$RUNNER_TEMP/osv-scanner" \
+ "https://github.com/google/osv-scanner/releases/download/v${OSV_VERSION}/osv-scanner_linux_amd64"
+ printf '%s %s\n' "$OSV_SHA256" "$RUNNER_TEMP/osv-scanner" | sha256sum --check --strict
+ chmod +x "$RUNNER_TEMP/osv-scanner"
+
- name: Verify pip install in clean environment
run: |
# ============================================================
@@ -477,6 +489,14 @@ jobs:
echo "=== Verifying dependency tree is consistent ==="
pip check
+ echo "=== Auditing exact installed dependency versions ==="
+ python "$GITHUB_WORKSPACE/.github/scripts/audit_installed_dependencies.py" \
+ --python /tmp/pip-verify-env/bin/python \
+ --scanner "$RUNNER_TEMP/osv-scanner" \
+ --manifest "$GITHUB_WORKSPACE/release-package.json" \
+ --policy "$GITHUB_WORKSPACE/osv-scanner.toml" \
+ --output "$RUNNER_TEMP/installed-dependency-audit/base"
+
echo ""
echo "=== twine check (PyPI metadata + long_description rendering) ==="
# Twine validates that the wheel's PKG-INFO is well-formed and the
@@ -499,6 +519,13 @@ jobs:
# base-install verification while letting the entry-point smoke
# exercise both scripts.
pip install --quiet --no-cache-dir "${WHEEL}[mcp]"
+ pip check
+ python "$GITHUB_WORKSPACE/.github/scripts/audit_installed_dependencies.py" \
+ --python /tmp/pip-verify-env/bin/python \
+ --scanner "$RUNNER_TEMP/osv-scanner" \
+ --manifest "$GITHUB_WORKSPACE/release-package.json" \
+ --policy "$GITHUB_WORKSPACE/osv-scanner.toml" \
+ --output "$RUNNER_TEMP/installed-dependency-audit/mcp"
echo ""
echo "=== CLI entry-point resolution ==="
@@ -536,6 +563,15 @@ jobs:
MANIFEST_SHA256: ${{ steps.package-manifest.outputs.manifest_sha256 }}
run: python .github/scripts/release_package.py verify
+ - name: Retain installed dependency audit evidence
+ if: always()
+ uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+ with:
+ name: installed-dependencies-3.12-wheel-${{ github.run_attempt }}
+ path: ${{ runner.temp }}/installed-dependency-audit/
+ if-no-files-found: warn
+ retention-days: 14
+
- name: Upload tested distributions
id: package-upload
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
@@ -592,6 +628,18 @@ jobs:
run: |
sudo apt-get update
sudo apt-get install -y libsqlcipher-dev
+ - name: Install verified OSV-Scanner for installed dependency audits
+ env:
+ OSV_VERSION: '2.6.0'
+ # Official v2.6.0 osv-scanner_SHA256SUMS, Linux amd64 binary.
+ OSV_SHA256: ca69b3d3cd08f889a49dc0a383122f71cc528b83803671df5fd874d97485b108
+ run: |
+ curl --fail --silent --show-error --location --retry 3 --proto '=https' \
+ --output "$RUNNER_TEMP/osv-scanner" \
+ "https://github.com/google/osv-scanner/releases/download/v${OSV_VERSION}/osv-scanner_linux_amd64"
+ printf '%s %s\n' "$OSV_SHA256" "$RUNNER_TEMP/osv-scanner" | sha256sum --check --strict
+ chmod +x "$RUNNER_TEMP/osv-scanner"
+
- name: Install distribution with fresh pip resolution
env:
DISTRIBUTION: ${{ matrix.distribution }}
@@ -615,6 +663,23 @@ jobs:
print(f"Installed v{local_deep_research.__version__} with frontend assets")
PYCODE
+ - name: Audit resolved installed dependencies
+ run: |
+ python .github/scripts/audit_installed_dependencies.py \
+ --python /tmp/package-install-env/bin/python \
+ --scanner "$RUNNER_TEMP/osv-scanner" \
+ --manifest release-package.json \
+ --policy osv-scanner.toml \
+ --output "$RUNNER_TEMP/installed-dependency-audit"
+ - name: Retain installed dependency audit evidence
+ if: always()
+ uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+ with:
+ name: installed-dependencies-${{ matrix.python }}-${{ matrix.distribution }}-${{ github.run_attempt }}
+ path: ${{ runner.temp }}/installed-dependency-audit/
+ if-no-files-found: warn
+ retention-days: 14
+
# ============================================
# CPU baseline smoke test under qemu-emulated CPUs
#
diff --git a/.github/workflows/release-package-tests.yml b/.github/workflows/release-package-tests.yml
index 0b2b83de1b..697196b8b8 100644
--- a/.github/workflows/release-package-tests.yml
+++ b/.github/workflows/release-package-tests.yml
@@ -8,6 +8,10 @@ on:
- ".github/workflows/release.yml"
- ".github/workflows/publish.yml"
- ".github/scripts/release_package.py"
+ - ".github/scripts/audit_installed_dependencies.py"
+ - ".github/scripts/run_osv_scan.py"
+ - "osv-scanner.toml"
+ - "tests/ci/standalone/test_installed_dependency_audit.py"
- "tests/ci/standalone/test_release_package.py"
workflow_dispatch:
@@ -33,3 +37,5 @@ jobs:
python-version: '3.12'
- name: Test CI policies without loading the application
run: python -m unittest discover -s tests/ci/standalone -p 'test_release_package*.py' -v
+ - name: Test installed dependency auditing without application imports
+ run: python -m unittest discover -s tests/ci/standalone -p 'test_installed_dependency_audit.py' -v
diff --git a/changelog.d/+installed-dependency-audit.security.md b/changelog.d/+installed-dependency-audit.security.md
new file mode 100644
index 0000000000..04f29fa485
--- /dev/null
+++ b/changelog.d/+installed-dependency-audit.security.md
@@ -0,0 +1 @@
+Audit the exact dependencies installed by release wheel, source-distribution and MCP-extra checks on supported Python versions.
diff --git a/docs/RELEASE_GUIDE.md b/docs/RELEASE_GUIDE.md
index 679a6a6412..038ffde1d4 100644
--- a/docs/RELEASE_GUIDE.md
+++ b/docs/RELEASE_GUIDE.md
@@ -458,3 +458,19 @@ Useful while iterating on a fragment locally.
- [ ] Release automatically created
- [ ] PyPI/Docker publishing approved
- [ ] `chore: clear changelog fragments for <X.Y.Z>` PR merged
+
+### Installed dependency vulnerability checks
+
+The release gate audits the exact installed distribution versions after each
+clean wheel/sdist installation on Python 3.12–3.14, and again after enabling the
+MCP extra. This complements repository lockfile scanning: pip consumers resolve
+versions from package metadata independently of the PDM lock.
+
+The audit reuses the verified OSV-Scanner binary and the existing
+`osv-scanner.toml` policy. Resolution is disabled for this complete installed
+inventory, so an unpublished candidate is not looked up on PyPI. Vulnerabilities
+and incomplete scans block the package
+job. Each job retains its exact installed name/version inventory, pinned scan
+input and native SARIF as `installed-dependencies-*` artifacts for 14 days.
+Auditing reads distribution metadata without importing application modules or
+installing audit tools into the environment under test.
diff --git a/tests/ci/standalone/test_installed_dependency_audit.py b/tests/ci/standalone/test_installed_dependency_audit.py
new file mode 100644
index 0000000000..97dd05f122
--- /dev/null
+++ b/tests/ci/standalone/test_installed_dependency_audit.py
@@ -0,0 +1,226 @@
+"""Exercise installed-environment audits without installing the application."""
+
+# allow: no-sut-import — imports only the standalone CI auditor.
+
+import importlib.util
+import json
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+import venv
+
+
+ROOT = Path(__file__).resolve().parents[3]
+SPEC = importlib.util.spec_from_file_location(
+ "installed_audit", ROOT / ".github/scripts/audit_installed_dependencies.py"
+)
+AUDIT = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(AUDIT)
+
+
+def collected(packages=None, **changes):
+ return {
+ "isolated_environment": True,
+ "python_version": "3.12.14",
+ "packages": packages
+ if packages is not None
+ else [
+ {"name": "local_deep_research", "version": "2.0.0"},
+ {"name": "example-dependency", "version": "1.2.3+cpu"},
+ ],
+ **changes,
+ }
+
+
+class InventoryTests(unittest.TestCase):
+ def collect(self, data):
+ with patch.object(AUDIT.subprocess, "run") as run:
+ run.return_value.stdout = json.dumps(data)
+ result = AUDIT.inventory(Path("venv/bin/python"), "2.0.0")
+ self.assertEqual(run.call_args.args[0][1:3], ["-I", "-c"])
+ return result
+
+ def test_only_installed_name_version_metadata_is_retained(self):
+ data = collected()
+ data["private_index"] = "synthetic-private-index"
+ data["packages"][1]["direct_url"] = "synthetic-private-url"
+ result = self.collect(data)
+ self.assertEqual(
+ result["packages"],
+ [
+ {"name": "example-dependency", "version": "1.2.3+cpu"},
+ {"name": "local-deep-research", "version": "2.0.0"},
+ ],
+ )
+ self.assertNotIn("synthetic-private", json.dumps(result))
+
+ def test_missing_or_wrong_project_version_fails(self):
+ for packages in (
+ [{"name": "other", "version": "2.0.0"}],
+ [{"name": "local-deep-research", "version": "1.0.0"}],
+ ):
+ with (
+ self.subTest(packages=packages),
+ self.assertRaisesRegex(ValueError, "manifest"),
+ ):
+ self.collect(collected(packages))
+
+ def test_empty_or_nonisolated_inventory_fails(self):
+ for data in (collected([]), collected(isolated_environment=False), {}):
+ with self.subTest(data=data), self.assertRaises(ValueError):
+ self.collect(data)
+
+ def test_ambiguous_distribution_names_fail(self):
+ data = collected()
+ data["packages"].append(
+ {"name": "local.deep.research", "version": "2.0.0"}
+ )
+ with self.assertRaisesRegex(ValueError, "duplicated"):
+ self.collect(data)
+
+ def test_requirement_injection_and_missing_metadata_fail(self):
+ for package in (
+ {"name": "dependency\n--index-url=bad", "version": "1"},
+ {"name": "dependency", "version": "1\nother==2"},
+ {"name": "dependency", "version": "https://example.invalid/wheel"},
+ {"name": None, "version": "1"},
+ {"name": "dependency"},
+ ):
+ with self.subTest(package=package), self.assertRaises(ValueError):
+ self.collect(collected([*collected()["packages"], package]))
+
+
+class AuditTests(unittest.TestCase):
+ def setUp(self):
+ self.directory = tempfile.TemporaryDirectory()
+ self.addCleanup(self.directory.cleanup)
+ self.root = Path(self.directory.name)
+ self.manifest = self.root / "release-package.json"
+ self.manifest.write_text('{"version":"2.0.0"}\n', encoding="utf-8")
+ self.policy = self.root / "osv-scanner.toml"
+ self.policy.write_text(
+ "# reviewed synthetic policy\n", encoding="utf-8"
+ )
+ self.output = self.root / "audit"
+
+ def run_audit(self, status=0):
+ def scan(scanner, output):
+ self.assertEqual(
+ (output / "osv-scanner.toml").read_bytes(),
+ self.policy.read_bytes(),
+ )
+ self.assertEqual(
+ (output / "requirements.txt")
+ .read_text(encoding="utf-8")
+ .splitlines()[1:],
+ [
+ "example-dependency==1.2.3+cpu",
+ "local-deep-research==2.0.0",
+ ],
+ )
+ return status
+
+ with (
+ patch.object(
+ AUDIT,
+ "inventory",
+ return_value={
+ "schema": 1,
+ "python_version": "3.12.14",
+ "project_version": "2.0.0",
+ "packages": [
+ {"name": "example-dependency", "version": "1.2.3+cpu"},
+ {"name": "local-deep-research", "version": "2.0.0"},
+ ],
+ },
+ ),
+ patch.object(AUDIT, "scan", side_effect=scan),
+ ):
+ return AUDIT.audit(
+ Path("python"),
+ Path("scanner"),
+ self.manifest,
+ self.policy,
+ self.output,
+ )
+
+ def test_inventory_is_pinned_and_uses_reviewed_policy(self):
+ self.assertEqual(self.run_audit(), 0)
+ self.assertTrue((self.output / "inventory.json").is_file())
+
+ def test_vulnerability_status_still_blocks(self):
+ self.assertEqual(self.run_audit(status=1), 1)
+
+ def test_missing_policy_fails_before_scanning(self):
+ self.policy.unlink()
+ with (
+ patch.object(AUDIT, "inventory") as collect,
+ self.assertRaisesRegex(ValueError, "policy"),
+ ):
+ AUDIT.audit(
+ Path("python"),
+ Path("scanner"),
+ self.manifest,
+ self.policy,
+ self.output,
+ )
+ collect.assert_not_called()
+
+ def test_real_interpreter_metadata_and_native_scan_contract(self):
+ environment = self.root / "venv"
+ venv.EnvBuilder(with_pip=False).create(environment)
+ python = environment / "bin/python"
+ purelib = Path(
+ subprocess.check_output(
+ [
+ str(python),
+ "-I",
+ "-c",
+ "import sysconfig; print(sysconfig.get_path('purelib'))",
+ ],
+ text=True,
+ ).strip()
+ )
+ for name, version in (
+ ("local_deep_research", "2.0.0"),
+ ("fixture_dependency", "1.2.3"),
+ ):
+ metadata = purelib / f"{name}-{version}.dist-info"
+ metadata.mkdir()
+ (metadata / "METADATA").write_text(
+ f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n",
+ encoding="utf-8",
+ )
+ scanner = self.root / "osv-scanner"
+ scanner.write_text(
+ f"#!{sys.executable}\n"
+ "import json, pathlib, sys\n"
+ "assert '--no-resolve' in sys.argv\n"
+ "path = next(x.split('=', 1)[1] for x in sys.argv if x.startswith('--output-file='))\n"
+ "assert 'fixture-dependency==1.2.3' in pathlib.Path('requirements.txt').read_text(encoding='utf-8')\n"
+ "report = {'version':'2.1.0', 'runs':[{'tool':{'driver':{'name':'osv-scanner'}}, 'results':[]}]}\n"
+ "pathlib.Path(path).write_text(json.dumps(report), encoding='utf-8')\n",
+ encoding="utf-8",
+ )
+ scanner.chmod(0o700)
+ self.assertEqual(
+ AUDIT.audit(
+ python, scanner, self.manifest, self.policy, self.output
+ ),
+ 0,
+ )
+ data = json.loads(
+ (self.output / "inventory.json").read_text(encoding="utf-8")
+ )
+ self.assertEqual(
+ {row["name"] for row in data["packages"]},
+ {"local-deep-research", "fixture-dependency"},
+ )
+ self.assertTrue((self.output / "results.sarif").is_file())
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/ci/test_package_artifact_policy.py b/tests/ci/test_package_artifact_policy.py
index 3d00f8fa57..d301793cf5 100644
--- a/tests/ci/test_package_artifact_policy.py
+++ b/tests/ci/test_package_artifact_policy.py
@@ -489,12 +489,17 @@ def _package_binding_violations(steps):
last_execution = max(executions, default=create)
if not any(i > last_execution for i in verifies):
problems.append("no hash verification after the package smoke tests")
+ # Only this artifact ID is exported to the release/publish chain.
+ # Inventory and scanner diagnostics are independent evidence uploads.
uploads = [
i
for i, step in enumerate(steps)
- if str(step.get("uses", "")).startswith("actions/upload-artifact@")
+ if step.get("id") == "package-upload"
+ and str(step.get("uses", "")).startswith("actions/upload-artifact@")
]
- if not uploads or not verifies or min(uploads) < max(verifies):
+ if len(uploads) != 1:
+ return [*problems, "expected exactly one tested distribution upload"]
+ if not verifies or min(uploads) < max(verifies):
problems.append("distributions are uploaded before re-verification")
elif any(max(verifies) < i < min(uploads) for i in executions):
problems.append("package code runs between verification and upload")
@@ -514,24 +519,52 @@ def _step_index(steps, text):
def test_release_gate_records_hashes_before_running_the_package():
workflow = yaml.safe_load(RELEASE_GATE.read_text(encoding="utf-8"))
- steps = workflow["jobs"]["pip-install-check"]["steps"]
+ job = workflow["jobs"]["pip-install-check"]
+ assert job["outputs"]["artifact_id"] == (
+ "${{ steps.package-upload.outputs.artifact-id }}"
+ )
+ steps = job["steps"]
create = _step_index(steps, "release_package.py create")
assert steps[create]["id"] == "package-manifest"
assert _package_binding_violations(steps) == []
+def test_package_binding_allows_independent_diagnostic_uploads():
+ workflow = yaml.safe_load(RELEASE_GATE.read_text(encoding="utf-8"))
+ steps = workflow["jobs"]["pip-install-check"]["steps"]
+ diagnostic = {
+ "name": "Retain scanner diagnostics",
+ "uses": "actions/upload-artifact@pinned",
+ "if": "always()",
+ "with": {"path": "scanner-evidence/"},
+ }
+ for index in range(len(steps) + 1):
+ with_diagnostics = copy.deepcopy(steps)
+ with_diagnostics.insert(index, diagnostic)
+ assert _package_binding_violations(with_diagnostics) == []
+
+
def _mutants(steps):
"""Yield (label, steps, expected problem) for each rule of the check."""
create = _step_index(steps, "release_package.py create")
install = _step_index(steps, "pip install")
verify = _step_index(steps, "release_package.py verify")
upload = next(
- i
- for i, step in enumerate(steps)
- if str(step.get("uses", "")).startswith("actions/upload-artifact@")
+ i for i, step in enumerate(steps) if step.get("id") == "package-upload"
)
+ missing_upload = [s for i, s in enumerate(steps) if i != upload]
+ yield "missing package upload", missing_upload, "exactly one tested"
+
+ for label, change in (
+ ("renamed package upload", {"id": "different-artifact"}),
+ ("wrong upload action", {"uses": "different/upload@pinned"}),
+ ):
+ wrong_upload = copy.deepcopy(steps)
+ wrong_upload[upload].update(change)
+ yield label, wrong_upload, "exactly one tested"
+
late_create = copy.deepcopy(steps)
late_create.insert(install + 1, late_create.pop(create))
yield "create after install", late_create, "runs package code before"
diff --git a/tests/ci/test_release_gate_integrity.py b/tests/ci/test_release_gate_integrity.py
index 8025ed0b38..8469538e09 100644
--- a/tests/ci/test_release_gate_integrity.py
+++ b/tests/ci/test_release_gate_integrity.py
@@ -33,6 +33,7 @@ DO NOT:
"""
from pathlib import Path
+import re
import yaml
import pytest
@@ -182,3 +183,137 @@ class TestPipInstallCheckJobExists:
"This catches dependency inconsistencies after installation. "
"Add 'pip check' to the verification step."
)
+
+
+AUDIT_SCRIPT = ".github/scripts/audit_installed_dependencies.py"
+
+# Every audit invocation must keep these bindings: --python pins the scan
+# to the interpreter that resolved the dependencies, --scanner to the
+# hash-verified osv-scanner binary, --manifest to the recorded release
+# package, --policy to the reviewed osv-scanner.toml and --output to the
+# evidence directory the retention step uploads. Dropping any one of
+# them silently changes what gets audited.
+AUDIT_BINDING_ARGS = (
+ "--python",
+ "--scanner",
+ "--manifest",
+ "--policy",
+ "--output",
+)
+
+
+def _audit_invocations(steps):
+ """Return each audit script call in ``steps`` as one joined command."""
+ calls = []
+ for run in (step.get("run", "") for step in steps):
+ lines = run.splitlines()
+ for start, line in enumerate(lines):
+ if AUDIT_SCRIPT not in line:
+ continue
+ end = start
+ while end < len(lines) and lines[end].rstrip().endswith("\\"):
+ end += 1
+ calls.append(
+ " ".join(
+ part.strip().rstrip("\\").strip()
+ for part in lines[start : end + 1]
+ )
+ )
+ return calls
+
+
+def _assert_audit_bindings(calls):
+ """Fail unless every audit call keeps the arguments that bind the scan."""
+ for call in calls:
+ for argument in AUDIT_BINDING_ARGS:
+ assert re.search(rf"{argument}\s+\S", call), (
+ f"CRITICAL: installed-dependency audit invocation lost its "
+ f"{argument} binding: {call}"
+ )
+
+
+class TestInstalledDependencyAuditWiring:
+ """
+ CRITICAL — DO NOT DELETE OR WEAKEN THESE TESTS.
+
+ The installed-dependency audit is what makes the exact versions pip
+ resolves a blocking part of the release gate: vulnerabilities,
+ collection failures and incomplete scans fail the package jobs. The
+ base and [mcp] audits live inside the shared pip-verification run
+ block with no step names of their own, so deleting the invocations
+ would otherwise leave every test green. These tests make that
+ removal loud, the same way the pip-resolution tests above guard
+ their job.
+ """
+
+ def test_pip_install_check_audits_base_and_mcp_installs(self, workflow):
+ """Both the base wheel install and the [mcp] extra install must be audited.
+
+ CRITICAL — DO NOT REMOVE the invocations: releases would ship with
+ the resolved dependency set unaudited.
+ """
+ job = workflow.get("jobs", {}).get("pip-install-check", {})
+ calls = _audit_invocations(job.get("steps", []))
+
+ assert len(calls) == 2, (
+ f"CRITICAL: pip-install-check must invoke {AUDIT_SCRIPT} exactly "
+ "twice — after the base wheel install and again after the [mcp] "
+ f"extra install. Found {len(calls)}."
+ )
+ _assert_audit_bindings(calls)
+
+ def test_package_install_matrix_audits_each_resolution(self, workflow):
+ """Every matrix leg must audit the dependencies it just resolved.
+
+ CRITICAL — DO NOT REMOVE the audit step: wheel and sdist installs
+ would pass the gate without their resolved dependencies being
+ scanned.
+ """
+ job = workflow.get("jobs", {}).get("package-install-matrix", {})
+ calls = _audit_invocations(job.get("steps", []))
+
+ assert len(calls) == 1, (
+ f"CRITICAL: package-install-matrix must invoke {AUDIT_SCRIPT} "
+ f"once per matrix leg. Found {len(calls)}."
+ )
+ _assert_audit_bindings(calls)
+
+ @pytest.mark.parametrize(
+ "job_id", ["pip-install-check", "package-install-matrix"]
+ )
+ def test_audit_evidence_is_retained_from_blocked_jobs(
+ self, workflow, job_id
+ ):
+ """Audit evidence must upload even when the audit fails the job.
+
+ CRITICAL — DO NOT REMOVE the retention step: the audit is
+ fail-closed, so without if: always() its evidence disappears
+ exactly when it blocks a release.
+ """
+ steps = workflow.get("jobs", {}).get(job_id, {}).get("steps", [])
+ uploads = [
+ step
+ for step in steps
+ if step.get("name") == "Retain installed dependency audit evidence"
+ ]
+
+ assert len(uploads) == 1, (
+ f"CRITICAL: {job_id} must have exactly one 'Retain installed "
+ f"dependency audit evidence' step. Found {len(uploads)}."
+ )
+ upload = uploads[0]
+ assert str(upload.get("uses", "")).startswith(
+ "actions/upload-artifact@"
+ ), (
+ "CRITICAL: audit evidence must be uploaded with actions/upload-artifact."
+ )
+ assert upload.get("if") == "always()", (
+ "CRITICAL: the audit evidence upload needs if: always() — "
+ "otherwise the evidence is lost whenever the audit blocks the job."
+ )
+ assert "installed-dependency-audit" in (upload.get("with") or {}).get(
+ "path", ""
+ ), (
+ "CRITICAL: the evidence upload must target the audit output "
+ "directory (installed-dependency-audit)."
+ )