Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

8.2 KB
Raw Blame
"""Exercise installed-environment audits without installing the application."""

# allow: no-sut-import — imports only the standalone CI auditor.

import importlib.util
import json
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
from unittest.mock import patch
import venv


ROOT = Path(__file__).resolve().parents[3]
SPEC = importlib.util.spec_from_file_location(
    "installed_audit", ROOT / ".github/scripts/audit_installed_dependencies.py"
)
AUDIT = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(AUDIT)


def collected(packages=None, **changes):
    return {
        "isolated_environment": True,
        "python_version": "3.12.14",
        "packages": packages
        if packages is not None
        else [
            {"name": "local_deep_research", "version": "2.0.0"},
            {"name": "example-dependency", "version": "1.2.3+cpu"},
        ],
        **changes,
    }


class InventoryTests(unittest.TestCase):
    def collect(self, data):
        with patch.object(AUDIT.subprocess, "run") as run:
            run.return_value.stdout = json.dumps(data)
            result = AUDIT.inventory(Path("venv/bin/python"), "2.0.0")
            self.assertEqual(run.call_args.args[0][1:3], ["-I", "-c"])
            return result

    def test_only_installed_name_version_metadata_is_retained(self):
        data = collected()
        data["private_index"] = "synthetic-private-index"
        data["packages"][1]["direct_url"] = "synthetic-private-url"
        result = self.collect(data)
        self.assertEqual(
            result["packages"],
            [
                {"name": "example-dependency", "version": "1.2.3+cpu"},
                {"name": "local-deep-research", "version": "2.0.0"},
            ],
        )
        self.assertNotIn("synthetic-private", json.dumps(result))

    def test_bootstrap_installer_remains_in_the_audited_inventory(self):
        data = collected()
        data["packages"].append({"name": "pip", "version": "26.2.1"})
        result = self.collect(data)
        self.assertIn({"name": "pip", "version": "26.2.1"}, result["packages"])

    def test_missing_or_wrong_project_version_fails(self):
        for packages in (
            [{"name": "other", "version": "2.0.0"}],
            [{"name": "local-deep-research", "version": "1.0.0"}],
        ):
            with (
                self.subTest(packages=packages),
                self.assertRaisesRegex(ValueError, "manifest"),
            ):
                self.collect(collected(packages))

    def test_empty_or_nonisolated_inventory_fails(self):
        for data in (collected([]), collected(isolated_environment=False), {}):
            with self.subTest(data=data), self.assertRaises(ValueError):
                self.collect(data)

    def test_ambiguous_distribution_names_fail(self):
        data = collected()
        data["packages"].append(
            {"name": "local.deep.research", "version": "2.0.0"}
        )
        with self.assertRaisesRegex(ValueError, "duplicated"):
            self.collect(data)

    def test_requirement_injection_and_missing_metadata_fail(self):
        for package in (
            {"name": "dependency\n--index-url=bad", "version": "1"},
            {"name": "dependency", "version": "1\nother==2"},
            {"name": "dependency", "version": "https://example.invalid/wheel"},
            {"name": None, "version": "1"},
            {"name": "dependency"},
        ):
            with self.subTest(package=package), self.assertRaises(ValueError):
                self.collect(collected([*collected()["packages"], package]))


class AuditTests(unittest.TestCase):
    def setUp(self):
        self.directory = tempfile.TemporaryDirectory()
        self.addCleanup(self.directory.cleanup)
        self.root = Path(self.directory.name)
        self.manifest = self.root / "release-package.json"
        self.manifest.write_text('{"version":"2.0.0"}\n', encoding="utf-8")
        self.policy = self.root / "osv-scanner.toml"
        self.policy.write_text(
            "# reviewed synthetic policy\n", encoding="utf-8"
        )
        self.output = self.root / "audit"

    def run_audit(self, status=0):
        def scan(scanner, output, *, release_policy=False):
            self.assertEqual(
                (output / "osv-scanner.toml").read_bytes(),
                self.policy.read_bytes(),
            )
            self.assertEqual(
                (output / "requirements.txt")
                .read_text(encoding="utf-8")
                .splitlines()[1:],
                [
                    "example-dependency==1.2.3+cpu",
                    "local-deep-research==2.0.0",
                ],
            )
            return status

        with (
            patch.object(
                AUDIT,
                "inventory",
                return_value={
                    "schema": 1,
                    "python_version": "3.12.14",
                    "project_version": "2.0.0",
                    "packages": [
                        {"name": "example-dependency", "version": "1.2.3+cpu"},
                        {"name": "local-deep-research", "version": "2.0.0"},
                    ],
                },
            ),
            patch.object(AUDIT, "scan", side_effect=scan),
        ):
            return AUDIT.audit(
                Path("python"),
                Path("scanner"),
                self.manifest,
                self.policy,
                self.output,
            )

    def test_inventory_is_pinned_and_uses_reviewed_policy(self):
        self.assertEqual(self.run_audit(), 0)
        self.assertTrue((self.output / "inventory.json").is_file())

    def test_vulnerability_status_still_blocks(self):
        self.assertEqual(self.run_audit(status=1), 1)

    def test_missing_policy_fails_before_scanning(self):
        self.policy.unlink()
        with (
            patch.object(AUDIT, "inventory") as collect,
            self.assertRaisesRegex(ValueError, "policy"),
        ):
            AUDIT.audit(
                Path("python"),
                Path("scanner"),
                self.manifest,
                self.policy,
                self.output,
            )
        collect.assert_not_called()

    def test_real_interpreter_metadata_and_native_scan_contract(self):
        environment = self.root / "venv"
        venv.EnvBuilder(with_pip=False).create(environment)
        python = environment / "bin/python"
        purelib = Path(
            subprocess.check_output(
                [
                    str(python),
                    "-I",
                    "-c",
                    "import sysconfig; print(sysconfig.get_path('purelib'))",
                ],
                text=True,
            ).strip()
        )
        for name, version in (
            ("local_deep_research", "2.0.0"),
            ("fixture_dependency", "1.2.3"),
        ):
            metadata = purelib / f"{name}-{version}.dist-info"
            metadata.mkdir()
            (metadata / "METADATA").write_text(
                f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n",
                encoding="utf-8",
            )
        scanner = self.root / "osv-scanner"
        scanner.write_text(
            f"#!{sys.executable}\n"
            "import json, pathlib, sys\n"
            "assert '--no-resolve' in sys.argv\n"
            "path = next(x.split('=', 1)[1] for x in sys.argv if x.startswith('--output-file='))\n"
            "assert 'fixture-dependency==1.2.3' in pathlib.Path('requirements.txt').read_text(encoding='utf-8')\n"
            "report = {'version':'2.1.0', 'runs':[{'tool':{'driver':{'name':'osv-scanner'}}, 'results':[]}]}\n"
            "pathlib.Path(path).write_text(json.dumps(report), encoding='utf-8')\n",
            encoding="utf-8",
        )
        scanner.chmod(0o700)
        self.assertEqual(
            AUDIT.audit(
                python, scanner, self.manifest, self.policy, self.output
            ),
            0,
        )
        data = json.loads(
            (self.output / "inventory.json").read_text(encoding="utf-8")
        )
        self.assertEqual(
            {row["name"] for row in data["packages"]},
            {"local-deep-research", "fixture-dependency"},
        )
        self.assertTrue((self.output / "results.sarif").is_file())


if __name__ == "__main__":
    unittest.main()