Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

10.7 KB
Raw Blame
"""Exercise the real security fixture in small, isolated pytest processes."""

import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import textwrap
import unittest

import pytest


ROOT = Path(__file__).resolve().parents[3]
# Only the interference probe starts two lightweight workers. Keep these
# subprocess checks out of the outer CI suite's parallel worker pool.
pytestmark = pytest.mark.serial


class SecurityDataDirectoryGuardTests(unittest.TestCase):
    def setUp(self):
        self.temporary = tempfile.TemporaryDirectory()
        self.addCleanup(self.temporary.cleanup)
        self.project = Path(self.temporary.name)
        self.shared_default = self.project / "shared-default"
        self.project.joinpath("pytest.ini").write_text("[pytest]\n")
        support = (
            "from pathlib import Path\n"
            "import importlib.util\n"
            "import time\n"
            f"PROJECT = Path({str(self.project)!r})\n"
            f"SHARED_DEFAULT = Path({str(self.shared_default)!r})\n"
            f"PATHS_FILE = Path({str(ROOT / 'src/local_deep_research/config/paths.py')!r})\n"
            + textwrap.dedent(
                """
                PROVIDER_CALLS = []

                def controlled_user_data_dir(appname=None, *args, **kwargs):
                    PROVIDER_CALLS.append((appname, args, kwargs))
                    if appname == "local-deep-research":
                        return str(SHARED_DEFAULT)
                    return "other-provider-result"

                def paths():
                    spec = importlib.util.spec_from_file_location(
                        "guard_probe_paths", PATHS_FILE
                    )
                    module = importlib.util.module_from_spec(spec)
                    spec.loader.exec_module(module)
                    return module

                def wait_for(name):
                    deadline = time.monotonic() + 15
                    while not PROJECT.joinpath(name).exists():
                        if time.monotonic() >= deadline:
                            raise AssertionError("peer did not reach " + name)
                        time.sleep(0.01)
                """
            )
        )
        self.project.joinpath("probe_support.py").write_text(support)
        self.project.joinpath("conftest.py").write_text(
            "import platformdirs\n"
            "from probe_support import controlled_user_data_dir\n"
            "platformdirs.user_data_dir = controlled_user_data_dir\n"
            'pytest_plugins = ["tests.security.conftest"]\n'
        )

    def run_probe(self, guarded, *, peer=None):
        self.project.joinpath("test_raw_config_toml_smuggling.py").write_text(
            textwrap.dedent(guarded)
        )
        command = [
            sys.executable,
            "-m",
            "pytest",
            "-q",
            "--tb=short",
            "--color=no",
            "--confcutdir",
            str(self.project),
            "--basetemp",
            str(self.project / "pytest-temp"),
            "-o",
            "addopts=",
        ]
        if peer is not None:
            self.project.joinpath("test_peer.py").write_text(
                textwrap.dedent(peer)
            )
            command += ["-p", "xdist.plugin", "-n", "2", "--dist=loadfile"]
        environment = dict(os.environ)
        for name in (
            "LDR_DATA_DIR",
            "PYTEST_ADDOPTS",
            "PYTEST_PLUGINS",
            "PYTEST_CURRENT_TEST",
            "PYTEST_XDIST_WORKER",
            "PYTEST_XDIST_WORKER_COUNT",
            "PYTEST_XDIST_TESTRUNUID",
        ):
            environment.pop(name, None)
        environment.update(
            PYTHONPATH=os.pathsep.join((str(ROOT), str(ROOT / "src"))),
            PYTEST_DISABLE_PLUGIN_AUTOLOAD="1",
            OMP_NUM_THREADS="1",
            OPENBLAS_NUM_THREADS="1",
            MKL_NUM_THREADS="1",
        )
        return subprocess.run(
            command,
            cwd=self.project,
            env=environment,
            text=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
            timeout=45,
            check=False,
        )

    def assert_guard_rejects(self, result):
        self.assertEqual(result.returncode, 1, result.stdout)
        self.assertIn("1 passed", result.stdout)
        self.assertIn("1 error", result.stdout)
        self.assertIn(
            "File-write-risk battery test leaked a write", result.stdout
        )

    def test_another_worker_default_write_is_not_attributed_to_guarded_test(
        self,
    ):
        result = self.run_probe(
            """
            from probe_support import PROJECT, wait_for

            def test_guarded():
                PROJECT.joinpath("ready").touch()
                wait_for("written")
            """,
            peer="""
            import json
            from probe_support import PROJECT, paths, wait_for

            def test_peer():
                wait_for("ready")
                directory = paths().get_library_directory()
                assert directory.is_dir()
                PROJECT.joinpath("peer-path.json").write_text(
                    json.dumps(str(directory))
                )
                PROJECT.joinpath("written").touch()
            """,
        )
        self.assertEqual(result.returncode, 0, result.stdout)
        self.assertIn("2 passed", result.stdout)
        peer_path = Path(
            json.loads(self.project.joinpath("peer-path.json").read_text())
        )
        self.assertTrue(peer_path.is_dir())
        self.assertTrue(peer_path.is_relative_to(self.project / "pytest-temp"))
        self.assertFalse(self.shared_default.exists())

    def test_guard_still_rejects_own_default_directory_creation(self):
        result = self.run_probe(
            """
            from probe_support import paths

            def test_guarded():
                assert paths().get_library_directory().is_dir()
            """
        )
        self.assert_guard_rejects(result)

    def test_guard_still_rejects_own_default_file_creation(self):
        result = self.run_probe(
            """
            from probe_support import paths

            def test_guarded():
                directory = paths().get_data_directory()
                directory.mkdir(parents=True, exist_ok=True)
                directory.joinpath("example.txt").write_text("fixture data")
            """
        )
        self.assert_guard_rejects(result)

    def test_guard_still_rejects_native_sqlite_file_creation(self):
        result = self.run_probe(
            """
            import sqlite3
            from probe_support import paths

            def test_guarded():
                directory = paths().get_data_directory()
                directory.mkdir(parents=True, exist_ok=True)
                connection = sqlite3.connect(directory / "example.db")
                connection.close()
            """
        )
        self.assert_guard_rejects(result)

    def test_explicit_test_data_directory_remains_usable(self):
        result = self.run_probe(
            """
            from probe_support import paths

            def test_guarded(tmp_path, monkeypatch):
                monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
                assert paths().get_library_directory() == tmp_path / "library"
                assert tmp_path.joinpath("library").is_dir()
            """
        )
        self.assertEqual(result.returncode, 0, result.stdout)
        self.assertFalse(self.shared_default.exists())

    def test_other_platformdirs_applications_keep_their_provider(self):
        result = self.run_probe(
            """
            import platformdirs
            from probe_support import PROVIDER_CALLS

            def test_guarded():
                value = platformdirs.user_data_dir(
                    "other-tool", appauthor=False, version="v2", roaming=True
                )
                assert value == "other-provider-result"
                assert PROVIDER_CALLS[-1] == (
                    "other-tool", (),
                    {"appauthor": False, "version": "v2", "roaming": True},
                )
            """
        )
        self.assertEqual(result.returncode, 0, result.stdout)

    def test_default_path_is_private_to_the_guarded_test(self):
        result = self.run_probe(
            """
            import json
            from probe_support import PROJECT, paths

            def test_guarded():
                directory = paths().get_data_directory()
                PROJECT.joinpath("default.json").write_text(
                    json.dumps(str(directory))
                )
            """
        )
        self.assertEqual(result.returncode, 0, result.stdout)
        directory = Path(
            json.loads(self.project.joinpath("default.json").read_text())
        )
        self.assertNotEqual(directory, self.shared_default)
        self.assertTrue(directory.is_relative_to(self.project / "pytest-temp"))
        self.assertFalse(self.shared_default.exists())

    def test_successive_tests_receive_distinct_default_paths(self):
        result = self.run_probe(
            """
            import json
            from probe_support import PROJECT, paths

            def test_first():
                PROJECT.joinpath("first.json").write_text(
                    json.dumps(str(paths().get_data_directory()))
                )

            def test_second():
                PROJECT.joinpath("second.json").write_text(
                    json.dumps(str(paths().get_data_directory()))
                )
            """
        )
        self.assertEqual(result.returncode, 0, result.stdout)
        first = json.loads(self.project.joinpath("first.json").read_text())
        second = json.loads(self.project.joinpath("second.json").read_text())
        self.assertNotEqual(first, second)
        self.assertFalse(self.shared_default.exists())

    def test_unaudited_tests_also_have_a_private_worker_default(self):
        self.project.joinpath("test_other.py").write_text(
            "import json\n"
            "from probe_support import PROJECT, paths\n"
            "def test_other():\n"
            "    directory = paths().get_library_directory()\n"
            "    PROJECT.joinpath('other.json').write_text(\n"
            "        json.dumps(str(directory)))\n"
        )
        result = self.run_probe("def test_guarded(): pass\n")
        self.assertEqual(result.returncode, 0, result.stdout)
        directory = Path(
            json.loads(self.project.joinpath("other.json").read_text())
        )
        self.assertTrue(directory.is_dir())
        self.assertTrue(directory.is_relative_to(self.project / "pytest-temp"))
        self.assertFalse(self.shared_default.exists())


if __name__ == "__main__":
    unittest.main()