Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

18.6 KB
Raw Blame
"""Security tests for the restricted FAISS docstore unpickler.

``local_deep_research.research_library.services.faiss_safe_load`` replaces a
raw ``pickle`` load (used by LangChain's dangerous ``FAISS.load_local``) with
``_RestrictedFaissUnpickler`` — a subclass that only resolves a tiny
allow-list of (module, name) globals. ``load_index_to_docstore_id`` is the
public entry point the phase-1 legacy migration
(``vector_stores/legacy_cleanup.py: _extract_map``) calls to read a
potentially-tampered on-disk ``.pkl`` BEFORE any login/DB access.

This file proves the anti-RCE guarantee actually bites:

  (a) a crafted pickle whose payload runs code via ``__reduce__`` (calling
      ``os.system`` / ``subprocess.Popen`` directly, not just an arbitrary
      helper) is rejected and — provably, via a sentinel file — never
      executes;
  (b) a pickle using the PERSID/BINPERSID (persistent id) opcode is rejected;
  (c) a class with the SAME name as an allow-listed class
      (``InMemoryDocstore``) but defined in a FOREIGN module is rejected,
      proving the allow-list keys on the (module, name) tuple, not the bare
      name;
  (d) a benign, well-formed ``(docstore, index_to_docstore_id)`` pickle —
      mirroring exactly what ``FAISS.save_local`` writes and what
      ``migrate_legacy_docstores`` reads — is accepted by
      ``load_index_to_docstore_id`` and yields ONLY the position->uuid map,
      with the chunk text/Document objects discarded and no code executed.
"""

import copyreg
import io
import pickle
import pickletools
import subprocess
import sys
import types

import pytest
from langchain_community.docstore.in_memory import InMemoryDocstore
from langchain_core.documents import Document

from local_deep_research.research_library.services.faiss_safe_load import (
    _RestrictedFaissUnpickler,
    load_index_to_docstore_id,
)


# ---------------------------------------------------------------------------
# (a) __reduce__ payloads that invoke os.system / subprocess directly
# ---------------------------------------------------------------------------
class _OsSystemPayload:
    """Pickling this object, when unpickled, calls ``os.system(cmd)``.

    ``__reduce__`` returns ``(callable, args)``; the (C or pure-Python)
    unpickler's REDUCE opcode invokes ``callable(*args)`` verbatim. Using
    ``os.system`` directly (rather than a project helper) proves the guard
    stops a *real* shell-executing builtin, not just an easily-blocked
    look-alike.
    """

    def __init__(self, cmd: str):
        self.cmd = cmd

    def __reduce__(self):
        import os

        return (os.system, (self.cmd,))


class _SubprocessPayload:
    """Pickling this object, when unpickled, spawns ``subprocess.Popen``."""

    def __init__(self, args):
        self.args = args

    def __reduce__(self):
        return (subprocess.Popen, (self.args,))


def _touch_cmd(path) -> str:
    # A portable one-liner sentinel command (avoids relying on the `touch`
    # binary existing on every CI image).
    return f"{sys.executable} -c \"open(r{str(path)!r}, 'w').close()\""


class TestMaliciousReduceRejectedWithoutExecution:
    def test_os_system_payload_is_rejected_and_never_executes(self, tmp_path):
        sentinel = tmp_path / "os_system.marker"
        blob = pickle.dumps((_OsSystemPayload(_touch_cmd(sentinel)), {0: "id"}))

        # Sanity check the assertion actually bites: a plain, unrestricted
        # pickle.load on this exact blob DOES run the command and create the
        # sentinel. If this stopped being true the test below would pass for
        # a meaningless reason (e.g. a broken __reduce__ payload).
        pickle.loads(blob)  # noqa: S301 -- intentional, demonstrates the RCE
        assert sentinel.exists(), "sanity check failed: payload never ran"
        sentinel.unlink()

        with pytest.raises(pickle.UnpicklingError):
            _RestrictedFaissUnpickler(io.BytesIO(blob)).load()
        assert not sentinel.exists(), (
            "os.system payload executed under the restricted unpickler"
        )

    def test_subprocess_popen_payload_is_rejected_and_never_executes(
        self, tmp_path
    ):
        sentinel = tmp_path / "subprocess.marker"
        args = [
            sys.executable,
            "-c",
            f"open(r{str(sentinel)!r}, 'w').close()",
        ]
        blob = pickle.dumps((_SubprocessPayload(args), {0: "id"}))

        proc, _id_map = pickle.loads(blob)  # noqa: S301 -- sanity check only
        proc.wait(timeout=10)
        assert sentinel.exists(), "sanity check failed: payload never ran"
        sentinel.unlink()

        with pytest.raises(pickle.UnpicklingError):
            _RestrictedFaissUnpickler(io.BytesIO(blob)).load()
        assert not sentinel.exists(), (
            "subprocess.Popen payload executed under the restricted unpickler"
        )

    def test_via_load_index_to_docstore_id_entry_point(self, tmp_path):
        """The public migration entry point must reject the same payload
        written to an on-disk ``.pkl`` — this is the exact code path
        ``legacy_cleanup._extract_map`` calls on a (possibly hostile) file
        found on disk during startup migration."""
        sentinel = tmp_path / "entrypoint.marker"
        pkl_path = tmp_path / "legacy.pkl"
        pkl_path.write_bytes(
            pickle.dumps((_OsSystemPayload(_touch_cmd(sentinel)), {0: "id"}))
        )

        with pytest.raises(pickle.UnpicklingError):
            load_index_to_docstore_id(str(pkl_path))
        assert not sentinel.exists()


# ---------------------------------------------------------------------------
# (b) PERSID / BINPERSID (persistent id) opcode rejected
# ---------------------------------------------------------------------------
class _HasPersistentRef:
    """A stand-in object whose pickled form is emitted as a persistent id."""


def _make_persistent_id_blob(protocol: int) -> bytes:
    """Build a pickle whose top-level object round-trips through
    ``persistent_id``/``persistent_load`` — i.e. it contains a genuine
    PERSID (protocol 0) or BINPERSID (protocol >= 1) opcode.
    """

    buf = io.BytesIO()

    class _PersistentPickler(pickle.Pickler):
        def persistent_id(self, obj):
            if isinstance(obj, _HasPersistentRef):
                # Persistent ids must be strings for PERSID (protocol 0);
                # any object is fine for BINPERSID (protocol >= 1). Use a
                # string so it is valid across both.
                return "attacker-controlled-persistent-ref"
            return None

    _PersistentPickler(buf, protocol=protocol).dump(
        (_HasPersistentRef(), {0: "id"})
    )
    return buf.getvalue()


class TestPersistentIdRejected:
    @pytest.mark.parametrize("protocol", [0, 2, 4])
    def test_persistent_id_opcode_is_rejected(self, protocol):
        blob = _make_persistent_id_blob(protocol)

        # Guard: confirm the blob really contains a PERSID/BINPERSID opcode,
        # not some other encoding, so the test fails for the right reason if
        # pickle's wire format ever changes.
        import pickletools

        op_names = [op.name for op, _, _ in pickletools.genops(blob)]
        assert any("PERSID" in n for n in op_names), op_names

        # Sanity: a pickler with a matching persistent_load DOES resolve it
        # (proves this is a real, working persistent-id pickle).
        class _PersistentUnpickler(pickle.Unpickler):
            def persistent_load(self, pid):
                return _HasPersistentRef()

        obj, id_map = _PersistentUnpickler(io.BytesIO(blob)).load()
        assert isinstance(obj, _HasPersistentRef)
        assert id_map == {0: "id"}

        with pytest.raises(pickle.UnpicklingError):
            _RestrictedFaissUnpickler(io.BytesIO(blob)).load()

    def test_persistent_id_rejected_via_load_index_to_docstore_id(
        self, tmp_path
    ):
        pkl_path = tmp_path / "legacy.pkl"
        pkl_path.write_bytes(_make_persistent_id_blob(protocol=2))

        with pytest.raises(pickle.UnpicklingError):
            load_index_to_docstore_id(str(pkl_path))


# ---------------------------------------------------------------------------
# (b2) copyreg extension (EXT1/EXT2/EXT4) opcode rejected via get_extension
# ---------------------------------------------------------------------------
def _install_extension_sentinel_module():
    """Register a fake, importable module holding a ``sentinel_fn`` that
    records its args when called. Returns ``(module_name, calls)`` where
    ``calls`` is the list ``sentinel_fn`` appends to if ever invoked.
    """
    mod_name = "ldr_test_ext_sentinel_module"
    mod = types.ModuleType(mod_name)
    calls: list = []

    def sentinel_fn(cmd):
        calls.append(cmd)

    # Pickle's save_global resolves globals by (__module__, __qualname__) and
    # requires the attribute to round-trip back via sys.modules, exactly like
    # the foreign-module trick above.
    sentinel_fn.__module__ = mod_name
    sentinel_fn.__qualname__ = "sentinel_fn"
    mod.sentinel_fn = sentinel_fn
    sys.modules[mod_name] = mod
    return mod_name, calls


def _make_copyreg_extension_blob(mod_name: str) -> bytes:
    """Build a protocol-2 pickle whose REDUCE payload calls ``sentinel_fn``,
    where ``sentinel_fn`` is registered as a ``copyreg`` extension so the
    pickler emits an EXT opcode (not GLOBAL/STACK_GLOBAL) to reference it.

    ``copyreg.add_extension(module, name, code)`` makes
    ``Pickler.save_global`` write EXT1/EXT2/EXT4 instead of a GLOBAL opcode
    whenever it would otherwise pickle a reference to ``module.name`` --this
    is a *second*, independent way to obtain an attacker-chosen callable that
    completely bypasses ``find_class``/GLOBAL, which is why the restricted
    unpickler must also override ``get_extension``.
    """
    sentinel_fn = sys.modules[mod_name].sentinel_fn

    # An extension code unregistered by anything else in the process. Any
    # value in the EXT4 range (> 0xffff) is fine; a genuinely unique code
    # keeps this test independent of other tests/processes touching the
    # shared copyreg extension registry.
    code = 999_999
    copyreg.add_extension(mod_name, "sentinel_fn", code)

    class _ExtensionPayload:
        def __reduce__(self):
            return (sentinel_fn, ("boom",))

    return pickle.dumps((_ExtensionPayload(), {0: "id"}), protocol=2)


class TestCopyregExtensionOpcodeRejected:
    def setup_method(self):
        self.mod_name, self.calls = _install_extension_sentinel_module()

    def teardown_method(self):
        sys.modules.pop(self.mod_name, None)
        copyreg._inverted_registry.pop(999_999, None)
        copyreg._extension_registry.pop((self.mod_name, "sentinel_fn"), None)
        copyreg._extension_cache.pop(999_999, None)

    def test_copyreg_extension_opcode_is_rejected_and_never_executes(self):
        blob = _make_copyreg_extension_blob(self.mod_name)

        # Guard: confirm the blob really contains an EXT opcode (not a plain
        # GLOBAL/STACK_GLOBAL reference), so the test fails for the right
        # reason if pickle's extension-registry wiring ever changes.
        op_names = [op.name for op, _, _ in pickletools.genops(blob)]
        assert any(n.startswith("EXT") for n in op_names), op_names

        # Sanity check the assertion actually bites: a plain, unrestricted
        # pickle.load on this exact blob DOES resolve the extension code and
        # invoke sentinel_fn.
        pickle.loads(blob)  # noqa: S301 -- intentional, demonstrates the bypass
        assert self.calls == ["boom"], "sanity check failed: payload never ran"
        self.calls.clear()

        with pytest.raises(pickle.UnpicklingError):
            _RestrictedFaissUnpickler(io.BytesIO(blob)).load()
        assert self.calls == [], (
            "copyreg extension payload executed under the restricted unpickler"
        )


# ---------------------------------------------------------------------------
# (c) Same class NAME, foreign module -> rejected (module, name)-tuple check
# ---------------------------------------------------------------------------
def _install_foreign_module_with_lookalikes():
    """Register a fake module (not the real langchain one) that defines
    classes literally named ``InMemoryDocstore`` and ``Document`` — the same
    bare names the allow-list permits, but from a different module path.
    Returns the module so its classes can be instantiated/pickled.
    """
    mod_name = "ldr_test_foreign_lookalike_module"
    mod = types.ModuleType(mod_name)

    class InMemoryDocstore:  # noqa: N801 -- deliberately shadows the real name
        def __init__(self, d):
            self._dict = d

    class Document:  # noqa: N801
        def __init__(self, page_content="", metadata=None):
            self.page_content = page_content
            self.metadata = metadata or {}

    # Pickle's default __reduce_ex__ (copyreg) resolves globals by
    # (__module__, __qualname__) and requires
    # ``getattr(sys.modules[__module__], __qualname__)`` to round-trip back
    # to the class -- a qualname still tagged with the enclosing function's
    # ``<locals>`` would fail to pickle at all. Overwrite both so these
    # classes present exactly as a real top-level ``module.ClassName``
    # class would (module attribute lookup below makes this true).
    InMemoryDocstore.__module__ = mod_name
    InMemoryDocstore.__qualname__ = "InMemoryDocstore"
    Document.__module__ = mod_name
    Document.__qualname__ = "Document"
    mod.InMemoryDocstore = InMemoryDocstore
    mod.Document = Document
    sys.modules[mod_name] = mod
    return mod


class TestForeignModuleSameNameRejected:
    def setup_method(self):
        self.mod = _install_foreign_module_with_lookalikes()

    def teardown_method(self):
        sys.modules.pop("ldr_test_foreign_lookalike_module", None)

    def test_foreign_in_memory_docstore_lookalike_rejected(self):
        foreign_docstore = self.mod.InMemoryDocstore({"id1": "not a Document"})
        blob = pickle.dumps((foreign_docstore, {0: "id1"}))

        # Sanity: a plain pickle.load happily resolves the foreign class —
        # confirms the blob is well-formed and the class is genuinely
        # reachable by (module, name), i.e. this isn't rejected for some
        # unrelated reason (e.g. a pickling error).
        obj, _ = pickle.loads(blob)  # noqa: S301
        assert type(obj).__name__ == "InMemoryDocstore"
        assert type(obj).__module__ == "ldr_test_foreign_lookalike_module"

        with pytest.raises(
            pickle.UnpicklingError,
            match="ldr_test_foreign_lookalike_module",
        ):
            _RestrictedFaissUnpickler(io.BytesIO(blob)).load()

    def test_foreign_document_lookalike_rejected(self):
        # Real, allow-listed InMemoryDocstore, but its stored "Document" is
        # the foreign look-alike class -- the allow-list must be checked for
        # EVERY resolved global, not just the outermost one.
        foreign_doc = self.mod.Document(page_content="hello", metadata={})
        docstore = InMemoryDocstore({"id1": foreign_doc})
        blob = pickle.dumps((docstore, {0: "id1"}))

        obj, _ = pickle.loads(blob)  # noqa: S301 -- sanity: it is well-formed
        assert type(obj._dict["id1"]).__module__ == (
            "ldr_test_foreign_lookalike_module"
        )

        with pytest.raises(
            pickle.UnpicklingError,
            match="ldr_test_foreign_lookalike_module",
        ):
            _RestrictedFaissUnpickler(io.BytesIO(blob)).load()

    def test_load_index_to_docstore_id_rejects_foreign_lookalike(
        self, tmp_path
    ):
        foreign_docstore = self.mod.InMemoryDocstore({"id1": "x"})
        pkl_path = tmp_path / "legacy.pkl"
        pkl_path.write_bytes(pickle.dumps((foreign_docstore, {0: "id1"})))

        with pytest.raises(pickle.UnpicklingError):
            load_index_to_docstore_id(str(pkl_path))


# ---------------------------------------------------------------------------
# (d) Benign, well-formed docstore/id-map pickle -> accepted, text-free
# ---------------------------------------------------------------------------
def _build_benign_legacy_pkl(tmp_path):
    """Build a ``.pkl`` byte-for-byte in the shape ``FAISS.save_local``
    writes and ``migrate_legacy_docstores``/``_extract_map`` reads: a
    2-tuple of ``(InMemoryDocstore, index_to_docstore_id)`` where the
    docstore maps uuid keys to real ``Document`` objects (chunk text +
    metadata) and ``index_to_docstore_id`` maps FAISS integer position to
    that same uuid key.
    """
    uuid_a = "11111111-1111-1111-1111-111111111111"
    uuid_b = "22222222-2222-2222-2222-222222222222"
    docstore = InMemoryDocstore(
        {
            uuid_a: Document(
                page_content="the quick brown fox",
                metadata={"source": "a.pdf", "page": 1},
            ),
            uuid_b: Document(
                page_content="jumps over the lazy dog",
                metadata={"source": "a.pdf", "page": 2},
            ),
        }
    )
    index_to_docstore_id = {0: uuid_a, 1: uuid_b}
    pkl_path = tmp_path / "legacy.pkl"
    pkl_path.write_bytes(pickle.dumps((docstore, index_to_docstore_id)))
    return pkl_path, index_to_docstore_id, {uuid_a, uuid_b}


class TestBenignDocstoreAccepted:
    def test_load_index_to_docstore_id_returns_only_the_id_map(self, tmp_path):
        pkl_path, expected_map, _uuids = _build_benign_legacy_pkl(tmp_path)

        result = load_index_to_docstore_id(str(pkl_path))

        assert result == expected_map
        # Exactly the position -> uuid map: no Document objects, no chunk
        # text, and no extra/missing entries leaked through.
        assert set(result.keys()) == {0, 1}
        for pos, key in result.items():
            assert isinstance(pos, int)
            assert isinstance(key, str)
        # The chunk text itself must not appear anywhere in the returned
        # value's repr (it must be a plain int->str map, nothing else).
        assert "quick brown fox" not in repr(result)
        assert "lazy dog" not in repr(result)

    def test_restricted_unpickler_directly_accepts_the_same_payload(
        self, tmp_path
    ):
        """Cross-check at the unpickler layer: the SAME allow-listed classes
        that ``load_index_to_docstore_id`` relies on are exactly what a real
        ``FAISS.save_local``-shaped payload needs -- nothing more, nothing
        less."""
        pkl_path, expected_map, uuids = _build_benign_legacy_pkl(tmp_path)

        with open(pkl_path, "rb") as f:
            docstore, index_to_docstore_id = _RestrictedFaissUnpickler(f).load()

        assert index_to_docstore_id == expected_map
        assert set(docstore._dict.keys()) == uuids
        contents = {d.page_content for d in docstore._dict.values()}
        assert contents == {
            "the quick brown fox",
            "jumps over the lazy dog",
        }