Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

3.6 KB
Raw Blame
"""Keep NLTK parsing within the isolated document worker's resource budget."""

import json
import sys
from types import ModuleType
from unittest.mock import Mock

import pytest

from local_deep_research.document_loaders import loader_registry
from local_deep_research.security import isolated_loader, isolated_worker


@pytest.fixture
def regex_settings(monkeypatch):
    module = ModuleType("nltk.redos")
    module.DEFAULT_TIMEOUT = 5.0
    monkeypatch.setitem(sys.modules, "nltk.redos", module)
    return module


def test_budget_applies_only_for_the_scoped_parse(regex_settings):
    with isolated_worker._nltk_regex_budget(".docx"):
        assert regex_settings.DEFAULT_TIMEOUT == (
            isolated_loader.PARSE_TIMEOUT_SECONDS
        )
        assert 0 < regex_settings.DEFAULT_TIMEOUT < float("inf")
    assert regex_settings.DEFAULT_TIMEOUT == 5.0


def test_budget_is_restored_when_parsing_fails(regex_settings):
    with pytest.raises(ValueError, match="controlled parse failure"):
        with isolated_worker._nltk_regex_budget(".docx"):
            raise ValueError("controlled parse failure")
    assert regex_settings.DEFAULT_TIMEOUT == 5.0


def test_older_nltk_without_the_budget_api_keeps_working(monkeypatch):
    def unavailable(name):
        assert name == "nltk.redos"
        raise ModuleNotFoundError("optional API absent", name=name)

    monkeypatch.setattr(isolated_worker, "import_module", unavailable)
    parser = Mock()
    with isolated_worker._nltk_regex_budget(".docx"):
        parser("document")
    parser.assert_called_once_with("document")


def test_other_missing_dependencies_are_not_suppressed(monkeypatch):
    def missing_dependency(name):
        raise ModuleNotFoundError("dependency absent", name="regex")

    monkeypatch.setattr(isolated_worker, "import_module", missing_dependency)
    with pytest.raises(ModuleNotFoundError) as error:
        with isolated_worker._nltk_regex_budget(".docx"):
            pytest.fail("a missing dependency must not enter the parser")
    assert error.value.name == "regex"


def test_other_loaders_do_not_import_nltk(monkeypatch):
    class OtherLoader:
        pass

    monkeypatch.setattr(
        loader_registry,
        "get_loader_class_for_extension",
        lambda extension: (OtherLoader, {}),
    )

    def unexpected_import(name):
        pytest.fail(f"unexpected NLTK import for another loader: {name}")

    monkeypatch.setattr(isolated_worker, "import_module", unexpected_import)
    parser = Mock()
    with isolated_worker._nltk_regex_budget(".enex"):
        parser("document")
    parser.assert_called_once_with("document")


def test_worker_entrypoint_applies_the_budget_before_loading_documents():
    script = "\n".join(
        [
            "import os, sys, types",
            "from local_deep_research.security import isolated_worker as worker",
            "from local_deep_research.security import isolated_loader as limits",
            "settings = types.ModuleType('nltk.redos')",
            "settings.DEFAULT_TIMEOUT = 5.0",
            "sys.modules['nltk.redos'] = settings",
            "def result(path, extension):",
            "    return {'budget': settings.DEFAULT_TIMEOUT}",
            "worker._result = result",
            "sys.argv = ['worker', 'unused', '.docx',",
            "            str(limits.MAX_CHILD_DATA_BYTES),",
            "            str(limits.CHILD_CPU_SECONDS), str(os.getppid())]",
            "raise SystemExit(worker.main())",
        ]
    )
    output = isolated_loader._run_capped([sys.executable, "-c", script], 30)
    assert json.loads(output) == {
        "budget": isolated_loader.PARSE_TIMEOUT_SECONDS
    }