~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(upload): align NLTK parsing with the isolated worker budget (#8063)
LearningCircuit9392de5cf03c
diff --git a/changelog.d/8034.bugfix.md b/changelog.d/8034.bugfix.mdnew file mode 100644index 0000000000..43a68db88e--- /dev/null+++ b/changelog.d/8034.bugfix.md@@ -0,0 +1,3 @@+Uploads with long paragraphs can use the existing bounded document timeout+with current NLTK releases. Server setup errors now explain the required+NLTK data-directory permissions.diff --git a/docs/library-and-rag.md b/docs/library-and-rag.mdindex 41d12f9aac..b0ced18b2a 100644--- a/docs/library-and-rag.md+++ b/docs/library-and-rag.md@@ -139,6 +139,10 @@ The parser child never downloads NLTK data. Its Word, PowerPoint, `.xlsx`,HTML, Markdown, email (`.eml`) and XML parsers (unstructured) classify textwith NLTK's `punkt_tab` and `averaged_perceptron_tagger_eng` data, and so dothe formats converted to them (`.doc`, `.ppt`, `.odt`, `.epub`, `.rtf`, `.rst`, `.org`).+Inside this child, NLTK text processing can use the existing document+timeout, including for long paragraphs. The parent deadline and the child's+CPU, memory and output limits still bound the whole parse. Other uses of+NLTK keep their existing library defaults.- **Docker image:** the image ships this data in`/usr/local/share/nltk_data`, which is on NLTK's default search path for@@ -147,14 +151,19 @@ the formats converted to them (`.doc`, `.ppt`, `.odt`, `.epub`, `.rtf`, `.rst`,commit of `nltk/nltk_data` and checks its SHA-256(`scripts/install_nltk_data.py`).- **pip and source installs:** the pip package does not include the data.- Until it is installed, an upload that needs it reports "NLTK data needed- to parse this file is not installed on the server" for that file, with+ Until usable data is available, an upload that needs it reports a server+ NLTK data setup error for that file, withthe command `python -m nltk.downloader punkt_tabaveraged_perceptron_tagger_eng`. Run that command once with the server's Python environment, asthe user that runs the server: by default it installs the data into thatuser's `~/nltk_data` (`%APPDATA%\nltk_data` on Windows). To keep itelsewhere, add `-d <directory>` to the command and set`NLTK_DATA=<directory>` in the server's environment.+ With current NLTK releases, existing data directories and the files in+ them must also be owned by the server user or administrator and must not+ be group- or world-writable (for example, fix with `chmod -R go-w` on the+ data directory). If installed data is reported as missing, check those+ permissions before downloading it again.The child starts with Python's `-P` option and without empty `PYTHONPATH`entries, which Python reads as the working directory. Set `PYTHONPATH`diff --git a/src/local_deep_research/security/isolated_loader.py b/src/local_deep_research/security/isolated_loader.pyindex 1d7c59f00a..d0e0239bb7 100644--- a/src/local_deep_research/security/isolated_loader.py+++ b/src/local_deep_research/security/isolated_loader.py@@ -60,9 +60,12 @@ NLTK_DATA_INSTALL_COMMAND = "python -m nltk.downloader " + " ".join(# Shown for the file that needed the data. The official Docker image ships# it, so this reaches source installs and images built some other way.NLTK_DATA_MISSING_MESSAGE = (- "NLTK data needed to parse this file is not installed on the server; "+ "NLTK data needed to parse this file is missing or inaccessible on the server; "f"install it with: {NLTK_DATA_INSTALL_COMMAND} (run it as the user that "- "runs the server; the official Docker image already includes this data)"+ "runs the server; the official Docker image already includes this data). "+ "If the data is already installed, check that its directories and files "+ "are owned by the server user or administrator and are not group- or "+ "world-writable.")# Legacy Office and pandoc inputs are converted before this decision, sodiff --git a/src/local_deep_research/security/isolated_worker.py b/src/local_deep_research/security/isolated_worker.pyindex 45d93a9195..be7187b468 100644--- a/src/local_deep_research/security/isolated_worker.py+++ b/src/local_deep_research/security/isolated_worker.py@@ -8,6 +8,9 @@ one bounded JSON result and is not stored on disk.from __future__ import annotations+from collections.abc import Iterator+from contextlib import contextmanager+from importlib import import_moduleimport jsonimport osimport signal@@ -130,6 +133,44 @@ def _import_failure_kind(exc: ImportError) -> str:return "ImportError"+@contextmanager+def _nltk_regex_budget(extension: str) -> Iterator[None]:+ """Use the bounded document budget for NLTK work in this parser child."""+ from langchain_community.document_loaders.unstructured import (+ UnstructuredBaseLoader,+ )++ from local_deep_research.document_loaders.loader_registry import (+ get_loader_class_for_extension,+ )++ from .isolated_loader import PARSE_TIMEOUT_SECONDS++ loader_info = get_loader_class_for_extension(extension)+ if loader_info is None or not issubclass(+ loader_info[0], UnstructuredBaseLoader+ ):+ yield+ return+ try:+ redos = import_module("nltk.redos")+ except ModuleNotFoundError as exc:+ # NLTK releases predating this optional API keep their existing path.+ if exc.name != "nltk.redos":+ raise+ yield+ return+ previous = redos.DEFAULT_TIMEOUT+ try:+ # The parent still enforces the entire parse deadline and the child+ # retains its CPU, memory and output caps. A long ordinary paragraph+ # can use that budget instead of failing at NLTK's shorter default.+ redos.DEFAULT_TIMEOUT = float(PARSE_TIMEOUT_SECONDS)+ yield+ finally:+ redos.DEFAULT_TIMEOUT = previous++def _result(path: str, extension: str) -> dict[str, Any]:from .isolated_loader import (MAX_DOCUMENTS,@@ -232,7 +273,8 @@ def main() -> int:_exit_with_parent(int(parent_pid))_limit_cpu_time(int(cpu_seconds))_cap_data_memory(int(data_cap))- result = _result(path, extension)+ with _nltk_regex_budget(extension):+ result = _result(path, extension)except MemoryError:# Leaving this block drops the traceback and the parser# objects it holds before anything new is allocated.diff --git a/tests/research_library/routes/test_rag_routes_upload_main_port.py b/tests/research_library/routes/test_rag_routes_upload_main_port.pyindex db2e07755d..0c7e06a94c 100644--- a/tests/research_library/routes/test_rag_routes_upload_main_port.py+++ b/tests/research_library/routes/test_rag_routes_upload_main_port.py@@ -1258,6 +1258,7 @@ class TestUploadToCollection:every SAVEPOINT closed."""from local_deep_research.security.isolated_loader import (+ NLTK_DATA_MISSING_MESSAGE,ParserDataMissingError,)@@ -1288,13 +1289,11 @@ class TestUploadToCollection:assert status == 200assert rdata["summary"]["successful"] == 0errors = {e["filename"]: e["error"] for e in rdata["errors"]}- assert errors["nodata.docx"] == (- "NLTK data needed to parse this file is not installed on the "- "server; install it with: python -m nltk.downloader punkt_tab "- "averaged_perceptron_tagger_eng (run it as the user that runs "- "the server; the official Docker image already includes this "- "data)"- )+ # The message now also covers data that exists but that NLTK refuses+ # because of its ownership or permissions (#8034), so compare with the+ # shared constant rather than a copy of an older wording.+ assert errors["nodata.docx"] == NLTK_DATA_MISSING_MESSAGE+ assert "python -m nltk.downloader punkt_tab" in errors["nodata.docx"]assert "Could not extract text" in errors["empty.docx"]assert _roles(db_session) == [("read", 1), ("read", 2), ("write", 2)]assert not any(sp.is_active for sp in db_session._savepoints)diff --git a/tests/security/test_nltk_parser_budget.py b/tests/security/test_nltk_parser_budget.pynew file mode 100644index 0000000000..e222f26e30--- /dev/null+++ b/tests/security/test_nltk_parser_budget.py@@ -0,0 +1,102 @@+"""Keep NLTK parsing within the isolated document worker's resource budget."""++import json+import sys+from types import ModuleType+from unittest.mock import Mock++import pytest++from local_deep_research.document_loaders import loader_registry+from local_deep_research.security import isolated_loader, isolated_worker+++def regex_settings(monkeypatch):+ module = ModuleType("nltk.redos")+ module.DEFAULT_TIMEOUT = 5.0+ monkeypatch.setitem(sys.modules, "nltk.redos", module)+ return module+++def test_budget_applies_only_for_the_scoped_parse(regex_settings):+ with isolated_worker._nltk_regex_budget(".docx"):+ assert regex_settings.DEFAULT_TIMEOUT == (+ isolated_loader.PARSE_TIMEOUT_SECONDS+ )+ assert 0 < regex_settings.DEFAULT_TIMEOUT < float("inf")+ assert regex_settings.DEFAULT_TIMEOUT == 5.0+++def test_budget_is_restored_when_parsing_fails(regex_settings):+ with pytest.raises(ValueError, match="controlled parse failure"):+ with isolated_worker._nltk_regex_budget(".docx"):+ raise ValueError("controlled parse failure")+ assert regex_settings.DEFAULT_TIMEOUT == 5.0+++def test_older_nltk_without_the_budget_api_keeps_working(monkeypatch):+ def unavailable(name):+ assert name == "nltk.redos"+ raise ModuleNotFoundError("optional API absent", name=name)++ monkeypatch.setattr(isolated_worker, "import_module", unavailable)+ parser = Mock()+ with isolated_worker._nltk_regex_budget(".docx"):+ parser("document")+ parser.assert_called_once_with("document")+++def test_other_missing_dependencies_are_not_suppressed(monkeypatch):+ def missing_dependency(name):+ raise ModuleNotFoundError("dependency absent", name="regex")++ monkeypatch.setattr(isolated_worker, "import_module", missing_dependency)+ with pytest.raises(ModuleNotFoundError) as error:+ with isolated_worker._nltk_regex_budget(".docx"):+ pytest.fail("a missing dependency must not enter the parser")+ assert error.value.name == "regex"+++def test_other_loaders_do_not_import_nltk(monkeypatch):+ class OtherLoader:+ pass++ monkeypatch.setattr(+ loader_registry,+ "get_loader_class_for_extension",+ lambda extension: (OtherLoader, {}),+ )++ def unexpected_import(name):+ pytest.fail(f"unexpected NLTK import for another loader: {name}")++ monkeypatch.setattr(isolated_worker, "import_module", unexpected_import)+ parser = Mock()+ with isolated_worker._nltk_regex_budget(".enex"):+ parser("document")+ parser.assert_called_once_with("document")+++def test_worker_entrypoint_applies_the_budget_before_loading_documents():+ script = "\n".join(+ [+ "import os, sys, types",+ "from local_deep_research.security import isolated_worker as worker",+ "from local_deep_research.security import isolated_loader as limits",+ "settings = types.ModuleType('nltk.redos')",+ "settings.DEFAULT_TIMEOUT = 5.0",+ "sys.modules['nltk.redos'] = settings",+ "def result(path, extension):",+ " return {'budget': settings.DEFAULT_TIMEOUT}",+ "worker._result = result",+ "sys.argv = ['worker', 'unused', '.docx',",+ " str(limits.MAX_CHILD_DATA_BYTES),",+ " str(limits.CHILD_CPU_SECONDS), str(os.getppid())]",+ "raise SystemExit(worker.main())",+ ]+ )+ output = isolated_loader._run_capped([sys.executable, "-c", script], 30)+ assert json.loads(output) == {+ "budget": isolated_loader.PARSE_TIMEOUT_SECONDS+ }