Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(upload): align NLTK parsing with the isolated worker budget (#8063)

LearningCircuit9392de5cf03c
diff --git a/changelog.d/8034.bugfix.md b/changelog.d/8034.bugfix.md
new file mode 100644
index 0000000000..43a68db88e
--- /dev/null
+++ b/changelog.d/8034.bugfix.md
@@ -0,0 +1,3 @@
+Uploads with long paragraphs can use the existing bounded document timeout
+with current NLTK releases. Server setup errors now explain the required
+NLTK data-directory permissions.
diff --git a/docs/library-and-rag.md b/docs/library-and-rag.md
index 41d12f9aac..b0ced18b2a 100644
--- a/docs/library-and-rag.md
+++ b/docs/library-and-rag.md
@@ -139,6 +139,10 @@ The parser child never downloads NLTK data. Its Word, PowerPoint, `.xlsx`,
HTML, Markdown, email (`.eml`) and XML parsers (unstructured) classify text
with NLTK's `punkt_tab` and `averaged_perceptron_tagger_eng` data, and so do
the formats converted to them (`.doc`, `.ppt`, `.odt`, `.epub`, `.rtf`, `.rst`, `.org`).
+Inside this child, NLTK text processing can use the existing document
+timeout, including for long paragraphs. The parent deadline and the child's
+CPU, memory and output limits still bound the whole parse. Other uses of
+NLTK keep their existing library defaults.
- **Docker image:** the image ships this data in
`/usr/local/share/nltk_data`, which is on NLTK's default search path for
@@ -147,14 +151,19 @@ the formats converted to them (`.doc`, `.ppt`, `.odt`, `.epub`, `.rtf`, `.rst`,
commit of `nltk/nltk_data` and checks its SHA-256
(`scripts/install_nltk_data.py`).
- **pip and source installs:** the pip package does not include the data.
- Until it is installed, an upload that needs it reports "NLTK data needed
- to parse this file is not installed on the server" for that file, with
+ Until usable data is available, an upload that needs it reports a server
+ NLTK data setup error for that file, with
the command `python -m nltk.downloader punkt_tab
averaged_perceptron_tagger_eng`. Run that command once with the server's Python environment, as
the user that runs the server: by default it installs the data into that
user's `~/nltk_data` (`%APPDATA%\nltk_data` on Windows). To keep it
elsewhere, add `-d <directory>` to the command and set
`NLTK_DATA=<directory>` in the server's environment.
+ With current NLTK releases, existing data directories and the files in
+ them must also be owned by the server user or administrator and must not
+ be group- or world-writable (for example, fix with `chmod -R go-w` on the
+ data directory). If installed data is reported as missing, check those
+ permissions before downloading it again.
The child starts with Python's `-P` option and without empty `PYTHONPATH`
entries, which Python reads as the working directory. Set `PYTHONPATH`
diff --git a/src/local_deep_research/security/isolated_loader.py b/src/local_deep_research/security/isolated_loader.py
index 1d7c59f00a..d0e0239bb7 100644
--- a/src/local_deep_research/security/isolated_loader.py
+++ b/src/local_deep_research/security/isolated_loader.py
@@ -60,9 +60,12 @@ NLTK_DATA_INSTALL_COMMAND = "python -m nltk.downloader " + " ".join(
# Shown for the file that needed the data. The official Docker image ships
# it, so this reaches source installs and images built some other way.
NLTK_DATA_MISSING_MESSAGE = (
- "NLTK data needed to parse this file is not installed on the server; "
+ "NLTK data needed to parse this file is missing or inaccessible on the server; "
f"install it with: {NLTK_DATA_INSTALL_COMMAND} (run it as the user that "
- "runs the server; the official Docker image already includes this data)"
+ "runs the server; the official Docker image already includes this data). "
+ "If the data is already installed, check that its directories and files "
+ "are owned by the server user or administrator and are not group- or "
+ "world-writable."
)
# Legacy Office and pandoc inputs are converted before this decision, so
diff --git a/src/local_deep_research/security/isolated_worker.py b/src/local_deep_research/security/isolated_worker.py
index 45d93a9195..be7187b468 100644
--- a/src/local_deep_research/security/isolated_worker.py
+++ b/src/local_deep_research/security/isolated_worker.py
@@ -8,6 +8,9 @@ one bounded JSON result and is not stored on disk.
from __future__ import annotations
+from collections.abc import Iterator
+from contextlib import contextmanager
+from importlib import import_module
import json
import os
import signal
@@ -130,6 +133,44 @@ def _import_failure_kind(exc: ImportError) -> str:
return "ImportError"
+@contextmanager
+def _nltk_regex_budget(extension: str) -> Iterator[None]:
+ """Use the bounded document budget for NLTK work in this parser child."""
+ from langchain_community.document_loaders.unstructured import (
+ UnstructuredBaseLoader,
+ )
+
+ from local_deep_research.document_loaders.loader_registry import (
+ get_loader_class_for_extension,
+ )
+
+ from .isolated_loader import PARSE_TIMEOUT_SECONDS
+
+ loader_info = get_loader_class_for_extension(extension)
+ if loader_info is None or not issubclass(
+ loader_info[0], UnstructuredBaseLoader
+ ):
+ yield
+ return
+ try:
+ redos = import_module("nltk.redos")
+ except ModuleNotFoundError as exc:
+ # NLTK releases predating this optional API keep their existing path.
+ if exc.name != "nltk.redos":
+ raise
+ yield
+ return
+ previous = redos.DEFAULT_TIMEOUT
+ try:
+ # The parent still enforces the entire parse deadline and the child
+ # retains its CPU, memory and output caps. A long ordinary paragraph
+ # can use that budget instead of failing at NLTK's shorter default.
+ redos.DEFAULT_TIMEOUT = float(PARSE_TIMEOUT_SECONDS)
+ yield
+ finally:
+ redos.DEFAULT_TIMEOUT = previous
+
+
def _result(path: str, extension: str) -> dict[str, Any]:
from .isolated_loader import (
MAX_DOCUMENTS,
@@ -232,7 +273,8 @@ def main() -> int:
_exit_with_parent(int(parent_pid))
_limit_cpu_time(int(cpu_seconds))
_cap_data_memory(int(data_cap))
- result = _result(path, extension)
+ with _nltk_regex_budget(extension):
+ result = _result(path, extension)
except MemoryError:
# Leaving this block drops the traceback and the parser
# objects it holds before anything new is allocated.
diff --git a/tests/research_library/routes/test_rag_routes_upload_main_port.py b/tests/research_library/routes/test_rag_routes_upload_main_port.py
index db2e07755d..0c7e06a94c 100644
--- a/tests/research_library/routes/test_rag_routes_upload_main_port.py
+++ b/tests/research_library/routes/test_rag_routes_upload_main_port.py
@@ -1258,6 +1258,7 @@ class TestUploadToCollection:
every SAVEPOINT closed.
"""
from local_deep_research.security.isolated_loader import (
+ NLTK_DATA_MISSING_MESSAGE,
ParserDataMissingError,
)
@@ -1288,13 +1289,11 @@ class TestUploadToCollection:
assert status == 200
assert rdata["summary"]["successful"] == 0
errors = {e["filename"]: e["error"] for e in rdata["errors"]}
- assert errors["nodata.docx"] == (
- "NLTK data needed to parse this file is not installed on the "
- "server; install it with: python -m nltk.downloader punkt_tab "
- "averaged_perceptron_tagger_eng (run it as the user that runs "
- "the server; the official Docker image already includes this "
- "data)"
- )
+ # The message now also covers data that exists but that NLTK refuses
+ # because of its ownership or permissions (#8034), so compare with the
+ # shared constant rather than a copy of an older wording.
+ assert errors["nodata.docx"] == NLTK_DATA_MISSING_MESSAGE
+ assert "python -m nltk.downloader punkt_tab" in errors["nodata.docx"]
assert "Could not extract text" in errors["empty.docx"]
assert _roles(db_session) == [("read", 1), ("read", 2), ("write", 2)]
assert not any(sp.is_active for sp in db_session._savepoints)
diff --git a/tests/security/test_nltk_parser_budget.py b/tests/security/test_nltk_parser_budget.py
new file mode 100644
index 0000000000..e222f26e30
--- /dev/null
+++ b/tests/security/test_nltk_parser_budget.py
@@ -0,0 +1,102 @@
+"""Keep NLTK parsing within the isolated document worker's resource budget."""
+
+import json
+import sys
+from types import ModuleType
+from unittest.mock import Mock
+
+import pytest
+
+from local_deep_research.document_loaders import loader_registry
+from local_deep_research.security import isolated_loader, isolated_worker
+
+
+def regex_settings(monkeypatch):
+ module = ModuleType("nltk.redos")
+ module.DEFAULT_TIMEOUT = 5.0
+ monkeypatch.setitem(sys.modules, "nltk.redos", module)
+ return module
+
+
+def test_budget_applies_only_for_the_scoped_parse(regex_settings):
+ with isolated_worker._nltk_regex_budget(".docx"):
+ assert regex_settings.DEFAULT_TIMEOUT == (
+ isolated_loader.PARSE_TIMEOUT_SECONDS
+ )
+ assert 0 < regex_settings.DEFAULT_TIMEOUT < float("inf")
+ assert regex_settings.DEFAULT_TIMEOUT == 5.0
+
+
+def test_budget_is_restored_when_parsing_fails(regex_settings):
+ with pytest.raises(ValueError, match="controlled parse failure"):
+ with isolated_worker._nltk_regex_budget(".docx"):
+ raise ValueError("controlled parse failure")
+ assert regex_settings.DEFAULT_TIMEOUT == 5.0
+
+
+def test_older_nltk_without_the_budget_api_keeps_working(monkeypatch):
+ def unavailable(name):
+ assert name == "nltk.redos"
+ raise ModuleNotFoundError("optional API absent", name=name)
+
+ monkeypatch.setattr(isolated_worker, "import_module", unavailable)
+ parser = Mock()
+ with isolated_worker._nltk_regex_budget(".docx"):
+ parser("document")
+ parser.assert_called_once_with("document")
+
+
+def test_other_missing_dependencies_are_not_suppressed(monkeypatch):
+ def missing_dependency(name):
+ raise ModuleNotFoundError("dependency absent", name="regex")
+
+ monkeypatch.setattr(isolated_worker, "import_module", missing_dependency)
+ with pytest.raises(ModuleNotFoundError) as error:
+ with isolated_worker._nltk_regex_budget(".docx"):
+ pytest.fail("a missing dependency must not enter the parser")
+ assert error.value.name == "regex"
+
+
+def test_other_loaders_do_not_import_nltk(monkeypatch):
+ class OtherLoader:
+ pass
+
+ monkeypatch.setattr(
+ loader_registry,
+ "get_loader_class_for_extension",
+ lambda extension: (OtherLoader, {}),
+ )
+
+ def unexpected_import(name):
+ pytest.fail(f"unexpected NLTK import for another loader: {name}")
+
+ monkeypatch.setattr(isolated_worker, "import_module", unexpected_import)
+ parser = Mock()
+ with isolated_worker._nltk_regex_budget(".enex"):
+ parser("document")
+ parser.assert_called_once_with("document")
+
+
+def test_worker_entrypoint_applies_the_budget_before_loading_documents():
+ script = "\n".join(
+ [
+ "import os, sys, types",
+ "from local_deep_research.security import isolated_worker as worker",
+ "from local_deep_research.security import isolated_loader as limits",
+ "settings = types.ModuleType('nltk.redos')",
+ "settings.DEFAULT_TIMEOUT = 5.0",
+ "sys.modules['nltk.redos'] = settings",
+ "def result(path, extension):",
+ " return {'budget': settings.DEFAULT_TIMEOUT}",
+ "worker._result = result",
+ "sys.argv = ['worker', 'unused', '.docx',",
+ " str(limits.MAX_CHILD_DATA_BYTES),",
+ " str(limits.CHILD_CPU_SECONDS), str(os.getppid())]",
+ "raise SystemExit(worker.main())",
+ ]
+ )
+ output = isolated_loader._run_capped([sys.executable, "-c", script], 30)
+ assert json.loads(output) == {
+ "budget": isolated_loader.PARSE_TIMEOUT_SECONDS
+ }