~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Bound the in-process document loaders after upload preflight/conversion.
The upload path has already checked ZIP containers and converted legacy
formats before calling this module. The worker runs only the final registered
loader. Parser output travels over a bounded pipe, never through a plaintext
temporary output file.
"""
from __future__ import annotations
import json
import os
import re
import signal
import subprocess
import sys
import threading
import time
from typing import Any
from langchain_core.documents import Document
from local_deep_research.config.unstructured_telemetry import (
opt_out_of_unstructured_telemetry,
)
from local_deep_research.utilities.resource_utils import safe_close
from .isolated_metadata import metadata_from_output
PARSE_TIMEOUT_SECONDS = 90
MAX_PROTOCOL_BYTES = 40_000_000
# Counted as the upload path joins the text (see joined_text_chars).
MAX_TEXT_CHARS = 10_000_000
MAX_DOCUMENTS = 10_000
# Metadata fields across the result, including empty values.
MAX_METADATA_FIELDS = 100_000
# Scan syntax before json.loads allocates collections. The protocol has a
# root object, a document array and two objects per document; reserve one
# additional collection per metadata field for the supported metadata types.
_JSON_SYNTAX = re.compile(rb'["\\{}\[\]:,]')
_MAX_JSON_DEPTH = 5
# Linux RLIMIT_DATA, soft limit only (see isolated_worker._cap_data_memory).
MAX_CHILD_DATA_BYTES = 3 * 1024 * 1024 * 1024
# The child's own CPU-time limit (POSIX). It ends a parse that outlives this
# process, as after a crash or kill here, when nothing enforces the deadline.
# The deadline ends a parse that uses one CPU first; the limit counts the CPU
# time of all the child's threads, so one using several can reach it first.
CHILD_CPU_SECONDS = PARSE_TIMEOUT_SECONDS + 30
MAX_CONCURRENT_PARSERS = 2
_parser_slots = threading.BoundedSemaphore(MAX_CONCURRENT_PARSERS)
# The NLTK data unstructured's text classification needs. The child never
# downloads it (see _run_capped). The Docker image ships it (Dockerfile,
# nltk-data stage, scripts/install_nltk_data.py); other installs install it
# once with NLTK_DATA_INSTALL_COMMAND.
NLTK_DATA_PACKAGES = ("punkt_tab", "averaged_perceptron_tagger_eng")
NLTK_DATA_INSTALL_COMMAND = "python -m nltk.downloader " + " ".join(
NLTK_DATA_PACKAGES
)
# Shown for the file that needed the data. The official Docker image ships
# it, so this reaches source installs and images built some other way.
NLTK_DATA_MISSING_MESSAGE = (
"NLTK data needed to parse this file is missing or inaccessible on the server; "
f"install it with: {NLTK_DATA_INSTALL_COMMAND} (run it as the user that "
"runs the server; the official Docker image already includes this data). "
"If the data is already installed, check that its directories and files "
"are owned by the server user or administrator and are not group- or "
"world-writable."
)
# Legacy Office and pandoc inputs are converted before this decision, so
# .doc/.ppt -> .docx/.pptx and .odt/.epub/.rtf/.rst/.org -> .html. Saved
# web archives (.mht/.mhtml) hold HTML parts that their loader parses
# with BeautifulSoup, the same class of work as an .html upload.
ISOLATED_FINAL_EXTENSIONS = frozenset(
{
".docx",
".pptx",
".xlsx",
".xls",
".html",
".htm",
".mht",
".mhtml",
".md",
".markdown",
".eml",
".xml",
".enex",
}
)
class IsolatedLoaderError(ValueError):
"""A bounded document parse timed out, exceeded a limit, or failed."""
class ParserBusyError(IsolatedLoaderError):
"""Every parser slot was in use; the same document can be retried."""
class ParserDataMissingError(IsolatedLoaderError):
"""The parse needed NLTK data (NLTK_DATA_PACKAGES) that is not installed."""
class ParserMemoryLimitError(IsolatedLoaderError):
"""The parse ran out of the child's data-memory limit."""
class ParserResourceError(IsolatedLoaderError):
"""The child could not map a required parser library into memory."""
def joined_text_chars(total: int, text: str) -> int:
"""Add *text* to *total*, counted as it appears in the joined text.
``extract_text_from_bytes`` joins the non-empty document texts with
two-character separators, which ``bounded_pdf`` also counts for PDF
pages. The worker and the parent both count this way, so the joined
text of a bounded parse never exceeds ``MAX_TEXT_CHARS``.
"""
if not text:
return total
return total + len(text) + (2 if total else 0)
def _kill_tree(process: subprocess.Popen[bytes]) -> None:
"""Stop a parser and its converter/OCR descendants after a failure.
On POSIX this kills the child's process group. A descendant that starts
its own session leaves that group and is not killed here, so the
isolated loaders must not start such helpers.
"""
if os.name == "posix":
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
except OSError:
if process.poll() is None:
process.kill()
else: # pragma: no cover - Windows only; no CI job runs this on Windows
if process.poll() is None:
try:
subprocess.run(
["taskkill", "/F", "/T", "/PID", str(process.pid)],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=5,
check=False,
)
except (OSError, subprocess.TimeoutExpired):
process.kill()
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
process.kill()
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
pass
def _read_capped(pipe: Any, deadline: float) -> bytes:
"""Read a nonblocking raw pipe without waiting for a writer to close it."""
output = bytearray()
while True:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise IsolatedLoaderError("document parser timed out")
try:
chunk = pipe.read(
min(64 * 1024, MAX_PROTOCOL_BYTES - len(output) + 1)
)
except (OSError, ValueError) as exc:
raise IsolatedLoaderError(
"document parser output could not be read"
) from exc
if chunk is None:
# RawIOBase.read returns None when a nonblocking pipe has no
# bytes ready; b"" alone means EOF. No reader thread can remain
# blocked holding a buffered stream lock during cleanup.
time.sleep(min(0.01, remaining))
continue
if not chunk:
return bytes(output)
if len(output) + len(chunk) > MAX_PROTOCOL_BYTES:
raise IsolatedLoaderError(
"document parser output exceeded its limit"
)
output.extend(chunk)
def _run_capped(command: list[str], timeout_s: float) -> bytes:
"""Read a child pipe with a hard wall clock and parent-side byte cap."""
child_env = os.environ.copy()
# OpenBLAS/OpenMP thread pools reserve large virtual arenas per worker.
# One thread keeps the parser's memory budget meaningful and predictable.
for name in (
"OPENBLAS_NUM_THREADS",
"OMP_NUM_THREADS",
"MKL_NUM_THREADS",
"NUMEXPR_NUM_THREADS",
):
child_env[name] = "1"
# Each child imports unstructured afresh, and its first unstructured
# partition sends a usage request unless opted out. Package import sets
# the opt-out too; this holds even if the server's environment changed.
opt_out_of_unstructured_telemetry(child_env)
# Importing unstructured downloads NLTK_DATA_PACKAGES when either is
# missing, unless AUTO_DOWNLOAD_NLTK holds a value other than "true". In
# the child that download would run under the parse deadline and outside
# any egress policy, so it never runs there; a parse that needs the
# missing data fails with ParserDataMissingError instead.
child_env["AUTO_DOWNLOAD_NLTK"] = "false"
# load_documents_isolated starts the child with -P to keep this
# process's working directory off its sys.path, but an empty PYTHONPATH
# entry (as in "src:" or ":src") is read as that directory. Drop empty
# entries; named ones, relative ones included, keep their meaning.
pythonpath = [
entry
for entry in child_env.get("PYTHONPATH", "").split(os.pathsep)
if entry
]
if pythonpath:
child_env["PYTHONPATH"] = os.pathsep.join(pythonpath)
else:
child_env.pop("PYTHONPATH", None)
try:
process = subprocess.Popen( # noqa: S603 fixed executable and module argv
command,
env=child_env,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
bufsize=0,
start_new_session=os.name == "posix",
creationflags=(
subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0
),
)
except (OSError, ValueError) as exc:
raise IsolatedLoaderError("document parser could not start") from exc
if process.stdout is None:
_kill_tree(process)
raise IsolatedLoaderError("document parser pipe unavailable")
deadline = time.monotonic() + timeout_s
completed = False
try:
try:
# Python 3.12+, our minimum version, supports this for pipes
# on Windows as well as POSIX. Refuse if the OS cannot set it.
os.set_blocking(process.stdout.fileno(), False)
except (OSError, ValueError) as exc:
raise IsolatedLoaderError(
"document parser pipe could not be configured"
) from exc
output = _read_capped(process.stdout, deadline)
try:
process.wait(timeout=max(0, deadline - time.monotonic()))
except subprocess.TimeoutExpired as exc:
raise IsolatedLoaderError("document parser timed out") from exc
if process.returncode != 0:
raise IsolatedLoaderError("document parser failed")
completed = True
return output
finally:
if not completed:
_kill_tree(process)
safe_close(process.stdout, "document parser pipe")
def load_documents_isolated(
input_path: str, extension: str, *, timeout_s: float = PARSE_TIMEOUT_SECONDS
) -> list[Document]:
"""Load a preflighted temporary document within time, memory and output caps."""
if not 0 < timeout_s <= PARSE_TIMEOUT_SECONDS:
raise ValueError("invalid document parser timeout")
if extension not in ISOLATED_FINAL_EXTENSIONS:
raise ValueError("extension does not require isolated parsing")
# Upload rate limits allow bursts. Keep a slot through both child parsing
# and parent-side result validation, whose JSON objects also use memory.
# Refuse instead of waiting: uploads parse on the shared default
# executor, so queued parses would hold threads other requests need.
if not _parser_slots.acquire(blocking=False):
raise ParserBusyError("document parser capacity reached")
try:
output = _run_capped(
[
sys.executable,
# "-m" alone would put the server's working directory first
# on the child's sys.path, so a module file there named like
# one the parser imports would run in every parse. -P (3.11+)
# leaves it off; PYTHONPATH (less its empty entries, see
# _run_capped) and site-packages still apply.
"-P",
"-m",
f"{__package__}.isolated_worker",
input_path,
extension,
str(MAX_CHILD_DATA_BYTES),
str(CHILD_CPU_SECONDS),
# On Linux the child is killed when this process exits, and
# does not parse if this process has already exited.
str(os.getpid()),
],
timeout_s,
)
return _documents_from_output(output, extension=extension)
finally:
_parser_slots.release()
def _validate_protocol_structure(output: bytes) -> None:
"""Bound JSON collection allocation before decoding a worker result."""
if len(output) > MAX_PROTOCOL_BYTES:
raise IsolatedLoaderError("document parser output exceeded its limit")
# The worker writes UTF-8. Raw NULs are not valid JSON text, and also
# prevent an alternate UTF-16/32 encoding from bypassing this byte scan.
if b"\0" in output:
raise IsolatedLoaderError("document parser returned invalid output")
max_collections = 2 + 2 * MAX_DOCUMENTS + MAX_METADATA_FIELDS
max_tokens = 6 + 8 * MAX_DOCUMENTS + 12 * MAX_METADATA_FIELDS
stack: list[int] = []
in_string = False
escaped_position = -1
collections = tokens = 0
for match in _JSON_SYNTAX.finditer(output):
position = match.start()
token = output[position]
if in_string:
if position == escaped_position:
continue
if token == 92: # backslash quotes the next byte
escaped_position = position + 1
elif token == 34:
in_string = False
continue
if token == 34:
in_string = True
continue
tokens += 1
if token in (91, 123):
stack.append(token)
collections += 1
if len(stack) > _MAX_JSON_DEPTH or collections > max_collections:
raise IsolatedLoaderError(
"document parser returned invalid documents"
)
elif token in (93, 125):
if not stack or stack.pop() != (91 if token == 93 else 123):
raise IsolatedLoaderError(
"document parser returned invalid output"
)
if tokens > max_tokens:
raise IsolatedLoaderError(
"document parser returned invalid documents"
)
def _documents_from_output(
output: bytes, *, extension: str | None = None
) -> list[Document]:
"""Validate the bounded worker result while its parser slot is held."""
_validate_protocol_structure(output)
try:
payload = json.loads(output)
except (UnicodeError, ValueError, RecursionError) as exc:
# Deeply nested JSON exhausts the decoder's recursion limit; it is
# malformed output like any other, not a server error.
raise IsolatedLoaderError(
"document parser returned invalid output"
) from exc
if not isinstance(payload, dict) or payload.get("status") != "ok":
if isinstance(payload, dict) and payload.get("kind") == "ImportError":
raise ImportError("document parser dependency unavailable")
if isinstance(payload, dict) and payload.get("kind") == "nltk_data":
raise ParserDataMissingError(
"document parser NLTK data is not installed"
)
if isinstance(payload, dict) and payload.get("kind") == "memory":
raise ParserMemoryLimitError(
"document parser exceeded its memory limit"
)
if isinstance(payload, dict) and payload.get("kind") == "resource":
raise ParserResourceError(
"document parser could not load its resources"
)
if (
extension == ".enex"
and isinstance(payload, dict)
and payload.get("kind") == "DTDForbidden"
):
from defusedxml.common import DTDForbidden
raise DTDForbidden("en-export", None, None)
raise IsolatedLoaderError("document parser refused the document")
records = payload.get("documents")
if not isinstance(records, list) or len(records) > MAX_DOCUMENTS:
raise IsolatedLoaderError("document parser returned invalid documents")
documents: list[Document] = []
total_chars = 0
metadata_fields = 0
for record in records:
if not isinstance(record, dict):
raise IsolatedLoaderError(
"document parser returned invalid documents"
)
content, metadata = record.get("page_content"), record.get("metadata")
# String metadata is the default. Evernote also returns dates,
# optional values and tags, which have an explicit wire format.
if not isinstance(content, str) or not isinstance(metadata, dict):
raise IsolatedLoaderError(
"document parser returned invalid documents"
)
metadata_fields += len(metadata)
if metadata_fields > MAX_METADATA_FIELDS:
raise IsolatedLoaderError(
"document parser output exceeded its limit"
)
try:
metadata = metadata_from_output(metadata, extension)
except ValueError as exc:
raise IsolatedLoaderError(
"document parser returned invalid documents"
) from exc
total_chars = joined_text_chars(total_chars, content)
if total_chars > MAX_TEXT_CHARS:
raise IsolatedLoaderError(
"document parser output exceeded its limit"
)
documents.append(Document(page_content=content, metadata=metadata))
return documents