~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Child entry point for a bounded upload document parse.
Only the already-preflighted, already-converted temporary input path is
accepted from the parent, with the limits to apply and the parent's process
ID. Parser stdout/stderr never reaches the protocol pipe; the pipe carries
one bounded JSON result and is not stored on disk.
"""
from __future__ import annotations
from collections.abc import Iterator
from contextlib import contextmanager
from importlib import import_module
import json
import os
import signal
import sys
from typing import Any
# SIGXCPU's default action ends the process. Code that catches it still gets
# SIGKILL at the hard limit, this many seconds of CPU time later.
_CPU_KILL_GRACE_SECONDS = 5
_PR_SET_PDEATHSIG = 1 # from <linux/prctl.h>
# Encoded before any limit applies, so reporting a memory refusal needs no
# new allocation once the data limit is reached.
_MEMORY_RESULT = b'{"status": "error", "kind": "memory"}'
# langchain_core imports this package only for BaseLoader.load_and_split,
# which the worker never calls, and treats it as optional. Importing it
# also imports sentence-transformers, transformers and torch, which adds
# seconds and hundreds of MB to every parse. A None entry in sys.modules
# makes that import raise ImportError, which langchain_core handles.
_UNUSED_PACKAGES = ("langchain_text_splitters",)
def _skip_unused_imports() -> None:
for name in _UNUSED_PACKAGES:
sys.modules.setdefault(name, None) # type: ignore[arg-type]
def _exit_with_parent(parent_pid: int) -> None:
# Linux only. A parent that crashed or was killed no longer stops this
# parse at its deadline, and a restarted server's slots do not count it,
# so ask the kernel to SIGKILL this child when its parent exits. The
# kernel sends the signal when the parent *thread* that started the child
# exits; that thread waits in _run_capped until this child has exited.
if sys.platform != "linux":
return
try:
import ctypes
ctypes.CDLL(None, use_errno=True).prctl(
_PR_SET_PDEATHSIG,
ctypes.c_ulong(signal.SIGKILL),
ctypes.c_ulong(0),
ctypes.c_ulong(0),
ctypes.c_ulong(0),
)
except (ImportError, OSError, AttributeError):
# No ctypes, no C library or no prctl in it: the CPU-time limit
# still ends a parse that outlives its parent.
pass
# A parent that exited before that request sends no signal, and nothing
# would read the result. The parent starts this interpreter directly
# (sys.executable), so while it lives it is this process's parent.
if os.getppid() != parent_pid:
raise OSError("document parser's parent process has exited")
def _limit_cpu_time(seconds: int) -> None:
# POSIX only. The parent's deadline stops a parse while the parent lives;
# this limit stops one that outlives it: SIGXCPU after *seconds* of CPU
# time, SIGKILL at the hard limit. The hard limit is lowered as well, so
# code in the child cannot raise either limit again.
if os.name != "posix":
return
if seconds < 1:
raise ValueError("invalid CPU-time limit")
import resource
# SIGXCPU's default action also dumps core, which would write the
# parser's memory, document text included, to disk.
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
soft, hard = resource.getrlimit(resource.RLIMIT_CPU)
hard_cap = seconds + _CPU_KILL_GRACE_SECONDS
if hard != resource.RLIM_INFINITY:
hard_cap = min(hard, hard_cap)
soft_cap = min(seconds, hard_cap)
if soft != resource.RLIM_INFINITY:
soft_cap = min(soft, soft_cap)
resource.setrlimit(resource.RLIMIT_CPU, (soft_cap, hard_cap))
def _cap_data_memory(target: int) -> None:
# This limit also covers anonymous mmap on supported Linux kernels.
# Only the soft limit is lowered, so it is a soft limit: code in the
# child could raise it back to the hard limit. The child runs only the
# registered loaders. It counts the data segment and private writable
# mappings (VmData), not the stack or read-only and shared mappings.
if sys.platform != "linux":
return
import resource
soft, hard = resource.getrlimit(resource.RLIMIT_DATA)
cap = min(
target,
soft if soft != resource.RLIM_INFINITY else target,
hard if hard != resource.RLIM_INFINITY else target,
)
resource.setrlimit(resource.RLIMIT_DATA, (cap, hard))
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
def _is_missing_nltk_data(
exc: BaseException, packages: tuple[str, ...]
) -> bool:
# nltk.data.find raises a plain LookupError (not KeyError or another
# subclass) that names the resource it could not find. The child never
# downloads that data, so the parent can tell the operator to install it.
return type(exc) is LookupError and any(
package in str(exc) for package in packages
)
def _import_failure_kind(exc: ImportError) -> str:
# The dynamic linker reports an unavailable mapping as ImportError,
# even when the dependency exists. Do not report it as an absent package.
detail = str(exc).lower()
if "cannot allocate memory" in detail or "not enough memory" in detail:
return "memory"
if "failed to map segment" in detail:
return "resource"
return "ImportError"
@contextmanager
def _nltk_regex_budget(extension: str) -> Iterator[None]:
"""Use the bounded document budget for NLTK work in this parser child."""
from langchain_community.document_loaders.unstructured import (
UnstructuredBaseLoader,
)
from local_deep_research.document_loaders.loader_registry import (
get_loader_class_for_extension,
)
from .isolated_loader import PARSE_TIMEOUT_SECONDS
loader_info = get_loader_class_for_extension(extension)
if loader_info is None or not issubclass(
loader_info[0], UnstructuredBaseLoader
):
yield
return
try:
redos = import_module("nltk.redos")
except ModuleNotFoundError as exc:
# NLTK releases predating this optional API keep their existing path.
if exc.name != "nltk.redos":
raise
yield
return
previous = redos.DEFAULT_TIMEOUT
try:
# The parent still enforces the entire parse deadline and the child
# retains its CPU, memory and output caps. A long ordinary paragraph
# can use that budget instead of failing at NLTK's shorter default.
redos.DEFAULT_TIMEOUT = float(PARSE_TIMEOUT_SECONDS)
yield
finally:
redos.DEFAULT_TIMEOUT = previous
def _result(path: str, extension: str) -> dict[str, Any]:
from .isolated_loader import (
MAX_DOCUMENTS,
MAX_METADATA_FIELDS,
MAX_PROTOCOL_BYTES,
MAX_TEXT_CHARS,
NLTK_DATA_PACKAGES,
joined_text_chars,
)
from local_deep_research.document_loaders.loader_registry import (
get_loader_class_for_extension,
)
from .isolated_metadata import metadata_for_output
# The parent's openpyxl patch is process-local. Reapply it before the
# child loads any spreadsheet, or chartsheet drawings can multiply image
# reads after the parent's ZIP guard has approved the package.
if extension == ".xlsx":
from local_deep_research.document_loaders.openpyxl_hardening import (
disable_openpyxl_drawing_reads,
)
if not disable_openpyxl_drawing_reads():
return {"status": "error", "kind": "guard"}
loader_info = get_loader_class_for_extension(extension)
if loader_info is None:
return {"status": "error", "kind": "unsupported"}
loader_class, loader_kwargs = loader_info
try:
documents = loader_class(path, **loader_kwargs).load()
except ImportError as exc:
return {"status": "error", "kind": _import_failure_kind(exc)}
except MemoryError:
# The data limit, not a document the parser could not read; main()
# reports it as such.
raise
except Exception as exc:
if extension == ".enex":
from defusedxml.common import DTDForbidden
if isinstance(exc, DTDForbidden):
return {"status": "error", "kind": "DTDForbidden"}
if _is_missing_nltk_data(exc, NLTK_DATA_PACKAGES):
return {"status": "error", "kind": "nltk_data"}
return {"status": "error", "kind": "parser"}
if len(documents) > MAX_DOCUMENTS:
return {"status": "error", "kind": "limit"}
total_chars = 0
metadata_fields = 0
records = []
for doc in documents:
content = doc.page_content
if not isinstance(content, str):
return {"status": "error", "kind": "parser"}
total_chars = joined_text_chars(total_chars, content)
if total_chars > MAX_TEXT_CHARS:
return {"status": "error", "kind": "limit"}
metadata_fields += len(doc.metadata)
if metadata_fields > MAX_METADATA_FIELDS:
return {"status": "error", "kind": "limit"}
try:
metadata = metadata_for_output(doc.metadata, extension)
except ValueError:
return {"status": "error", "kind": "parser"}
records.append({"page_content": content, "metadata": metadata})
answer = {"status": "ok", "documents": records}
# The parent has an independent cap, but refuse oversized output before
# writing any part of the result to its pipe.
encoded = json.dumps(answer, ensure_ascii=False, default=str).encode(
"utf-8"
)
if len(encoded) > MAX_PROTOCOL_BYTES:
return {"status": "error", "kind": "limit"}
return answer
def main() -> int:
# Before any loader import; tests call _result in their own process.
_skip_unused_imports()
# Preserve a separate pipe descriptor for the protocol, then redirect
# ordinary stdout. Native parser libraries can print document content
# directly to fd 1, so redirecting only sys.stdout is insufficient.
protocol_fd = os.dup(1)
devnull_fd = os.open(os.devnull, os.O_WRONLY)
os.dup2(devnull_fd, 1)
os.close(devnull_fd)
try:
out_of_memory = False
result: dict[str, Any] | None = None
if len(sys.argv) != 6:
result = {"status": "error", "kind": "arguments"}
else:
path, extension, data_cap, cpu_seconds, parent_pid = sys.argv[1:]
try:
# Before the parser runs, so no parse outlives its parent or
# escapes these limits.
_exit_with_parent(int(parent_pid))
_limit_cpu_time(int(cpu_seconds))
_cap_data_memory(int(data_cap))
with _nltk_regex_budget(extension):
result = _result(path, extension)
except MemoryError:
# Leaving this block drops the traceback and the parser
# objects it holds before anything new is allocated.
out_of_memory = True
except ImportError as exc:
result = {"status": "error", "kind": _import_failure_kind(exc)}
except (OSError, ValueError):
result = {"status": "error", "kind": "limit"}
except Exception:
result = {"status": "error", "kind": "parser"}
encoded = _MEMORY_RESULT
if not out_of_memory:
try:
encoded = json.dumps(
result, ensure_ascii=False, default=str
).encode("utf-8")
except MemoryError:
encoded = _MEMORY_RESULT
remaining = memoryview(encoded)
while remaining:
remaining = remaining[os.write(protocol_fd, remaining) :]
return 0
finally:
os.close(protocol_fd)
if __name__ == "__main__":
raise SystemExit(main())