Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

11.7 KB
Raw Blame
"""Child entry point for a bounded upload document parse.

Only the already-preflighted, already-converted temporary input path is
accepted from the parent, with the limits to apply and the parent's process
ID. Parser stdout/stderr never reaches the protocol pipe; the pipe carries
one bounded JSON result and is not stored on disk.
"""

from __future__ import annotations

from collections.abc import Iterator
from contextlib import contextmanager
from importlib import import_module
import json
import os
import signal
import sys
from typing import Any

# SIGXCPU's default action ends the process. Code that catches it still gets
# SIGKILL at the hard limit, this many seconds of CPU time later.
_CPU_KILL_GRACE_SECONDS = 5
_PR_SET_PDEATHSIG = 1  # from <linux/prctl.h>
# Encoded before any limit applies, so reporting a memory refusal needs no
# new allocation once the data limit is reached.
_MEMORY_RESULT = b'{"status": "error", "kind": "memory"}'

# langchain_core imports this package only for BaseLoader.load_and_split,
# which the worker never calls, and treats it as optional. Importing it
# also imports sentence-transformers, transformers and torch, which adds
# seconds and hundreds of MB to every parse. A None entry in sys.modules
# makes that import raise ImportError, which langchain_core handles.
_UNUSED_PACKAGES = ("langchain_text_splitters",)


def _skip_unused_imports() -> None:
    for name in _UNUSED_PACKAGES:
        sys.modules.setdefault(name, None)  # type: ignore[arg-type]


def _exit_with_parent(parent_pid: int) -> None:
    # Linux only. A parent that crashed or was killed no longer stops this
    # parse at its deadline, and a restarted server's slots do not count it,
    # so ask the kernel to SIGKILL this child when its parent exits. The
    # kernel sends the signal when the parent *thread* that started the child
    # exits; that thread waits in _run_capped until this child has exited.
    if sys.platform != "linux":
        return
    try:
        import ctypes

        ctypes.CDLL(None, use_errno=True).prctl(
            _PR_SET_PDEATHSIG,
            ctypes.c_ulong(signal.SIGKILL),
            ctypes.c_ulong(0),
            ctypes.c_ulong(0),
            ctypes.c_ulong(0),
        )
    except (ImportError, OSError, AttributeError):
        # No ctypes, no C library or no prctl in it: the CPU-time limit
        # still ends a parse that outlives its parent.
        pass
    # A parent that exited before that request sends no signal, and nothing
    # would read the result. The parent starts this interpreter directly
    # (sys.executable), so while it lives it is this process's parent.
    if os.getppid() != parent_pid:
        raise OSError("document parser's parent process has exited")


def _limit_cpu_time(seconds: int) -> None:
    # POSIX only. The parent's deadline stops a parse while the parent lives;
    # this limit stops one that outlives it: SIGXCPU after *seconds* of CPU
    # time, SIGKILL at the hard limit. The hard limit is lowered as well, so
    # code in the child cannot raise either limit again.
    if os.name != "posix":
        return
    if seconds < 1:
        raise ValueError("invalid CPU-time limit")
    import resource

    # SIGXCPU's default action also dumps core, which would write the
    # parser's memory, document text included, to disk.
    resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
    soft, hard = resource.getrlimit(resource.RLIMIT_CPU)
    hard_cap = seconds + _CPU_KILL_GRACE_SECONDS
    if hard != resource.RLIM_INFINITY:
        hard_cap = min(hard, hard_cap)
    soft_cap = min(seconds, hard_cap)
    if soft != resource.RLIM_INFINITY:
        soft_cap = min(soft, soft_cap)
    resource.setrlimit(resource.RLIMIT_CPU, (soft_cap, hard_cap))


def _cap_data_memory(target: int) -> None:
    # This limit also covers anonymous mmap on supported Linux kernels.
    # Only the soft limit is lowered, so it is a soft limit: code in the
    # child could raise it back to the hard limit. The child runs only the
    # registered loaders. It counts the data segment and private writable
    # mappings (VmData), not the stack or read-only and shared mappings.
    if sys.platform != "linux":
        return
    import resource

    soft, hard = resource.getrlimit(resource.RLIMIT_DATA)
    cap = min(
        target,
        soft if soft != resource.RLIM_INFINITY else target,
        hard if hard != resource.RLIM_INFINITY else target,
    )
    resource.setrlimit(resource.RLIMIT_DATA, (cap, hard))
    resource.setrlimit(resource.RLIMIT_CORE, (0, 0))


def _is_missing_nltk_data(
    exc: BaseException, packages: tuple[str, ...]
) -> bool:
    # nltk.data.find raises a plain LookupError (not KeyError or another
    # subclass) that names the resource it could not find. The child never
    # downloads that data, so the parent can tell the operator to install it.
    return type(exc) is LookupError and any(
        package in str(exc) for package in packages
    )


def _import_failure_kind(exc: ImportError) -> str:
    # The dynamic linker reports an unavailable mapping as ImportError,
    # even when the dependency exists. Do not report it as an absent package.
    detail = str(exc).lower()
    if "cannot allocate memory" in detail or "not enough memory" in detail:
        return "memory"
    if "failed to map segment" in detail:
        return "resource"
    return "ImportError"


@contextmanager
def _nltk_regex_budget(extension: str) -> Iterator[None]:
    """Use the bounded document budget for NLTK work in this parser child."""
    from langchain_community.document_loaders.unstructured import (
        UnstructuredBaseLoader,
    )

    from local_deep_research.document_loaders.loader_registry import (
        get_loader_class_for_extension,
    )

    from .isolated_loader import PARSE_TIMEOUT_SECONDS

    loader_info = get_loader_class_for_extension(extension)
    if loader_info is None or not issubclass(
        loader_info[0], UnstructuredBaseLoader
    ):
        yield
        return
    try:
        redos = import_module("nltk.redos")
    except ModuleNotFoundError as exc:
        # NLTK releases predating this optional API keep their existing path.
        if exc.name != "nltk.redos":
            raise
        yield
        return
    previous = redos.DEFAULT_TIMEOUT
    try:
        # The parent still enforces the entire parse deadline and the child
        # retains its CPU, memory and output caps. A long ordinary paragraph
        # can use that budget instead of failing at NLTK's shorter default.
        redos.DEFAULT_TIMEOUT = float(PARSE_TIMEOUT_SECONDS)
        yield
    finally:
        redos.DEFAULT_TIMEOUT = previous


def _result(path: str, extension: str) -> dict[str, Any]:
    from .isolated_loader import (
        MAX_DOCUMENTS,
        MAX_METADATA_FIELDS,
        MAX_PROTOCOL_BYTES,
        MAX_TEXT_CHARS,
        NLTK_DATA_PACKAGES,
        joined_text_chars,
    )
    from local_deep_research.document_loaders.loader_registry import (
        get_loader_class_for_extension,
    )
    from .isolated_metadata import metadata_for_output

    # The parent's openpyxl patch is process-local. Reapply it before the
    # child loads any spreadsheet, or chartsheet drawings can multiply image
    # reads after the parent's ZIP guard has approved the package.
    if extension == ".xlsx":
        from local_deep_research.document_loaders.openpyxl_hardening import (
            disable_openpyxl_drawing_reads,
        )

        if not disable_openpyxl_drawing_reads():
            return {"status": "error", "kind": "guard"}

    loader_info = get_loader_class_for_extension(extension)
    if loader_info is None:
        return {"status": "error", "kind": "unsupported"}
    loader_class, loader_kwargs = loader_info
    try:
        documents = loader_class(path, **loader_kwargs).load()
    except ImportError as exc:
        return {"status": "error", "kind": _import_failure_kind(exc)}
    except MemoryError:
        # The data limit, not a document the parser could not read; main()
        # reports it as such.
        raise
    except Exception as exc:
        if extension == ".enex":
            from defusedxml.common import DTDForbidden

            if isinstance(exc, DTDForbidden):
                return {"status": "error", "kind": "DTDForbidden"}
        if _is_missing_nltk_data(exc, NLTK_DATA_PACKAGES):
            return {"status": "error", "kind": "nltk_data"}
        return {"status": "error", "kind": "parser"}

    if len(documents) > MAX_DOCUMENTS:
        return {"status": "error", "kind": "limit"}
    total_chars = 0
    metadata_fields = 0
    records = []
    for doc in documents:
        content = doc.page_content
        if not isinstance(content, str):
            return {"status": "error", "kind": "parser"}
        total_chars = joined_text_chars(total_chars, content)
        if total_chars > MAX_TEXT_CHARS:
            return {"status": "error", "kind": "limit"}
        metadata_fields += len(doc.metadata)
        if metadata_fields > MAX_METADATA_FIELDS:
            return {"status": "error", "kind": "limit"}
        try:
            metadata = metadata_for_output(doc.metadata, extension)
        except ValueError:
            return {"status": "error", "kind": "parser"}
        records.append({"page_content": content, "metadata": metadata})

    answer = {"status": "ok", "documents": records}
    # The parent has an independent cap, but refuse oversized output before
    # writing any part of the result to its pipe.
    encoded = json.dumps(answer, ensure_ascii=False, default=str).encode(
        "utf-8"
    )
    if len(encoded) > MAX_PROTOCOL_BYTES:
        return {"status": "error", "kind": "limit"}
    return answer


def main() -> int:
    # Before any loader import; tests call _result in their own process.
    _skip_unused_imports()
    # Preserve a separate pipe descriptor for the protocol, then redirect
    # ordinary stdout. Native parser libraries can print document content
    # directly to fd 1, so redirecting only sys.stdout is insufficient.
    protocol_fd = os.dup(1)
    devnull_fd = os.open(os.devnull, os.O_WRONLY)
    os.dup2(devnull_fd, 1)
    os.close(devnull_fd)
    try:
        out_of_memory = False
        result: dict[str, Any] | None = None
        if len(sys.argv) != 6:
            result = {"status": "error", "kind": "arguments"}
        else:
            path, extension, data_cap, cpu_seconds, parent_pid = sys.argv[1:]
            try:
                # Before the parser runs, so no parse outlives its parent or
                # escapes these limits.
                _exit_with_parent(int(parent_pid))
                _limit_cpu_time(int(cpu_seconds))
                _cap_data_memory(int(data_cap))
                with _nltk_regex_budget(extension):
                    result = _result(path, extension)
            except MemoryError:
                # Leaving this block drops the traceback and the parser
                # objects it holds before anything new is allocated.
                out_of_memory = True
            except ImportError as exc:
                result = {"status": "error", "kind": _import_failure_kind(exc)}
            except (OSError, ValueError):
                result = {"status": "error", "kind": "limit"}
            except Exception:
                result = {"status": "error", "kind": "parser"}
        encoded = _MEMORY_RESULT
        if not out_of_memory:
            try:
                encoded = json.dumps(
                    result, ensure_ascii=False, default=str
                ).encode("utf-8")
            except MemoryError:
                encoded = _MEMORY_RESULT
        remaining = memoryview(encoded)
        while remaining:
            remaining = remaining[os.write(protocol_fd, remaining) :]
        return 0
    finally:
        os.close(protocol_fd)


if __name__ == "__main__":
    raise SystemExit(main())