Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

17.2 KB
Raw Blame
"""Bound the in-process document loaders after upload preflight/conversion.

The upload path has already checked ZIP containers and converted legacy
formats before calling this module. The worker runs only the final registered
loader. Parser output travels over a bounded pipe, never through a plaintext
temporary output file.
"""

from __future__ import annotations

import json
import os
import re
import signal
import subprocess
import sys
import threading
import time
from typing import Any

from langchain_core.documents import Document

from local_deep_research.config.unstructured_telemetry import (
    opt_out_of_unstructured_telemetry,
)
from local_deep_research.utilities.resource_utils import safe_close

from .isolated_metadata import metadata_from_output


PARSE_TIMEOUT_SECONDS = 90
MAX_PROTOCOL_BYTES = 40_000_000
# Counted as the upload path joins the text (see joined_text_chars).
MAX_TEXT_CHARS = 10_000_000
MAX_DOCUMENTS = 10_000
# Metadata fields across the result, including empty values.
MAX_METADATA_FIELDS = 100_000
# Scan syntax before json.loads allocates collections. The protocol has a
# root object, a document array and two objects per document; reserve one
# additional collection per metadata field for the supported metadata types.
_JSON_SYNTAX = re.compile(rb'["\\{}\[\]:,]')
_MAX_JSON_DEPTH = 5
# Linux RLIMIT_DATA, soft limit only (see isolated_worker._cap_data_memory).
MAX_CHILD_DATA_BYTES = 3 * 1024 * 1024 * 1024
# The child's own CPU-time limit (POSIX). It ends a parse that outlives this
# process, as after a crash or kill here, when nothing enforces the deadline.
# The deadline ends a parse that uses one CPU first; the limit counts the CPU
# time of all the child's threads, so one using several can reach it first.
CHILD_CPU_SECONDS = PARSE_TIMEOUT_SECONDS + 30
MAX_CONCURRENT_PARSERS = 2
_parser_slots = threading.BoundedSemaphore(MAX_CONCURRENT_PARSERS)
# The NLTK data unstructured's text classification needs. The child never
# downloads it (see _run_capped). The Docker image ships it (Dockerfile,
# nltk-data stage, scripts/install_nltk_data.py); other installs install it
# once with NLTK_DATA_INSTALL_COMMAND.
NLTK_DATA_PACKAGES = ("punkt_tab", "averaged_perceptron_tagger_eng")
NLTK_DATA_INSTALL_COMMAND = "python -m nltk.downloader " + " ".join(
    NLTK_DATA_PACKAGES
)
# Shown for the file that needed the data. The official Docker image ships
# it, so this reaches source installs and images built some other way.
NLTK_DATA_MISSING_MESSAGE = (
    "NLTK data needed to parse this file is missing or inaccessible on the server; "
    f"install it with: {NLTK_DATA_INSTALL_COMMAND} (run it as the user that "
    "runs the server; the official Docker image already includes this data). "
    "If the data is already installed, check that its directories and files "
    "are owned by the server user or administrator and are not group- or "
    "world-writable."
)

# Legacy Office and pandoc inputs are converted before this decision, so
# .doc/.ppt -> .docx/.pptx and .odt/.epub/.rtf/.rst/.org -> .html. Saved
# web archives (.mht/.mhtml) hold HTML parts that their loader parses
# with BeautifulSoup, the same class of work as an .html upload.
ISOLATED_FINAL_EXTENSIONS = frozenset(
    {
        ".docx",
        ".pptx",
        ".xlsx",
        ".xls",
        ".html",
        ".htm",
        ".mht",
        ".mhtml",
        ".md",
        ".markdown",
        ".eml",
        ".xml",
        ".enex",
    }
)


class IsolatedLoaderError(ValueError):
    """A bounded document parse timed out, exceeded a limit, or failed."""


class ParserBusyError(IsolatedLoaderError):
    """Every parser slot was in use; the same document can be retried."""


class ParserDataMissingError(IsolatedLoaderError):
    """The parse needed NLTK data (NLTK_DATA_PACKAGES) that is not installed."""


class ParserMemoryLimitError(IsolatedLoaderError):
    """The parse ran out of the child's data-memory limit."""


class ParserResourceError(IsolatedLoaderError):
    """The child could not map a required parser library into memory."""


def joined_text_chars(total: int, text: str) -> int:
    """Add *text* to *total*, counted as it appears in the joined text.

    ``extract_text_from_bytes`` joins the non-empty document texts with
    two-character separators, which ``bounded_pdf`` also counts for PDF
    pages. The worker and the parent both count this way, so the joined
    text of a bounded parse never exceeds ``MAX_TEXT_CHARS``.
    """
    if not text:
        return total
    return total + len(text) + (2 if total else 0)


def _kill_tree(process: subprocess.Popen[bytes]) -> None:
    """Stop a parser and its converter/OCR descendants after a failure.

    On POSIX this kills the child's process group. A descendant that starts
    its own session leaves that group and is not killed here, so the
    isolated loaders must not start such helpers.
    """
    if os.name == "posix":
        try:
            os.killpg(process.pid, signal.SIGKILL)
        except ProcessLookupError:
            pass
        except OSError:
            if process.poll() is None:
                process.kill()
    else:  # pragma: no cover - Windows only; no CI job runs this on Windows
        if process.poll() is None:
            try:
                subprocess.run(
                    ["taskkill", "/F", "/T", "/PID", str(process.pid)],
                    stdout=subprocess.DEVNULL,
                    stderr=subprocess.DEVNULL,
                    timeout=5,
                    check=False,
                )
            except (OSError, subprocess.TimeoutExpired):
                process.kill()
    try:
        process.wait(timeout=5)
    except subprocess.TimeoutExpired:
        process.kill()
        try:
            process.wait(timeout=5)
        except subprocess.TimeoutExpired:
            pass


def _read_capped(pipe: Any, deadline: float) -> bytes:
    """Read a nonblocking raw pipe without waiting for a writer to close it."""
    output = bytearray()
    while True:
        remaining = deadline - time.monotonic()
        if remaining <= 0:
            raise IsolatedLoaderError("document parser timed out")
        try:
            chunk = pipe.read(
                min(64 * 1024, MAX_PROTOCOL_BYTES - len(output) + 1)
            )
        except (OSError, ValueError) as exc:
            raise IsolatedLoaderError(
                "document parser output could not be read"
            ) from exc
        if chunk is None:
            # RawIOBase.read returns None when a nonblocking pipe has no
            # bytes ready; b"" alone means EOF. No reader thread can remain
            # blocked holding a buffered stream lock during cleanup.
            time.sleep(min(0.01, remaining))
            continue
        if not chunk:
            return bytes(output)
        if len(output) + len(chunk) > MAX_PROTOCOL_BYTES:
            raise IsolatedLoaderError(
                "document parser output exceeded its limit"
            )
        output.extend(chunk)


def _run_capped(command: list[str], timeout_s: float) -> bytes:
    """Read a child pipe with a hard wall clock and parent-side byte cap."""
    child_env = os.environ.copy()
    # OpenBLAS/OpenMP thread pools reserve large virtual arenas per worker.
    # One thread keeps the parser's memory budget meaningful and predictable.
    for name in (
        "OPENBLAS_NUM_THREADS",
        "OMP_NUM_THREADS",
        "MKL_NUM_THREADS",
        "NUMEXPR_NUM_THREADS",
    ):
        child_env[name] = "1"
    # Each child imports unstructured afresh, and its first unstructured
    # partition sends a usage request unless opted out. Package import sets
    # the opt-out too; this holds even if the server's environment changed.
    opt_out_of_unstructured_telemetry(child_env)
    # Importing unstructured downloads NLTK_DATA_PACKAGES when either is
    # missing, unless AUTO_DOWNLOAD_NLTK holds a value other than "true". In
    # the child that download would run under the parse deadline and outside
    # any egress policy, so it never runs there; a parse that needs the
    # missing data fails with ParserDataMissingError instead.
    child_env["AUTO_DOWNLOAD_NLTK"] = "false"
    # load_documents_isolated starts the child with -P to keep this
    # process's working directory off its sys.path, but an empty PYTHONPATH
    # entry (as in "src:" or ":src") is read as that directory. Drop empty
    # entries; named ones, relative ones included, keep their meaning.
    pythonpath = [
        entry
        for entry in child_env.get("PYTHONPATH", "").split(os.pathsep)
        if entry
    ]
    if pythonpath:
        child_env["PYTHONPATH"] = os.pathsep.join(pythonpath)
    else:
        child_env.pop("PYTHONPATH", None)
    try:
        process = subprocess.Popen(  # noqa: S603 fixed executable and module argv
            command,
            env=child_env,
            stdin=subprocess.DEVNULL,
            stdout=subprocess.PIPE,
            stderr=subprocess.DEVNULL,
            bufsize=0,
            start_new_session=os.name == "posix",
            creationflags=(
                subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0
            ),
        )
    except (OSError, ValueError) as exc:
        raise IsolatedLoaderError("document parser could not start") from exc
    if process.stdout is None:
        _kill_tree(process)
        raise IsolatedLoaderError("document parser pipe unavailable")
    deadline = time.monotonic() + timeout_s
    completed = False
    try:
        try:
            # Python 3.12+, our minimum version, supports this for pipes
            # on Windows as well as POSIX. Refuse if the OS cannot set it.
            os.set_blocking(process.stdout.fileno(), False)
        except (OSError, ValueError) as exc:
            raise IsolatedLoaderError(
                "document parser pipe could not be configured"
            ) from exc
        output = _read_capped(process.stdout, deadline)
        try:
            process.wait(timeout=max(0, deadline - time.monotonic()))
        except subprocess.TimeoutExpired as exc:
            raise IsolatedLoaderError("document parser timed out") from exc
        if process.returncode != 0:
            raise IsolatedLoaderError("document parser failed")
        completed = True
        return output
    finally:
        if not completed:
            _kill_tree(process)
        safe_close(process.stdout, "document parser pipe")


def load_documents_isolated(
    input_path: str, extension: str, *, timeout_s: float = PARSE_TIMEOUT_SECONDS
) -> list[Document]:
    """Load a preflighted temporary document within time, memory and output caps."""
    if not 0 < timeout_s <= PARSE_TIMEOUT_SECONDS:
        raise ValueError("invalid document parser timeout")
    if extension not in ISOLATED_FINAL_EXTENSIONS:
        raise ValueError("extension does not require isolated parsing")
    # Upload rate limits allow bursts. Keep a slot through both child parsing
    # and parent-side result validation, whose JSON objects also use memory.
    # Refuse instead of waiting: uploads parse on the shared default
    # executor, so queued parses would hold threads other requests need.
    if not _parser_slots.acquire(blocking=False):
        raise ParserBusyError("document parser capacity reached")
    try:
        output = _run_capped(
            [
                sys.executable,
                # "-m" alone would put the server's working directory first
                # on the child's sys.path, so a module file there named like
                # one the parser imports would run in every parse. -P (3.11+)
                # leaves it off; PYTHONPATH (less its empty entries, see
                # _run_capped) and site-packages still apply.
                "-P",
                "-m",
                f"{__package__}.isolated_worker",
                input_path,
                extension,
                str(MAX_CHILD_DATA_BYTES),
                str(CHILD_CPU_SECONDS),
                # On Linux the child is killed when this process exits, and
                # does not parse if this process has already exited.
                str(os.getpid()),
            ],
            timeout_s,
        )
        return _documents_from_output(output, extension=extension)
    finally:
        _parser_slots.release()


def _validate_protocol_structure(output: bytes) -> None:
    """Bound JSON collection allocation before decoding a worker result."""
    if len(output) > MAX_PROTOCOL_BYTES:
        raise IsolatedLoaderError("document parser output exceeded its limit")
    # The worker writes UTF-8. Raw NULs are not valid JSON text, and also
    # prevent an alternate UTF-16/32 encoding from bypassing this byte scan.
    if b"\0" in output:
        raise IsolatedLoaderError("document parser returned invalid output")
    max_collections = 2 + 2 * MAX_DOCUMENTS + MAX_METADATA_FIELDS
    max_tokens = 6 + 8 * MAX_DOCUMENTS + 12 * MAX_METADATA_FIELDS
    stack: list[int] = []
    in_string = False
    escaped_position = -1
    collections = tokens = 0
    for match in _JSON_SYNTAX.finditer(output):
        position = match.start()
        token = output[position]
        if in_string:
            if position == escaped_position:
                continue
            if token == 92:  # backslash quotes the next byte
                escaped_position = position + 1
            elif token == 34:
                in_string = False
            continue
        if token == 34:
            in_string = True
            continue
        tokens += 1
        if token in (91, 123):
            stack.append(token)
            collections += 1
            if len(stack) > _MAX_JSON_DEPTH or collections > max_collections:
                raise IsolatedLoaderError(
                    "document parser returned invalid documents"
                )
        elif token in (93, 125):
            if not stack or stack.pop() != (91 if token == 93 else 123):
                raise IsolatedLoaderError(
                    "document parser returned invalid output"
                )
        if tokens > max_tokens:
            raise IsolatedLoaderError(
                "document parser returned invalid documents"
            )


def _documents_from_output(
    output: bytes, *, extension: str | None = None
) -> list[Document]:
    """Validate the bounded worker result while its parser slot is held."""
    _validate_protocol_structure(output)
    try:
        payload = json.loads(output)
    except (UnicodeError, ValueError, RecursionError) as exc:
        # Deeply nested JSON exhausts the decoder's recursion limit; it is
        # malformed output like any other, not a server error.
        raise IsolatedLoaderError(
            "document parser returned invalid output"
        ) from exc
    if not isinstance(payload, dict) or payload.get("status") != "ok":
        if isinstance(payload, dict) and payload.get("kind") == "ImportError":
            raise ImportError("document parser dependency unavailable")
        if isinstance(payload, dict) and payload.get("kind") == "nltk_data":
            raise ParserDataMissingError(
                "document parser NLTK data is not installed"
            )
        if isinstance(payload, dict) and payload.get("kind") == "memory":
            raise ParserMemoryLimitError(
                "document parser exceeded its memory limit"
            )
        if isinstance(payload, dict) and payload.get("kind") == "resource":
            raise ParserResourceError(
                "document parser could not load its resources"
            )
        if (
            extension == ".enex"
            and isinstance(payload, dict)
            and payload.get("kind") == "DTDForbidden"
        ):
            from defusedxml.common import DTDForbidden

            raise DTDForbidden("en-export", None, None)
        raise IsolatedLoaderError("document parser refused the document")
    records = payload.get("documents")
    if not isinstance(records, list) or len(records) > MAX_DOCUMENTS:
        raise IsolatedLoaderError("document parser returned invalid documents")
    documents: list[Document] = []
    total_chars = 0
    metadata_fields = 0
    for record in records:
        if not isinstance(record, dict):
            raise IsolatedLoaderError(
                "document parser returned invalid documents"
            )
        content, metadata = record.get("page_content"), record.get("metadata")
        # String metadata is the default. Evernote also returns dates,
        # optional values and tags, which have an explicit wire format.
        if not isinstance(content, str) or not isinstance(metadata, dict):
            raise IsolatedLoaderError(
                "document parser returned invalid documents"
            )
        metadata_fields += len(metadata)
        if metadata_fields > MAX_METADATA_FIELDS:
            raise IsolatedLoaderError(
                "document parser output exceeded its limit"
            )
        try:
            metadata = metadata_from_output(metadata, extension)
        except ValueError as exc:
            raise IsolatedLoaderError(
                "document parser returned invalid documents"
            ) from exc
        total_chars = joined_text_chars(total_chars, content)
        if total_chars > MAX_TEXT_CHARS:
            raise IsolatedLoaderError(
                "document parser output exceeded its limit"
            )
        documents.append(Document(page_content=content, metadata=metadata))
    return documents