Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

10.5 KB
Raw Blame
"""
Load documents from bytes content.

This module provides functions to load documents from in-memory bytes,
which is useful for handling file uploads via HTTP.
"""

import tempfile
from pathlib import Path
from typing import Optional

from langchain_core.documents import Document
from loguru import logger

from local_deep_research.security.filename_sanitizer import sanitize_filename
from local_deep_research.security.isolated_loader import (
    ISOLATED_FINAL_EXTENSIONS,
    NLTK_DATA_INSTALL_COMMAND,
    IsolatedLoaderError,
    ParserBusyError,
    ParserDataMissingError,
    load_documents_isolated,
)
from .openpyxl_hardening import disable_openpyxl_drawing_reads
from .zip_container_guard import (
    DecompressionBombError,
    validate_zip_container,
)

from .loader_registry import (
    get_loader_class_for_extension,
    is_extension_supported,
)
from ..utilities.office_conversion import (
    OfficeConversionError,
    convert_legacy_office_bytes,
    converted_extension,
    is_legacy_office_extension,
)
from ..utilities.pandoc_conversion import (
    PandocConversionError,
    is_pandoc_backed_extension,
    pandoc_convert_bytes_to_html,
)


def load_from_bytes(
    content: bytes,
    extension: str,
    filename: str = "upload",
    source_url: Optional[str] = None,
) -> list[Document]:
    """
    Load documents from bytes content.

    PDFs are extracted directly from memory with shared page, character,
    and CPU ceilings. Other formats use a temporary file, with Office,
    spreadsheet, HTML, Markdown, email, XML and Evernote parsing
    isolated in a bounded child process, then clean it up.

    Args:
        content: File content as bytes
        extension: File extension (with or without leading dot)
        filename: Original filename for metadata
        source_url: Optional source URL for metadata

    Returns:
        List of Document objects with extracted content

    Raises:
        ValueError: If the extension is not supported
        IsolatedLoaderError: If the bounded parser timed out, exceeded a
            limit or failed, (as ParserBusyError) had no free slot, or
            (as ParserDataMissingError) needed NLTK data that is not
            installed
    """
    # Defense in depth: re-sanitize filename even though callers
    # should have sanitized already
    try:
        filename = sanitize_filename(filename)
    except Exception:
        filename = "upload"

    # Normalize extension
    ext = (
        extension.lower()
        if extension.startswith(".")
        else f".{extension.lower()}"
    )

    # Check if extension is supported
    if not is_extension_supported(ext):
        raise ValueError(f"Unsupported file extension: {ext}")

    # Legacy OLE binaries (.doc/.ppt) convert through a bounded soffice
    # subprocess FIRST: the raw bytes are not zip containers, and the
    # converted output then flows through the same zip guard as any
    # other container below. The converter refuses input without the
    # OLE signature, so a zip renamed .doc/.ppt never reaches soffice.
    if is_legacy_office_extension(ext):
        content = convert_legacy_office_bytes(content, ext)
        ext = converted_extension(ext)

    # Zip-container documents (.docx/.xlsx/.odt) decompress inside the
    # parsers, whose reads are bounded only by the archive's declared
    # entry sizes — the request-body cap never sees the expanded side.
    # Refuse hostile declarations before any parser runs. This bounds
    # archive reads and specific fan-outs, not parser memory or time in
    # general (see "Known gaps" in zip_container_guard). This runs
    # BEFORE the pandoc conversion below so an .epub container is
    # guarded as the original archive.
    validate_zip_container(content, ext)

    # openpyxl re-reads an image once per drawing reference and keeps
    # every copy, which no per-entry ceiling bounds; drawings carry no
    # text, so the .xlsx path never reads them.
    if ext == ".xlsx" and not disable_openpyxl_drawing_reads():
        raise DecompressionBombError(
            "Cannot safely read .xlsx: openpyxl drawing guard unavailable"
        )

    # Pandoc-backed formats (.rtf/.epub/.rst/.org/.odt) convert to HTML
    # through a bounded subprocess (timeout, heap cap, output ceiling)
    # before the HTML loader parses them; their registered unstructured
    # loaders would otherwise run pandoc with no bounds at all.
    if is_pandoc_backed_extension(ext):
        content = pandoc_convert_bytes_to_html(content, ext)
        ext = ".html"

    # Get the loader class for this extension
    loader_info = get_loader_class_for_extension(ext)
    if loader_info is None:
        raise ValueError(f"No loader found for extension: {ext}")

    loader_class, loader_kwargs = loader_info

    tmp_path = None
    try:
        # PDF text must use the same page/character/CPU ceilings as the
        # library download and research-upload extractors. Both RAG uploads
        # and Zotero sync enter through this byte-backed loader.
        if ext == ".pdf":
            from .bounded_pdf import load_bounded_pdf

            documents = load_bounded_pdf(content, filename)
        else:
            with tempfile.NamedTemporaryFile(
                suffix=ext, delete=False, prefix="ldr_upload_"
            ) as tmp:
                tmp_path = tmp.name
                tmp.write(content)
            # Run the registered structured document parsers behind a
            # killable boundary. The guard and bounded converters above run
            # first; a raw .doc/.ppt or pandoc loader would bypass them.
            if ext in ISOLATED_FINAL_EXTENSIONS:
                documents = load_documents_isolated(tmp_path, ext)
            else:
                loader = loader_class(tmp_path, **loader_kwargs)
                documents = loader.load()

        # Add metadata to all documents
        for doc in documents:
            doc.metadata["original_filename"] = filename
            if source_url:
                doc.metadata["source_url"] = source_url

        logger.info(
            f"Loaded {len(documents)} document(s) from {filename} ({ext})"
        )
        return documents

    except ParserBusyError:
        logger.warning("Document parser busy: every parser slot is in use")
        raise
    except ParserDataMissingError:
        logger.warning(
            "Document parser needs NLTK data that is not installed; install "
            f"it with: {NLTK_DATA_INSTALL_COMMAND}"
        )
        raise
    except IsolatedLoaderError:
        logger.warning("Document parser exceeded a limit or failed")
        raise
    except (ImportError, ModuleNotFoundError):
        # The extension is registered but the loader's runtime parser
        # dependency is missing. The registry normally prevents this
        # (see loader_registry capability flags), so surface it loudly
        # rather than as a generic extraction failure.
        logger.exception(
            f"Missing optional dependency for {ext} files while loading "
            f"{filename}; the loader for this format is not fully installed"
        )
        raise

    except Exception:
        logger.exception(f"Error loading {filename} ({ext})")
        raise

    finally:
        # Clean up temp file
        if tmp_path:
            try:
                # nosemgrep: semgrep.rules.path-traversal-risk, reason: tmp_path is this call's NamedTemporaryFile name; the upload filename is not used as a path
                Path(tmp_path).unlink(missing_ok=True)
            except Exception:
                logger.warning(f"Failed to clean up temp file: {tmp_path}")


def extract_text_from_bytes(
    content: bytes,
    extension: str,
    filename: str = "upload",
) -> Optional[str]:
    """
    Extract text from bytes content.

    This is a convenience function that loads documents and joins their content.

    Args:
        content: File content as bytes
        extension: File extension (with or without leading dot)
        filename: Original filename for logging

    Returns:
        Extracted text as string, or None if extraction failed

    Raises:
        ParserBusyError: Every bounded parser slot was in use. Unlike a
            failed extraction, which returns None, this says nothing about
            the file, so the caller can ask for the same file again.
        ParserDataMissingError: The bounded parser needed NLTK data that
            is not installed on this server. This too says nothing about
            the file; the caller can tell the operator to install it.
    """
    try:
        documents = load_from_bytes(content, extension, filename)
        if documents:
            # The bounded child parse and bounded_pdf already count this
            # join against their character limits; the other loaders,
            # which run in this process, have no character limit.
            return "\n\n".join(
                doc.page_content for doc in documents if doc.page_content
            )
        return None
    except (ParserBusyError, ParserDataMissingError):
        raise
    except IsolatedLoaderError:
        logger.warning("Refused document: parsing or text exceeded its limits")
        return None
    except DecompressionBombError:
        # Every refusal of the zip-container guard lands here, ordinary
        # documents over its size, structure or parsing-work limits as
        # well as hostile ones, so the log names no cause; it is logged
        # without the user-controlled filename, entry names or the
        # exception's message (which can carry an entry name), and
        # surfaces to the upload route as a per-file error entry.
        logger.warning(
            "Refused document: it exceeds the parsing limits for its "
            "format (size, structure or estimated parsing work)"
        )
        return None
    except OfficeConversionError:
        # Timed-out, failed, or oversized legacy conversion: scrubbed
        # message only, no user-controlled content.
        logger.warning(
            "Rejected legacy office document: conversion failed, timed "
            "out, or produced an oversized result"
        )
        return None
    except PandocConversionError:
        logger.warning(
            "Rejected pandoc-backed document: conversion failed, timed "
            "out, or produced an oversized result"
        )
        return None
    except ValueError:
        logger.warning(
            f"No text extracted from {filename}: unsupported format or "
            "unrecognized text encoding"
        )
        return None
    except Exception:
        logger.exception(f"Error extracting text from {filename}")
        return None