~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Load documents from bytes content.
This module provides functions to load documents from in-memory bytes,
which is useful for handling file uploads via HTTP.
"""
import tempfile
from pathlib import Path
from typing import Optional
from langchain_core.documents import Document
from loguru import logger
from local_deep_research.security.filename_sanitizer import sanitize_filename
from local_deep_research.security.isolated_loader import (
ISOLATED_FINAL_EXTENSIONS,
NLTK_DATA_INSTALL_COMMAND,
IsolatedLoaderError,
ParserBusyError,
ParserDataMissingError,
load_documents_isolated,
)
from .openpyxl_hardening import disable_openpyxl_drawing_reads
from .zip_container_guard import (
DecompressionBombError,
validate_zip_container,
)
from .loader_registry import (
get_loader_class_for_extension,
is_extension_supported,
)
from ..utilities.office_conversion import (
OfficeConversionError,
convert_legacy_office_bytes,
converted_extension,
is_legacy_office_extension,
)
from ..utilities.pandoc_conversion import (
PandocConversionError,
is_pandoc_backed_extension,
pandoc_convert_bytes_to_html,
)
def load_from_bytes(
content: bytes,
extension: str,
filename: str = "upload",
source_url: Optional[str] = None,
) -> list[Document]:
"""
Load documents from bytes content.
PDFs are extracted directly from memory with shared page, character,
and CPU ceilings. Other formats use a temporary file, with Office,
spreadsheet, HTML, Markdown, email, XML and Evernote parsing
isolated in a bounded child process, then clean it up.
Args:
content: File content as bytes
extension: File extension (with or without leading dot)
filename: Original filename for metadata
source_url: Optional source URL for metadata
Returns:
List of Document objects with extracted content
Raises:
ValueError: If the extension is not supported
IsolatedLoaderError: If the bounded parser timed out, exceeded a
limit or failed, (as ParserBusyError) had no free slot, or
(as ParserDataMissingError) needed NLTK data that is not
installed
"""
# Defense in depth: re-sanitize filename even though callers
# should have sanitized already
try:
filename = sanitize_filename(filename)
except Exception:
filename = "upload"
# Normalize extension
ext = (
extension.lower()
if extension.startswith(".")
else f".{extension.lower()}"
)
# Check if extension is supported
if not is_extension_supported(ext):
raise ValueError(f"Unsupported file extension: {ext}")
# Legacy OLE binaries (.doc/.ppt) convert through a bounded soffice
# subprocess FIRST: the raw bytes are not zip containers, and the
# converted output then flows through the same zip guard as any
# other container below. The converter refuses input without the
# OLE signature, so a zip renamed .doc/.ppt never reaches soffice.
if is_legacy_office_extension(ext):
content = convert_legacy_office_bytes(content, ext)
ext = converted_extension(ext)
# Zip-container documents (.docx/.xlsx/.odt) decompress inside the
# parsers, whose reads are bounded only by the archive's declared
# entry sizes — the request-body cap never sees the expanded side.
# Refuse hostile declarations before any parser runs. This bounds
# archive reads and specific fan-outs, not parser memory or time in
# general (see "Known gaps" in zip_container_guard). This runs
# BEFORE the pandoc conversion below so an .epub container is
# guarded as the original archive.
validate_zip_container(content, ext)
# openpyxl re-reads an image once per drawing reference and keeps
# every copy, which no per-entry ceiling bounds; drawings carry no
# text, so the .xlsx path never reads them.
if ext == ".xlsx" and not disable_openpyxl_drawing_reads():
raise DecompressionBombError(
"Cannot safely read .xlsx: openpyxl drawing guard unavailable"
)
# Pandoc-backed formats (.rtf/.epub/.rst/.org/.odt) convert to HTML
# through a bounded subprocess (timeout, heap cap, output ceiling)
# before the HTML loader parses them; their registered unstructured
# loaders would otherwise run pandoc with no bounds at all.
if is_pandoc_backed_extension(ext):
content = pandoc_convert_bytes_to_html(content, ext)
ext = ".html"
# Get the loader class for this extension
loader_info = get_loader_class_for_extension(ext)
if loader_info is None:
raise ValueError(f"No loader found for extension: {ext}")
loader_class, loader_kwargs = loader_info
tmp_path = None
try:
# PDF text must use the same page/character/CPU ceilings as the
# library download and research-upload extractors. Both RAG uploads
# and Zotero sync enter through this byte-backed loader.
if ext == ".pdf":
from .bounded_pdf import load_bounded_pdf
documents = load_bounded_pdf(content, filename)
else:
with tempfile.NamedTemporaryFile(
suffix=ext, delete=False, prefix="ldr_upload_"
) as tmp:
tmp_path = tmp.name
tmp.write(content)
# Run the registered structured document parsers behind a
# killable boundary. The guard and bounded converters above run
# first; a raw .doc/.ppt or pandoc loader would bypass them.
if ext in ISOLATED_FINAL_EXTENSIONS:
documents = load_documents_isolated(tmp_path, ext)
else:
loader = loader_class(tmp_path, **loader_kwargs)
documents = loader.load()
# Add metadata to all documents
for doc in documents:
doc.metadata["original_filename"] = filename
if source_url:
doc.metadata["source_url"] = source_url
logger.info(
f"Loaded {len(documents)} document(s) from {filename} ({ext})"
)
return documents
except ParserBusyError:
logger.warning("Document parser busy: every parser slot is in use")
raise
except ParserDataMissingError:
logger.warning(
"Document parser needs NLTK data that is not installed; install "
f"it with: {NLTK_DATA_INSTALL_COMMAND}"
)
raise
except IsolatedLoaderError:
logger.warning("Document parser exceeded a limit or failed")
raise
except (ImportError, ModuleNotFoundError):
# The extension is registered but the loader's runtime parser
# dependency is missing. The registry normally prevents this
# (see loader_registry capability flags), so surface it loudly
# rather than as a generic extraction failure.
logger.exception(
f"Missing optional dependency for {ext} files while loading "
f"{filename}; the loader for this format is not fully installed"
)
raise
except Exception:
logger.exception(f"Error loading {filename} ({ext})")
raise
finally:
# Clean up temp file
if tmp_path:
try:
# nosemgrep: semgrep.rules.path-traversal-risk, reason: tmp_path is this call's NamedTemporaryFile name; the upload filename is not used as a path
Path(tmp_path).unlink(missing_ok=True)
except Exception:
logger.warning(f"Failed to clean up temp file: {tmp_path}")
def extract_text_from_bytes(
content: bytes,
extension: str,
filename: str = "upload",
) -> Optional[str]:
"""
Extract text from bytes content.
This is a convenience function that loads documents and joins their content.
Args:
content: File content as bytes
extension: File extension (with or without leading dot)
filename: Original filename for logging
Returns:
Extracted text as string, or None if extraction failed
Raises:
ParserBusyError: Every bounded parser slot was in use. Unlike a
failed extraction, which returns None, this says nothing about
the file, so the caller can ask for the same file again.
ParserDataMissingError: The bounded parser needed NLTK data that
is not installed on this server. This too says nothing about
the file; the caller can tell the operator to install it.
"""
try:
documents = load_from_bytes(content, extension, filename)
if documents:
# The bounded child parse and bounded_pdf already count this
# join against their character limits; the other loaders,
# which run in this process, have no character limit.
return "\n\n".join(
doc.page_content for doc in documents if doc.page_content
)
return None
except (ParserBusyError, ParserDataMissingError):
raise
except IsolatedLoaderError:
logger.warning("Refused document: parsing or text exceeded its limits")
return None
except DecompressionBombError:
# Every refusal of the zip-container guard lands here, ordinary
# documents over its size, structure or parsing-work limits as
# well as hostile ones, so the log names no cause; it is logged
# without the user-controlled filename, entry names or the
# exception's message (which can carry an entry name), and
# surfaces to the upload route as a per-file error entry.
logger.warning(
"Refused document: it exceeds the parsing limits for its "
"format (size, structure or estimated parsing work)"
)
return None
except OfficeConversionError:
# Timed-out, failed, or oversized legacy conversion: scrubbed
# message only, no user-controlled content.
logger.warning(
"Rejected legacy office document: conversion failed, timed "
"out, or produced an oversized result"
)
return None
except PandocConversionError:
logger.warning(
"Rejected pandoc-backed document: conversion failed, timed "
"out, or produced an oversized result"
)
return None
except ValueError:
logger.warning(
f"No text extracted from {filename}: unsupported format or "
"unrecognized text encoding"
)
return None
except Exception:
logger.exception(f"Error extracting text from {filename}")
return None