~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Centralized file upload validation for security.
Provides validation for file uploads to prevent:
- Memory exhaustion attacks (file size limits)
- Malicious file uploads (structure validation)
- Resource abuse (file count limits)
- Type confusion attacks (MIME validation)
"""
import io
from typing import Optional, Tuple
import pdfplumber
from loguru import logger
from pdfminer.pdfpage import PDFPage
from pdfplumber.page import Page
from ..settings.manager import check_env_setting
# Built-in default for the per-file upload cap, in megabytes.
# Override via the LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB environment variable
# (or its server-config alias `security.upload_max_file_size_mb`).
#
# A separate library-side cap (`research_library.max_pdf_size_mb`,
# default 3072) gates whether a PDF can be *stored* after upload. Both
# values default to 3 GB so the two boundaries stay aligned; raising the
# library cap above this value has no effect because uploads above this
# cap are rejected before they reach storage.
_DEFAULT_MAX_FILE_SIZE_MB = 3072 # 3 GB
def _format_size(size_bytes: int) -> str:
"""Render a byte count as ``X.YGB`` when >=1 GB, else ``X.YMB``."""
one_gb = 1024 * 1024 * 1024
if size_bytes >= one_gb:
return f"{size_bytes / one_gb:.1f}GB"
return f"{size_bytes / (1024 * 1024):.1f}MB"
def _resolve_max_file_size() -> int:
"""Resolve the per-file upload cap, in bytes.
Reads ``LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB`` (via
``settings.manager.check_env_setting``) so deployments can lower the
cap without code changes. Falls back to ``_DEFAULT_MAX_FILE_SIZE_MB``
when the variable is unset or unparseable.
"""
env_value = check_env_setting("security.upload_max_file_size_mb")
if env_value is not None:
try:
mb = int(env_value)
except ValueError:
logger.warning(
"LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB={!r} is not an integer; "
"falling back to {} MB default.",
env_value,
_DEFAULT_MAX_FILE_SIZE_MB,
)
else:
# Reject zero / negative values — they would silently break
# all uploads. Fall back to the default and log so the
# operator can spot the misconfiguration.
if mb > 0:
return mb * 1024 * 1024
logger.warning(
"LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB={!r} must be > 0; "
"falling back to {} MB default.",
env_value,
_DEFAULT_MAX_FILE_SIZE_MB,
)
return _DEFAULT_MAX_FILE_SIZE_MB * 1024 * 1024
class FileUploadValidator:
"""Centralized file upload validation for security."""
# Security constants. ``MAX_FILE_SIZE`` is loaded from server config so
# it can be lowered per-deployment via the env var or UI setting; the
# built-in default is 3 GB (see ``server_config._DEFAULTS``).
MAX_FILE_SIZE = _resolve_max_file_size()
MAX_FILES_PER_REQUEST = 200 # Maximum number of files in single request
PDF_MAGIC_BYTES = b"%PDF" # PDF file signature
ALLOWED_MIME_TYPES = {"application/pdf"}
@staticmethod
def validate_file_size(
content_length: Optional[int], file_content: Optional[bytes] = None
) -> Tuple[bool, Optional[str]]:
"""
Validate file size to prevent memory exhaustion attacks.
Args:
content_length: Content-Length header value (if available)
file_content: Actual file bytes (if already read)
Returns:
Tuple of (is_valid, error_message)
"""
# Check Content-Length header first (before reading file)
if content_length is not None:
if content_length > FileUploadValidator.MAX_FILE_SIZE:
return (
False,
f"File too large: {_format_size(content_length)} "
f"(max: {_format_size(FileUploadValidator.MAX_FILE_SIZE)})",
)
# Check actual file size if content is provided
if file_content is not None:
actual_size = len(file_content)
if actual_size > FileUploadValidator.MAX_FILE_SIZE:
return (
False,
f"File too large: {_format_size(actual_size)} "
f"(max: {_format_size(FileUploadValidator.MAX_FILE_SIZE)})",
)
return True, None
@staticmethod
def validate_file_count(file_count: int) -> Tuple[bool, Optional[str]]:
"""
Validate number of files to prevent resource abuse.
Args:
file_count: Number of files in the request
Returns:
Tuple of (is_valid, error_message)
"""
if file_count > FileUploadValidator.MAX_FILES_PER_REQUEST:
return (
False,
f"Too many files: {file_count} (max: {FileUploadValidator.MAX_FILES_PER_REQUEST})",
)
if file_count <= 0:
return False, "No files provided"
return True, None
@staticmethod
def validate_mime_type(
filename: str, file_content: bytes
) -> Tuple[bool, Optional[str]]:
"""
Validate file MIME type and extension.
Args:
filename: Original filename
file_content: File content bytes
Returns:
Tuple of (is_valid, error_message)
"""
# Check file extension
if not filename.lower().endswith(".pdf"):
return (
False,
f"Invalid file type: {filename}. Only PDF files allowed",
)
# Check PDF magic bytes (file signature)
if not file_content.startswith(FileUploadValidator.PDF_MAGIC_BYTES):
return (
False,
f"Invalid PDF file: {filename}. File signature mismatch",
)
return True, None
@staticmethod
def validate_pdf_structure(
filename: str, file_content: bytes
) -> Tuple[bool, Optional[str]]:
"""
Validate PDF structure to detect malicious or corrupted files.
This goes beyond just checking the magic bytes and actually attempts
to parse the PDF structure. Only the first page is inspected, so the
cost does not grow with the number of pages the file declares.
Args:
filename: Original filename
file_content: File content bytes
Returns:
Tuple of (is_valid, error_message)
"""
try:
# Not a ``with`` block: pdfplumber's ``PDF.pages`` -- which
# ``PDF.close()`` walks too -- parses and builds every page up
# front, so an 11 MB upload declaring 100,000 pages cost hundreds
# of MB and over ten seconds here (#6771). The page tree is walked
# lazily instead and stops at the first page; nothing needs
# closing for an in-memory stream.
pdf = pdfplumber.open(io.BytesIO(file_content))
page_obj = next(PDFPage.create_pages(pdf.doc), None)
if page_obj is None:
return False, f"Invalid PDF: {filename}. No pages found"
# pdfplumber's Page validates the first page's boxes exactly as
# building ``pdf.pages`` would.
first_page = Page(pdf, page_obj, page_number=1)
_ = first_page.width # Access basic metadata
_ = first_page.height
return True, None
except Exception:
logger.warning(f"PDF structure validation failed for {filename}")
return False, f"Invalid or corrupted PDF file: {filename}"
@classmethod
def validate_upload(
cls,
filename: str,
file_content: bytes,
content_length: Optional[int] = None,
) -> Tuple[bool, Optional[str]]:
"""
Comprehensive validation for a single file upload.
Runs all validation checks in sequence. Stops at first failure.
Args:
filename: Original filename
file_content: File content bytes
content_length: Content-Length header (if available)
Returns:
Tuple of (is_valid, error_message)
"""
# 1. Validate file size
is_valid, error = cls.validate_file_size(content_length, file_content)
if not is_valid:
return is_valid, error
# 2. Validate MIME type and extension
is_valid, error = cls.validate_mime_type(filename, file_content)
if not is_valid:
return is_valid, error
# 3. Validate PDF structure (more thorough check)
is_valid, error = cls.validate_pdf_structure(filename, file_content)
if not is_valid:
return is_valid, error
return True, None