Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

8.7 KB
Raw Blame
"""
Centralized file upload validation for security.

Provides validation for file uploads to prevent:
- Memory exhaustion attacks (file size limits)
- Malicious file uploads (structure validation)
- Resource abuse (file count limits)
- Type confusion attacks (MIME validation)
"""

import io
from typing import Optional, Tuple

import pdfplumber
from loguru import logger
from pdfminer.pdfpage import PDFPage
from pdfplumber.page import Page

from ..settings.manager import check_env_setting

# Built-in default for the per-file upload cap, in megabytes.
# Override via the LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB environment variable
# (or its server-config alias `security.upload_max_file_size_mb`).
#
# A separate library-side cap (`research_library.max_pdf_size_mb`,
# default 3072) gates whether a PDF can be *stored* after upload. Both
# values default to 3 GB so the two boundaries stay aligned; raising the
# library cap above this value has no effect because uploads above this
# cap are rejected before they reach storage.
_DEFAULT_MAX_FILE_SIZE_MB = 3072  # 3 GB


def _format_size(size_bytes: int) -> str:
    """Render a byte count as ``X.YGB`` when >=1 GB, else ``X.YMB``."""
    one_gb = 1024 * 1024 * 1024
    if size_bytes >= one_gb:
        return f"{size_bytes / one_gb:.1f}GB"
    return f"{size_bytes / (1024 * 1024):.1f}MB"


def _resolve_max_file_size() -> int:
    """Resolve the per-file upload cap, in bytes.

    Reads ``LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB`` (via
    ``settings.manager.check_env_setting``) so deployments can lower the
    cap without code changes. Falls back to ``_DEFAULT_MAX_FILE_SIZE_MB``
    when the variable is unset or unparseable.
    """
    env_value = check_env_setting("security.upload_max_file_size_mb")
    if env_value is not None:
        try:
            mb = int(env_value)
        except ValueError:
            logger.warning(
                "LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB={!r} is not an integer; "
                "falling back to {} MB default.",
                env_value,
                _DEFAULT_MAX_FILE_SIZE_MB,
            )
        else:
            # Reject zero / negative values — they would silently break
            # all uploads. Fall back to the default and log so the
            # operator can spot the misconfiguration.
            if mb > 0:
                return mb * 1024 * 1024
            logger.warning(
                "LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB={!r} must be > 0; "
                "falling back to {} MB default.",
                env_value,
                _DEFAULT_MAX_FILE_SIZE_MB,
            )
    return _DEFAULT_MAX_FILE_SIZE_MB * 1024 * 1024


class FileUploadValidator:
    """Centralized file upload validation for security."""

    # Security constants. ``MAX_FILE_SIZE`` is loaded from server config so
    # it can be lowered per-deployment via the env var or UI setting; the
    # built-in default is 3 GB (see ``server_config._DEFAULTS``).
    MAX_FILE_SIZE = _resolve_max_file_size()
    MAX_FILES_PER_REQUEST = 200  # Maximum number of files in single request
    PDF_MAGIC_BYTES = b"%PDF"  # PDF file signature
    ALLOWED_MIME_TYPES = {"application/pdf"}

    @staticmethod
    def validate_file_size(
        content_length: Optional[int], file_content: Optional[bytes] = None
    ) -> Tuple[bool, Optional[str]]:
        """
        Validate file size to prevent memory exhaustion attacks.

        Args:
            content_length: Content-Length header value (if available)
            file_content: Actual file bytes (if already read)

        Returns:
            Tuple of (is_valid, error_message)
        """
        # Check Content-Length header first (before reading file)
        if content_length is not None:
            if content_length > FileUploadValidator.MAX_FILE_SIZE:
                return (
                    False,
                    f"File too large: {_format_size(content_length)} "
                    f"(max: {_format_size(FileUploadValidator.MAX_FILE_SIZE)})",
                )

        # Check actual file size if content is provided
        if file_content is not None:
            actual_size = len(file_content)
            if actual_size > FileUploadValidator.MAX_FILE_SIZE:
                return (
                    False,
                    f"File too large: {_format_size(actual_size)} "
                    f"(max: {_format_size(FileUploadValidator.MAX_FILE_SIZE)})",
                )

        return True, None

    @staticmethod
    def validate_file_count(file_count: int) -> Tuple[bool, Optional[str]]:
        """
        Validate number of files to prevent resource abuse.

        Args:
            file_count: Number of files in the request

        Returns:
            Tuple of (is_valid, error_message)
        """
        if file_count > FileUploadValidator.MAX_FILES_PER_REQUEST:
            return (
                False,
                f"Too many files: {file_count} (max: {FileUploadValidator.MAX_FILES_PER_REQUEST})",
            )

        if file_count <= 0:
            return False, "No files provided"

        return True, None

    @staticmethod
    def validate_mime_type(
        filename: str, file_content: bytes
    ) -> Tuple[bool, Optional[str]]:
        """
        Validate file MIME type and extension.

        Args:
            filename: Original filename
            file_content: File content bytes

        Returns:
            Tuple of (is_valid, error_message)
        """
        # Check file extension
        if not filename.lower().endswith(".pdf"):
            return (
                False,
                f"Invalid file type: {filename}. Only PDF files allowed",
            )

        # Check PDF magic bytes (file signature)
        if not file_content.startswith(FileUploadValidator.PDF_MAGIC_BYTES):
            return (
                False,
                f"Invalid PDF file: {filename}. File signature mismatch",
            )

        return True, None

    @staticmethod
    def validate_pdf_structure(
        filename: str, file_content: bytes
    ) -> Tuple[bool, Optional[str]]:
        """
        Validate PDF structure to detect malicious or corrupted files.

        This goes beyond just checking the magic bytes and actually attempts
        to parse the PDF structure. Only the first page is inspected, so the
        cost does not grow with the number of pages the file declares.

        Args:
            filename: Original filename
            file_content: File content bytes

        Returns:
            Tuple of (is_valid, error_message)
        """
        try:
            # Not a ``with`` block: pdfplumber's ``PDF.pages`` -- which
            # ``PDF.close()`` walks too -- parses and builds every page up
            # front, so an 11 MB upload declaring 100,000 pages cost hundreds
            # of MB and over ten seconds here (#6771). The page tree is walked
            # lazily instead and stops at the first page; nothing needs
            # closing for an in-memory stream.
            pdf = pdfplumber.open(io.BytesIO(file_content))
            page_obj = next(PDFPage.create_pages(pdf.doc), None)
            if page_obj is None:
                return False, f"Invalid PDF: {filename}. No pages found"

            # pdfplumber's Page validates the first page's boxes exactly as
            # building ``pdf.pages`` would.
            first_page = Page(pdf, page_obj, page_number=1)
            _ = first_page.width  # Access basic metadata
            _ = first_page.height

            return True, None

        except Exception:
            logger.warning(f"PDF structure validation failed for {filename}")
            return False, f"Invalid or corrupted PDF file: {filename}"

    @classmethod
    def validate_upload(
        cls,
        filename: str,
        file_content: bytes,
        content_length: Optional[int] = None,
    ) -> Tuple[bool, Optional[str]]:
        """
        Comprehensive validation for a single file upload.

        Runs all validation checks in sequence. Stops at first failure.

        Args:
            filename: Original filename
            file_content: File content bytes
            content_length: Content-Length header (if available)

        Returns:
            Tuple of (is_valid, error_message)
        """
        # 1. Validate file size
        is_valid, error = cls.validate_file_size(content_length, file_content)
        if not is_valid:
            return is_valid, error

        # 2. Validate MIME type and extension
        is_valid, error = cls.validate_mime_type(filename, file_content)
        if not is_valid:
            return is_valid, error

        # 3. Validate PDF structure (more thorough check)
        is_valid, error = cls.validate_pdf_structure(filename, file_content)
        if not is_valid:
            return is_valid, error

        return True, None