Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

feat(security): bump default upload size cap to 3GB and make it configurable (#4196)

LearningCircuitea5c99ce1849
diff --git a/changelog.d/+upload-max-file-size-3gb.feature.md b/changelog.d/+upload-max-file-size-3gb.feature.md
new file mode 100644
index 0000000000..3e72652a98
--- /dev/null
+++ b/changelog.d/+upload-max-file-size-3gb.feature.md
@@ -0,0 +1 @@
+Per-file upload cap (`FileUploadValidator.MAX_FILE_SIZE`) is now configurable and defaults to 3 GB (was 50 MB) so large academic datasets and PDFs fit out of the box. Deployments that want a tighter bound can lower it via the `LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB` environment variable or the `security.upload_max_file_size_mb` setting. Memory usage stays bounded by the existing 5 MB spool-to-disk threshold on multipart requests.
diff --git a/src/local_deep_research/security/file_upload_validator.py b/src/local_deep_research/security/file_upload_validator.py
index 6092e744c6..155f89a5cb 100644
--- a/src/local_deep_research/security/file_upload_validator.py
+++ b/src/local_deep_research/security/file_upload_validator.py
@@ -14,12 +14,63 @@ from typing import Optional, Tuple
import pdfplumber
from loguru import logger
+from ..settings.manager import check_env_setting
+
+# Built-in default for the per-file upload cap, in megabytes.
+# Override via the LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB environment variable
+# (or its server-config alias `security.upload_max_file_size_mb`).
+_DEFAULT_MAX_FILE_SIZE_MB = 3072 # 3 GB
+
+
+def _format_size(size_bytes: int) -> str:
+ """Render a byte count as ``X.YGB`` when >=1 GB, else ``X.YMB``."""
+ one_gb = 1024 * 1024 * 1024
+ if size_bytes >= one_gb:
+ return f"{size_bytes / one_gb:.1f}GB"
+ return f"{size_bytes / (1024 * 1024):.1f}MB"
+
+
+def _resolve_max_file_size() -> int:
+ """Resolve the per-file upload cap, in bytes.
+
+ Reads ``LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB`` (via
+ ``settings.manager.check_env_setting``) so deployments can lower the
+ cap without code changes. Falls back to ``_DEFAULT_MAX_FILE_SIZE_MB``
+ when the variable is unset or unparseable.
+ """
+ env_value = check_env_setting("security.upload_max_file_size_mb")
+ if env_value is not None:
+ try:
+ mb = int(env_value)
+ except ValueError:
+ logger.warning(
+ "LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB={!r} is not an integer; "
+ "falling back to {} MB default.",
+ env_value,
+ _DEFAULT_MAX_FILE_SIZE_MB,
+ )
+ else:
+ # Reject zero / negative values — they would silently break
+ # all uploads. Fall back to the default and log so the
+ # operator can spot the misconfiguration.
+ if mb > 0:
+ return mb * 1024 * 1024
+ logger.warning(
+ "LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB={!r} must be > 0; "
+ "falling back to {} MB default.",
+ env_value,
+ _DEFAULT_MAX_FILE_SIZE_MB,
+ )
+ return _DEFAULT_MAX_FILE_SIZE_MB * 1024 * 1024
+
class FileUploadValidator:
"""Centralized file upload validation for security."""
- # Security constants
- MAX_FILE_SIZE = 50 * 1024 * 1024 # 50MB per file
+ # Security constants. ``MAX_FILE_SIZE`` is loaded from server config so
+ # it can be lowered per-deployment via the env var or UI setting; the
+ # built-in default is 3 GB (see ``server_config._DEFAULTS``).
+ MAX_FILE_SIZE = _resolve_max_file_size()
MAX_FILES_PER_REQUEST = 200 # Maximum number of files in single request
PDF_MAGIC_BYTES = b"%PDF" # PDF file signature
ALLOWED_MIME_TYPES = {"application/pdf"}
@@ -41,22 +92,20 @@ class FileUploadValidator:
# Check Content-Length header first (before reading file)
if content_length is not None:
if content_length > FileUploadValidator.MAX_FILE_SIZE:
- size_mb = content_length / (1024 * 1024)
- max_mb = FileUploadValidator.MAX_FILE_SIZE / (1024 * 1024)
return (
False,
- f"File too large: {size_mb:.1f}MB (max: {max_mb}MB)",
+ f"File too large: {_format_size(content_length)} "
+ f"(max: {_format_size(FileUploadValidator.MAX_FILE_SIZE)})",
)
# Check actual file size if content is provided
if file_content is not None:
actual_size = len(file_content)
if actual_size > FileUploadValidator.MAX_FILE_SIZE:
- size_mb = actual_size / (1024 * 1024)
- max_mb = FileUploadValidator.MAX_FILE_SIZE / (1024 * 1024)
return (
False,
- f"File too large: {size_mb:.1f}MB (max: {max_mb}MB)",
+ f"File too large: {_format_size(actual_size)} "
+ f"(max: {_format_size(FileUploadValidator.MAX_FILE_SIZE)})",
)
return True, None
diff --git a/src/local_deep_research/web/app_factory.py b/src/local_deep_research/web/app_factory.py
index 82b2573267..6d5d84cc47 100644
--- a/src/local_deep_research/web/app_factory.py
+++ b/src/local_deep_research/web/app_factory.py
@@ -37,8 +37,10 @@ class DiskSpoolingRequest(Request):
files larger than max_form_memory_size to temporary files on disk instead
of keeping them in memory.
- Security fix for issue #1176: With 200 files × 50MB limit, the default
- behavior could consume 10GB+ of memory per request.
+ Security fix for issue #1176: a request right up against the per-file ×
+ per-request limits could otherwise consume tens of GB of memory in
+ one go. The spool-to-disk threshold keeps memory bounded regardless
+ of how high the per-file cap (FileUploadValidator.MAX_FILE_SIZE) is set.
"""
# Files larger than 5MB are spooled to disk instead of memory
diff --git a/tests/pdf_tests/test_file_validator.py b/tests/pdf_tests/test_file_validator.py
index cd1b80a0aa..64a0ec7272 100644
--- a/tests/pdf_tests/test_file_validator.py
+++ b/tests/pdf_tests/test_file_validator.py
@@ -9,13 +9,20 @@ Tests cover all validation methods:
- Comprehensive upload validation
"""
+from unittest.mock import patch
+
from local_deep_research.security.file_upload_validator import (
FileUploadValidator,
)
class TestValidateFileSize:
- """Tests for validate_file_size method."""
+ """Tests for validate_file_size method.
+
+ Oversized-path tests patch ``MAX_FILE_SIZE`` to a small value so they
+ can exercise the rejection branch without allocating multi-GB buffers
+ in CI.
+ """
def test_valid_size_via_content_length(self):
"""Test that valid size passes via Content-Length header."""
@@ -36,19 +43,23 @@ class TestValidateFileSize:
def test_oversized_via_content_length(self):
"""Test that oversized file is rejected via Content-Length."""
- is_valid, error = FileUploadValidator.validate_file_size(
- content_length=51 * 1024 * 1024 # 51MB (over 50MB limit)
- )
+ with patch.object(
+ FileUploadValidator, "MAX_FILE_SIZE", 50 * 1024 * 1024
+ ):
+ is_valid, error = FileUploadValidator.validate_file_size(
+ content_length=51 * 1024 * 1024 # 51MB over a 50MB cap
+ )
assert is_valid is False
assert error is not None
assert "too large" in error.lower()
def test_oversized_via_file_content(self):
"""Test that oversized file is rejected via actual content."""
- content = b"x" * (51 * 1024 * 1024) # 51MB (over 50MB limit)
- is_valid, error = FileUploadValidator.validate_file_size(
- content_length=None, file_content=content
- )
+ with patch.object(FileUploadValidator, "MAX_FILE_SIZE", 1024):
+ content = b"x" * 2048 # 2KB over a 1KB cap
+ is_valid, error = FileUploadValidator.validate_file_size(
+ content_length=None, file_content=content
+ )
assert is_valid is False
assert error is not None
assert "too large" in error.lower()
@@ -232,14 +243,13 @@ class TestValidateUpload:
def test_all_validations_run(self):
"""Test that all validations are executed."""
- # Invalid size should fail first
- large_content = b"%PDF-1.4\n" + (
- b"x" * (51 * 1024 * 1024)
- ) # 51MB (over 50MB limit)
- is_valid, error = FileUploadValidator.validate_upload(
- filename="test.pdf",
- file_content=large_content,
- )
+ # Invalid size should fail first. Patch the cap to keep memory low.
+ with patch.object(FileUploadValidator, "MAX_FILE_SIZE", 1024):
+ large_content = b"%PDF-1.4\n" + (b"x" * 2048) # 2KB over 1KB cap
+ is_valid, error = FileUploadValidator.validate_upload(
+ filename="test.pdf",
+ file_content=large_content,
+ )
assert is_valid is False
assert "too large" in error.lower()
@@ -256,11 +266,12 @@ class TestValidateUpload:
def test_returns_first_error(self):
"""Test that first validation error stops processing."""
# Large file with wrong extension - should fail on size first
- large_content = b"x" * (51 * 1024 * 1024) # 51MB (over 50MB limit)
- is_valid, error = FileUploadValidator.validate_upload(
- filename="test.txt",
- file_content=large_content,
- )
+ with patch.object(FileUploadValidator, "MAX_FILE_SIZE", 1024):
+ large_content = b"x" * 2048 # 2KB over 1KB cap
+ is_valid, error = FileUploadValidator.validate_upload(
+ filename="test.txt",
+ file_content=large_content,
+ )
assert is_valid is False
# Should fail on size before checking extension
assert "too large" in error.lower()
@@ -269,9 +280,9 @@ class TestValidateUpload:
class TestConstants:
"""Tests for class constants."""
- def test_max_file_size_is_50mb(self):
- """Verify MAX_FILE_SIZE is 50MB."""
- expected = 50 * 1024 * 1024
+ def test_max_file_size_default_is_3gb(self):
+ """Default MAX_FILE_SIZE matches the 3 GB server-config default."""
+ expected = 3 * 1024 * 1024 * 1024
assert FileUploadValidator.MAX_FILE_SIZE == expected
def test_max_files_is_200(self):
diff --git a/tests/security/test_file_upload_validator.py b/tests/security/test_file_upload_validator.py
index b65a04eba5..2ff391a731 100644
--- a/tests/security/test_file_upload_validator.py
+++ b/tests/security/test_file_upload_validator.py
@@ -13,10 +13,9 @@ class TestFileUploadValidatorConstants:
"""Tests for FileUploadValidator constants."""
def test_max_file_size_defined(self):
- """MAX_FILE_SIZE is defined and reasonable."""
+ """MAX_FILE_SIZE is defined and matches the 3 GB server-config default."""
assert FileUploadValidator.MAX_FILE_SIZE > 0
- # 50MB default
- assert FileUploadValidator.MAX_FILE_SIZE == 50 * 1024 * 1024
+ assert FileUploadValidator.MAX_FILE_SIZE == 3 * 1024 * 1024 * 1024
def test_max_files_per_request_defined(self):
"""MAX_FILES_PER_REQUEST is defined and reasonable."""
@@ -32,6 +31,57 @@ class TestFileUploadValidatorConstants:
assert "application/pdf" in FileUploadValidator.ALLOWED_MIME_TYPES
+class TestResolveMaxFileSize:
+ """The per-file cap honors LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB."""
+
+ def test_default_when_env_unset(self):
+ from local_deep_research.security import file_upload_validator as mod
+
+ with patch.object(mod, "check_env_setting", return_value=None):
+ assert (
+ mod._resolve_max_file_size()
+ == mod._DEFAULT_MAX_FILE_SIZE_MB * 1024 * 1024
+ )
+
+ def test_env_value_overrides_default(self):
+ from local_deep_research.security import file_upload_validator as mod
+
+ # 750MB override (env vars always arrive as strings).
+ with patch.object(mod, "check_env_setting", return_value="750"):
+ assert mod._resolve_max_file_size() == 750 * 1024 * 1024
+
+ def test_unparseable_env_falls_back_to_default(self):
+ from local_deep_research.security import file_upload_validator as mod
+
+ with patch.object(
+ mod, "check_env_setting", return_value="not-a-number"
+ ):
+ assert (
+ mod._resolve_max_file_size()
+ == mod._DEFAULT_MAX_FILE_SIZE_MB * 1024 * 1024
+ )
+
+ def test_zero_env_value_falls_back_to_default(self):
+ """A misconfigured cap of 0 would silently break all uploads."""
+ from local_deep_research.security import file_upload_validator as mod
+
+ with patch.object(mod, "check_env_setting", return_value="0"):
+ assert (
+ mod._resolve_max_file_size()
+ == mod._DEFAULT_MAX_FILE_SIZE_MB * 1024 * 1024
+ )
+
+ def test_negative_env_value_falls_back_to_default(self):
+ """Negative values are nonsensical for a size cap — reject them."""
+ from local_deep_research.security import file_upload_validator as mod
+
+ with patch.object(mod, "check_env_setting", return_value="-1"):
+ assert (
+ mod._resolve_max_file_size()
+ == mod._DEFAULT_MAX_FILE_SIZE_MB * 1024 * 1024
+ )
+
+
class TestValidateFileSize:
"""Tests for FileUploadValidator.validate_file_size()."""
@@ -56,11 +106,16 @@ class TestValidateFileSize:
assert "too large" in error.lower()
def test_file_content_exceeds_max(self):
- """Rejects file content exceeding max."""
- large_content = b"x" * (FileUploadValidator.MAX_FILE_SIZE + 1)
- is_valid, error = FileUploadValidator.validate_file_size(
- None, large_content
- )
+ """Rejects file content exceeding max.
+
+ Patches the cap to a small value so the test doesn't have to
+ allocate gigabytes to cross the real default.
+ """
+ with patch.object(FileUploadValidator, "MAX_FILE_SIZE", 1024):
+ large_content = b"x" * 2048 # 2KB over a 1KB cap
+ is_valid, error = FileUploadValidator.validate_file_size(
+ None, large_content
+ )
assert is_valid is False
assert "too large" in error.lower()
@@ -78,8 +133,11 @@ class TestValidateFileSize:
def test_error_message_includes_size(self):
"""Error message includes file size information."""
- large_size = 60 * 1024 * 1024 # 60MB
- is_valid, error = FileUploadValidator.validate_file_size(large_size)
+ with patch.object(
+ FileUploadValidator, "MAX_FILE_SIZE", 50 * 1024 * 1024
+ ):
+ large_size = 60 * 1024 * 1024 # 60MB, over a patched 50MB cap
+ is_valid, error = FileUploadValidator.validate_file_size(large_size)
assert is_valid is False
assert "60" in error or "MB" in error
@@ -292,10 +350,11 @@ class TestValidateUpload:
def test_upload_file_too_large(self):
"""Rejects upload that's too large."""
- large_content = b"%PDF-1.4" + b"x" * FileUploadValidator.MAX_FILE_SIZE
- is_valid, error = FileUploadValidator.validate_upload(
- "document.pdf", large_content
- )
+ with patch.object(FileUploadValidator, "MAX_FILE_SIZE", 1024):
+ large_content = b"%PDF-1.4" + b"x" * 2048 # 2KB over 1KB cap
+ is_valid, error = FileUploadValidator.validate_upload(
+ "document.pdf", large_content
+ )
assert is_valid is False
assert "too large" in error.lower()
diff --git a/tests/web/test_app_factory.py b/tests/web/test_app_factory.py
index 04148e5a70..1fe0772cf4 100644
--- a/tests/web/test_app_factory.py
+++ b/tests/web/test_app_factory.py
@@ -335,14 +335,24 @@ class TestFileUploadSecurity:
assert validator is not None
def test_max_form_memory_prevents_memory_exhaustion(self):
- """DiskSpoolingRequest prevents memory exhaustion."""
+ """DiskSpoolingRequest prevents memory exhaustion.
+
+ Files larger than the threshold are spooled to disk by Werkzeug
+ instead of being held in RAM — keeping memory bounded even though
+ the per-file cap (FileUploadValidator.MAX_FILE_SIZE) is large.
+ """
+ from local_deep_research.security.file_upload_validator import (
+ FileUploadValidator,
+ )
from local_deep_research.web.app_factory import DiskSpoolingRequest
- # 5MB threshold means files larger than this go to disk
- # This prevents 200 files x 50MB = 10GB memory consumption
+ # 5MB threshold means files larger than this go to disk.
threshold = DiskSpoolingRequest.max_form_memory_size
assert threshold == 5 * 1024 * 1024 # 5MB
- assert threshold < 50 * 1024 * 1024 # Less than 50MB
+ # Sanity: threshold must stay well below the per-file cap, otherwise
+ # large uploads would never spool to disk. Reference the configured
+ # cap directly so the bound scales when the cap is changed.
+ assert threshold < FileUploadValidator.MAX_FILE_SIZE
class TestSecureCookieMiddleware: