~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
feat(security): bump default upload size cap to 3GB and make it configurable (#4196)
LearningCircuitea5c99ce1849
diff --git a/changelog.d/+upload-max-file-size-3gb.feature.md b/changelog.d/+upload-max-file-size-3gb.feature.mdnew file mode 100644index 0000000000..3e72652a98--- /dev/null+++ b/changelog.d/+upload-max-file-size-3gb.feature.md@@ -0,0 +1 @@+Per-file upload cap (`FileUploadValidator.MAX_FILE_SIZE`) is now configurable and defaults to 3 GB (was 50 MB) so large academic datasets and PDFs fit out of the box. Deployments that want a tighter bound can lower it via the `LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB` environment variable or the `security.upload_max_file_size_mb` setting. Memory usage stays bounded by the existing 5 MB spool-to-disk threshold on multipart requests.diff --git a/src/local_deep_research/security/file_upload_validator.py b/src/local_deep_research/security/file_upload_validator.pyindex 6092e744c6..155f89a5cb 100644--- a/src/local_deep_research/security/file_upload_validator.py+++ b/src/local_deep_research/security/file_upload_validator.py@@ -14,12 +14,63 @@ from typing import Optional, Tupleimport pdfplumberfrom loguru import logger+from ..settings.manager import check_env_setting++# Built-in default for the per-file upload cap, in megabytes.+# Override via the LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB environment variable+# (or its server-config alias `security.upload_max_file_size_mb`).+_DEFAULT_MAX_FILE_SIZE_MB = 3072 # 3 GB+++def _format_size(size_bytes: int) -> str:+ """Render a byte count as ``X.YGB`` when >=1 GB, else ``X.YMB``."""+ one_gb = 1024 * 1024 * 1024+ if size_bytes >= one_gb:+ return f"{size_bytes / one_gb:.1f}GB"+ return f"{size_bytes / (1024 * 1024):.1f}MB"+++def _resolve_max_file_size() -> int:+ """Resolve the per-file upload cap, in bytes.++ Reads ``LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB`` (via+ ``settings.manager.check_env_setting``) so deployments can lower the+ cap without code changes. Falls back to ``_DEFAULT_MAX_FILE_SIZE_MB``+ when the variable is unset or unparseable.+ """+ env_value = check_env_setting("security.upload_max_file_size_mb")+ if env_value is not None:+ try:+ mb = int(env_value)+ except ValueError:+ logger.warning(+ "LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB={!r} is not an integer; "+ "falling back to {} MB default.",+ env_value,+ _DEFAULT_MAX_FILE_SIZE_MB,+ )+ else:+ # Reject zero / negative values — they would silently break+ # all uploads. Fall back to the default and log so the+ # operator can spot the misconfiguration.+ if mb > 0:+ return mb * 1024 * 1024+ logger.warning(+ "LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB={!r} must be > 0; "+ "falling back to {} MB default.",+ env_value,+ _DEFAULT_MAX_FILE_SIZE_MB,+ )+ return _DEFAULT_MAX_FILE_SIZE_MB * 1024 * 1024+class FileUploadValidator:"""Centralized file upload validation for security."""- # Security constants- MAX_FILE_SIZE = 50 * 1024 * 1024 # 50MB per file+ # Security constants. ``MAX_FILE_SIZE`` is loaded from server config so+ # it can be lowered per-deployment via the env var or UI setting; the+ # built-in default is 3 GB (see ``server_config._DEFAULTS``).+ MAX_FILE_SIZE = _resolve_max_file_size()MAX_FILES_PER_REQUEST = 200 # Maximum number of files in single requestPDF_MAGIC_BYTES = b"%PDF" # PDF file signatureALLOWED_MIME_TYPES = {"application/pdf"}@@ -41,22 +92,20 @@ class FileUploadValidator:# Check Content-Length header first (before reading file)if content_length is not None:if content_length > FileUploadValidator.MAX_FILE_SIZE:- size_mb = content_length / (1024 * 1024)- max_mb = FileUploadValidator.MAX_FILE_SIZE / (1024 * 1024)return (False,- f"File too large: {size_mb:.1f}MB (max: {max_mb}MB)",+ f"File too large: {_format_size(content_length)} "+ f"(max: {_format_size(FileUploadValidator.MAX_FILE_SIZE)})",)# Check actual file size if content is providedif file_content is not None:actual_size = len(file_content)if actual_size > FileUploadValidator.MAX_FILE_SIZE:- size_mb = actual_size / (1024 * 1024)- max_mb = FileUploadValidator.MAX_FILE_SIZE / (1024 * 1024)return (False,- f"File too large: {size_mb:.1f}MB (max: {max_mb}MB)",+ f"File too large: {_format_size(actual_size)} "+ f"(max: {_format_size(FileUploadValidator.MAX_FILE_SIZE)})",)return True, Nonediff --git a/src/local_deep_research/web/app_factory.py b/src/local_deep_research/web/app_factory.pyindex 82b2573267..6d5d84cc47 100644--- a/src/local_deep_research/web/app_factory.py+++ b/src/local_deep_research/web/app_factory.py@@ -37,8 +37,10 @@ class DiskSpoolingRequest(Request):files larger than max_form_memory_size to temporary files on disk insteadof keeping them in memory.- Security fix for issue #1176: With 200 files × 50MB limit, the default- behavior could consume 10GB+ of memory per request.+ Security fix for issue #1176: a request right up against the per-file ×+ per-request limits could otherwise consume tens of GB of memory in+ one go. The spool-to-disk threshold keeps memory bounded regardless+ of how high the per-file cap (FileUploadValidator.MAX_FILE_SIZE) is set."""# Files larger than 5MB are spooled to disk instead of memorydiff --git a/tests/pdf_tests/test_file_validator.py b/tests/pdf_tests/test_file_validator.pyindex cd1b80a0aa..64a0ec7272 100644--- a/tests/pdf_tests/test_file_validator.py+++ b/tests/pdf_tests/test_file_validator.py@@ -9,13 +9,20 @@ Tests cover all validation methods:- Comprehensive upload validation"""+from unittest.mock import patch+from local_deep_research.security.file_upload_validator import (FileUploadValidator,)class TestValidateFileSize:- """Tests for validate_file_size method."""+ """Tests for validate_file_size method.++ Oversized-path tests patch ``MAX_FILE_SIZE`` to a small value so they+ can exercise the rejection branch without allocating multi-GB buffers+ in CI.+ """def test_valid_size_via_content_length(self):"""Test that valid size passes via Content-Length header."""@@ -36,19 +43,23 @@ class TestValidateFileSize:def test_oversized_via_content_length(self):"""Test that oversized file is rejected via Content-Length."""- is_valid, error = FileUploadValidator.validate_file_size(- content_length=51 * 1024 * 1024 # 51MB (over 50MB limit)- )+ with patch.object(+ FileUploadValidator, "MAX_FILE_SIZE", 50 * 1024 * 1024+ ):+ is_valid, error = FileUploadValidator.validate_file_size(+ content_length=51 * 1024 * 1024 # 51MB over a 50MB cap+ )assert is_valid is Falseassert error is not Noneassert "too large" in error.lower()def test_oversized_via_file_content(self):"""Test that oversized file is rejected via actual content."""- content = b"x" * (51 * 1024 * 1024) # 51MB (over 50MB limit)- is_valid, error = FileUploadValidator.validate_file_size(- content_length=None, file_content=content- )+ with patch.object(FileUploadValidator, "MAX_FILE_SIZE", 1024):+ content = b"x" * 2048 # 2KB over a 1KB cap+ is_valid, error = FileUploadValidator.validate_file_size(+ content_length=None, file_content=content+ )assert is_valid is Falseassert error is not Noneassert "too large" in error.lower()@@ -232,14 +243,13 @@ class TestValidateUpload:def test_all_validations_run(self):"""Test that all validations are executed."""- # Invalid size should fail first- large_content = b"%PDF-1.4\n" + (- b"x" * (51 * 1024 * 1024)- ) # 51MB (over 50MB limit)- is_valid, error = FileUploadValidator.validate_upload(- filename="test.pdf",- file_content=large_content,- )+ # Invalid size should fail first. Patch the cap to keep memory low.+ with patch.object(FileUploadValidator, "MAX_FILE_SIZE", 1024):+ large_content = b"%PDF-1.4\n" + (b"x" * 2048) # 2KB over 1KB cap+ is_valid, error = FileUploadValidator.validate_upload(+ filename="test.pdf",+ file_content=large_content,+ )assert is_valid is Falseassert "too large" in error.lower()@@ -256,11 +266,12 @@ class TestValidateUpload:def test_returns_first_error(self):"""Test that first validation error stops processing."""# Large file with wrong extension - should fail on size first- large_content = b"x" * (51 * 1024 * 1024) # 51MB (over 50MB limit)- is_valid, error = FileUploadValidator.validate_upload(- filename="test.txt",- file_content=large_content,- )+ with patch.object(FileUploadValidator, "MAX_FILE_SIZE", 1024):+ large_content = b"x" * 2048 # 2KB over 1KB cap+ is_valid, error = FileUploadValidator.validate_upload(+ filename="test.txt",+ file_content=large_content,+ )assert is_valid is False# Should fail on size before checking extensionassert "too large" in error.lower()@@ -269,9 +280,9 @@ class TestValidateUpload:class TestConstants:"""Tests for class constants."""- def test_max_file_size_is_50mb(self):- """Verify MAX_FILE_SIZE is 50MB."""- expected = 50 * 1024 * 1024+ def test_max_file_size_default_is_3gb(self):+ """Default MAX_FILE_SIZE matches the 3 GB server-config default."""+ expected = 3 * 1024 * 1024 * 1024assert FileUploadValidator.MAX_FILE_SIZE == expecteddef test_max_files_is_200(self):diff --git a/tests/security/test_file_upload_validator.py b/tests/security/test_file_upload_validator.pyindex b65a04eba5..2ff391a731 100644--- a/tests/security/test_file_upload_validator.py+++ b/tests/security/test_file_upload_validator.py@@ -13,10 +13,9 @@ class TestFileUploadValidatorConstants:"""Tests for FileUploadValidator constants."""def test_max_file_size_defined(self):- """MAX_FILE_SIZE is defined and reasonable."""+ """MAX_FILE_SIZE is defined and matches the 3 GB server-config default."""assert FileUploadValidator.MAX_FILE_SIZE > 0- # 50MB default- assert FileUploadValidator.MAX_FILE_SIZE == 50 * 1024 * 1024+ assert FileUploadValidator.MAX_FILE_SIZE == 3 * 1024 * 1024 * 1024def test_max_files_per_request_defined(self):"""MAX_FILES_PER_REQUEST is defined and reasonable."""@@ -32,6 +31,57 @@ class TestFileUploadValidatorConstants:assert "application/pdf" in FileUploadValidator.ALLOWED_MIME_TYPES+class TestResolveMaxFileSize:+ """The per-file cap honors LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB."""++ def test_default_when_env_unset(self):+ from local_deep_research.security import file_upload_validator as mod++ with patch.object(mod, "check_env_setting", return_value=None):+ assert (+ mod._resolve_max_file_size()+ == mod._DEFAULT_MAX_FILE_SIZE_MB * 1024 * 1024+ )++ def test_env_value_overrides_default(self):+ from local_deep_research.security import file_upload_validator as mod++ # 750MB override (env vars always arrive as strings).+ with patch.object(mod, "check_env_setting", return_value="750"):+ assert mod._resolve_max_file_size() == 750 * 1024 * 1024++ def test_unparseable_env_falls_back_to_default(self):+ from local_deep_research.security import file_upload_validator as mod++ with patch.object(+ mod, "check_env_setting", return_value="not-a-number"+ ):+ assert (+ mod._resolve_max_file_size()+ == mod._DEFAULT_MAX_FILE_SIZE_MB * 1024 * 1024+ )++ def test_zero_env_value_falls_back_to_default(self):+ """A misconfigured cap of 0 would silently break all uploads."""+ from local_deep_research.security import file_upload_validator as mod++ with patch.object(mod, "check_env_setting", return_value="0"):+ assert (+ mod._resolve_max_file_size()+ == mod._DEFAULT_MAX_FILE_SIZE_MB * 1024 * 1024+ )++ def test_negative_env_value_falls_back_to_default(self):+ """Negative values are nonsensical for a size cap — reject them."""+ from local_deep_research.security import file_upload_validator as mod++ with patch.object(mod, "check_env_setting", return_value="-1"):+ assert (+ mod._resolve_max_file_size()+ == mod._DEFAULT_MAX_FILE_SIZE_MB * 1024 * 1024+ )++class TestValidateFileSize:"""Tests for FileUploadValidator.validate_file_size()."""@@ -56,11 +106,16 @@ class TestValidateFileSize:assert "too large" in error.lower()def test_file_content_exceeds_max(self):- """Rejects file content exceeding max."""- large_content = b"x" * (FileUploadValidator.MAX_FILE_SIZE + 1)- is_valid, error = FileUploadValidator.validate_file_size(- None, large_content- )+ """Rejects file content exceeding max.++ Patches the cap to a small value so the test doesn't have to+ allocate gigabytes to cross the real default.+ """+ with patch.object(FileUploadValidator, "MAX_FILE_SIZE", 1024):+ large_content = b"x" * 2048 # 2KB over a 1KB cap+ is_valid, error = FileUploadValidator.validate_file_size(+ None, large_content+ )assert is_valid is Falseassert "too large" in error.lower()@@ -78,8 +133,11 @@ class TestValidateFileSize:def test_error_message_includes_size(self):"""Error message includes file size information."""- large_size = 60 * 1024 * 1024 # 60MB- is_valid, error = FileUploadValidator.validate_file_size(large_size)+ with patch.object(+ FileUploadValidator, "MAX_FILE_SIZE", 50 * 1024 * 1024+ ):+ large_size = 60 * 1024 * 1024 # 60MB, over a patched 50MB cap+ is_valid, error = FileUploadValidator.validate_file_size(large_size)assert is_valid is Falseassert "60" in error or "MB" in error@@ -292,10 +350,11 @@ class TestValidateUpload:def test_upload_file_too_large(self):"""Rejects upload that's too large."""- large_content = b"%PDF-1.4" + b"x" * FileUploadValidator.MAX_FILE_SIZE- is_valid, error = FileUploadValidator.validate_upload(- "document.pdf", large_content- )+ with patch.object(FileUploadValidator, "MAX_FILE_SIZE", 1024):+ large_content = b"%PDF-1.4" + b"x" * 2048 # 2KB over 1KB cap+ is_valid, error = FileUploadValidator.validate_upload(+ "document.pdf", large_content+ )assert is_valid is Falseassert "too large" in error.lower()diff --git a/tests/web/test_app_factory.py b/tests/web/test_app_factory.pyindex 04148e5a70..1fe0772cf4 100644--- a/tests/web/test_app_factory.py+++ b/tests/web/test_app_factory.py@@ -335,14 +335,24 @@ class TestFileUploadSecurity:assert validator is not Nonedef test_max_form_memory_prevents_memory_exhaustion(self):- """DiskSpoolingRequest prevents memory exhaustion."""+ """DiskSpoolingRequest prevents memory exhaustion.++ Files larger than the threshold are spooled to disk by Werkzeug+ instead of being held in RAM — keeping memory bounded even though+ the per-file cap (FileUploadValidator.MAX_FILE_SIZE) is large.+ """+ from local_deep_research.security.file_upload_validator import (+ FileUploadValidator,+ )from local_deep_research.web.app_factory import DiskSpoolingRequest- # 5MB threshold means files larger than this go to disk- # This prevents 200 files x 50MB = 10GB memory consumption+ # 5MB threshold means files larger than this go to disk.threshold = DiskSpoolingRequest.max_form_memory_sizeassert threshold == 5 * 1024 * 1024 # 5MB- assert threshold < 50 * 1024 * 1024 # Less than 50MB+ # Sanity: threshold must stay well below the per-file cap, otherwise+ # large uploads would never spool to disk. Reference the configured+ # cap directly so the bound scales when the cap is changed.+ assert threshold < FileUploadValidator.MAX_FILE_SIZEclass TestSecureCookieMiddleware: