~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(library): scrub credentials from download-service errors before they reach the client (#4625)
LearningCircuita4757e192ccf
diff --git a/src/local_deep_research/research_library/downloaders/html.py b/src/local_deep_research/research_library/downloaders/html.pyindex 70f1262e9d..dd509ba4d8 100644--- a/src/local_deep_research/research_library/downloaders/html.py+++ b/src/local_deep_research/research_library/downloaders/html.py@@ -13,6 +13,7 @@ from bs4 import BeautifulSoupfrom .base import BaseDownloader, ContentType, DownloadResultfrom .extraction.pipeline import extract_content_with_metadatafrom ...constants import BROWSER_USER_AGENT+from ...security import sanitize_error_for_clientclass HTMLDownloader(BaseDownloader):@@ -106,7 +107,12 @@ class HTMLDownloader(BaseDownloader):except Exception as e:logger.exception(f"Failed to download HTML from {url}")- return DownloadResult(skip_reason=f"Error: {str(e)}")+ # skip_reason propagates to the browser via the download SSE+ # stream; the fetch URL can carry credentials — scrub before+ # returning (full detail stays in the server log above).+ return DownloadResult(+ skip_reason=sanitize_error_for_client(f"Error: {str(e)}")+ )def _fetch_html(self, url: str) -> Optional[str]:"""Fetch raw HTML content from URL."""diff --git a/src/local_deep_research/research_library/services/download_service.py b/src/local_deep_research/research_library/services/download_service.pyindex e27bbb54bf..7e7bcde0b6 100644--- a/src/local_deep_research/research_library/services/download_service.py+++ b/src/local_deep_research/research_library/services/download_service.py@@ -29,7 +29,7 @@ from ...database.models.download_tracker import (DownloadAttempt,DownloadTracker,)-from ...security import safe_get, sanitize_for_log+from ...security import safe_get, sanitize_error_for_clientfrom ...security.path_validator import PathValidatorfrom ...database.models.library import (Collection,@@ -919,7 +919,10 @@ class DownloadService:logger.exception(f"Download failed for {url}")attempt.succeeded = Falseattempt.error_type = type(e).__name__- attempt.error_message = sanitize_for_log(str(e), max_length=200)+ # Surfaced to the browser via the download SSE stream, and+ # request/network errors can echo a resource URL carrying an+ # api_key/token or user:pass@host — scrub before returning.+ attempt.error_message = sanitize_error_for_client(str(e))tracker.is_accessible = False# Sanitize error message before returning to APIsafe_error = attempt.error_message@@ -1300,8 +1303,8 @@ class DownloadService:except Exception as e:logger.exception(f"Failed to save text for resource {resource.id}")# Sanitize error message before returning to API- safe_error = sanitize_for_log(- f"Failed to save text: {str(e)}", max_length=200+ safe_error = sanitize_error_for_client(+ f"Failed to save text: {str(e)}")return False, safe_error@@ -1381,8 +1384,8 @@ class DownloadService:except Exception as e:logger.exception(f"Failed to save text for resource {resource.id}")# Sanitize error message before returning to API- safe_error = sanitize_for_log(- f"Failed to save text: {str(e)}", max_length=200+ safe_error = sanitize_error_for_client(+ f"Failed to save text: {str(e)}")return False, safe_errordiff --git a/src/local_deep_research/security/__init__.py b/src/local_deep_research/security/__init__.pyindex 2f6ac04e09..5e1f384403 100644--- a/src/local_deep_research/security/__init__.py+++ b/src/local_deep_research/security/__init__.py@@ -35,6 +35,7 @@ from .account_lockout import AccountLockoutManager, get_account_lockout_managerfrom .password_validator import PasswordValidatorfrom .log_sanitizer import (redact_secrets,+ sanitize_error_for_client,sanitize_error_message,sanitize_for_log,strip_control_chars,@@ -100,6 +101,7 @@ __all__ = ["get_account_lockout_manager","PasswordValidator","redact_secrets",+ "sanitize_error_for_client","sanitize_error_message","sanitize_for_log","strip_control_chars",diff --git a/src/local_deep_research/security/log_sanitizer.py b/src/local_deep_research/security/log_sanitizer.pyindex 6bd61091db..5f14e78475 100644--- a/src/local_deep_research/security/log_sanitizer.py+++ b/src/local_deep_research/security/log_sanitizer.py@@ -172,3 +172,20 @@ def sanitize_error_message(message: str) -> str:for pattern, replacement in _CREDENTIAL_PATTERNS:message = pattern.sub(replacement, message)return message+++def sanitize_error_for_client(message: str, max_length: int = 200) -> str:+ """Make an exception-derived string safe to return to an HTTP client.++ Composes :func:`sanitize_error_message` (credential redaction) and+ :func:`sanitize_for_log` (control-char strip + length cap). Credential+ scrubbing runs FIRST, on the full untruncated string, so a secret near+ the ``max_length`` boundary cannot be split by truncation and slip past+ the regexes.++ Use this for any exception text surfaced to the browser (API/JSON/SSE+ responses); keep the raw exception server-side via ``logger.exception``.+ """+ return sanitize_for_log(+ sanitize_error_message(message), max_length=max_length+ )diff --git a/tests/research_library/downloaders/test_html_downloader_deep_coverage.py b/tests/research_library/downloaders/test_html_downloader_deep_coverage.pyindex 70c6e6cc48..c93d04bdbe 100644--- a/tests/research_library/downloaders/test_html_downloader_deep_coverage.py+++ b/tests/research_library/downloaders/test_html_downloader_deep_coverage.py@@ -191,6 +191,24 @@ class TestDownloadWithResult:assert result.is_success is Falseassert "Error" in result.skip_reason+ def test_exception_skip_reason_scrubs_credentials(self, downloader):+ """The exception path's skip_reason propagates to the browser via the+ download SSE stream, so a credential echoed in the error (e.g. a fetch+ URL with an api_key) must be redacted, not leaked. Regression guard+ for the download-service credential-leak fix."""+ secret = "supersecret1234567890"+ boom = ValueError(+ f"fetch failed for https://api.example.com/doc?api_key={secret}"+ )+ with patch.object(downloader, "_fetch_html", side_effect=boom):+ result = downloader.download_with_result(+ "https://api.example.com/doc"+ )+ assert result.is_success is False+ assert secret not in result.skip_reason+ # Still an error message, just with the credential masked.+ assert result.skip_reason.startswith("Error:")+# ---------------------------------------------------------------------------# _fetch_htmldiff --git a/tests/research_library/services/test_download_service_coverage.py b/tests/research_library/services/test_download_service_coverage.pyindex 732f7bace3..d9a1ac34c1 100644--- a/tests/research_library/services/test_download_service_coverage.py+++ b/tests/research_library/services/test_download_service_coverage.py@@ -1162,7 +1162,9 @@ class TestTryApiTextExtraction:patch.object(svc, "_save_text_with_db", side_effect=RuntimeError("db error")),- patch(f"{MODULE}.sanitize_for_log", return_value="safe error"),+ patch(+ f"{MODULE}.sanitize_error_for_client", return_value="safe error"+ ),):result = svc._try_api_text_extraction(session, resource)assert result[0] is False@@ -1271,7 +1273,7 @@ class TestFallbackPdfExtraction:patch.object(svc, "_save_text_with_db", side_effect=RuntimeError("db")),- patch(f"{MODULE}.sanitize_for_log", return_value="safe"),+ patch(f"{MODULE}.sanitize_error_for_client", return_value="safe"),):success, msg = svc._fallback_pdf_extraction(session, resource)assert success is False@@ -2072,7 +2074,9 @@ class TestDownloadPdf:downloader.__class__.__name__ = "DirectPDFDownloader"svc.downloaders = [downloader]- with patch(f"{MODULE}.sanitize_for_log", return_value="safe error"):+ with patch(+ f"{MODULE}.sanitize_error_for_client", return_value="safe error"+ ):success, reason, _status_code = svc._download_pdf(resource, tracker, session)diff --git a/tests/research_library/services/test_download_service_deep_coverage.py b/tests/research_library/services/test_download_service_deep_coverage.pyindex 4f4fe127cf..9452a76c77 100644--- a/tests/research_library/services/test_download_service_deep_coverage.py+++ b/tests/research_library/services/test_download_service_deep_coverage.py@@ -367,7 +367,9 @@ class TestDownloadPdfDeep:side_effect=RuntimeError("db unavailable"),),patch(f"{MODULE}.PDFStorageManager", return_value=mock_storage),- patch(f"{MODULE}.sanitize_for_log", return_value="safe error"),+ patch(+ f"{MODULE}.sanitize_error_for_client", return_value="safe error"+ ),):success, reason, _status_code = svc._download_pdf(resource, tracker, session@@ -469,7 +471,7 @@ class TestDownloadPdfDeep:assert success is True # text extraction failure does not faildef test_outer_exception_caught_and_sanitized(self, svc):- """Exceptions inside the try block are caught and sanitized via sanitize_for_log."""+ """Exceptions inside the try block are caught and sanitized via sanitize_error_for_client."""session = MagicMock()resource = MagicMock()resource.id = 16@@ -493,7 +495,10 @@ class TestDownloadPdfDeep:f"{MODULE}.get_document_for_resource",side_effect=RuntimeError("DB exploded secret_token"),),- patch(f"{MODULE}.sanitize_for_log", return_value="redacted error"),+ patch(+ f"{MODULE}.sanitize_error_for_client",+ return_value="redacted error",+ ),):success, reason, _status_code = svc._download_pdf(resource, tracker, sessiondiff --git a/tests/security/test_log_sanitizer.py b/tests/security/test_log_sanitizer.pyindex e6a44e278d..6601f50afb 100644--- a/tests/security/test_log_sanitizer.py+++ b/tests/security/test_log_sanitizer.py@@ -4,6 +4,7 @@ import pytestfrom local_deep_research.security.log_sanitizer import (redact_secrets,+ sanitize_error_for_client,sanitize_for_log,strip_control_chars,)@@ -203,3 +204,42 @@ class TestRedactSecrets:# to scrub.result = redact_secrets("encoded=%2Bsk-abc12345", "+sk-abc12345")assert "%2Bsk-abc12345" in result+++class TestSanitizeErrorForClient:+ """The client-facing composition used for exception text returned over+ HTTP/SSE (e.g. the library download stream)."""++ def test_redacts_api_key_in_url(self):+ msg = (+ "HTTPError for https://api.example.com/doc?api_key=secret1234567890"+ )+ result = sanitize_error_for_client(msg)+ assert "secret1234567890" not in result++ def test_redacts_url_embedded_credentials(self):+ msg = "ConnectionError: https://alice:hunter2pass@host/file.pdf failed"+ result = sanitize_error_for_client(msg)+ assert "hunter2pass" not in result++ def test_truncates_to_max_length(self):+ result = sanitize_error_for_client("e" * 500, max_length=200)+ assert len(result) <= 200++ def test_credential_scrubbed_before_truncation(self):+ # The whole point of scrub-before-truncate: a key sitting past the+ # max_length boundary must still be redacted, not merely cut off+ # (a truncate-first order would leave the leading chars exposed if+ # the cut landed mid-token).+ key = "sk-abcdefghijklmnopqrstuvwxyz0123456789"+ msg = "x" * 190 + " " + key+ result = sanitize_error_for_client(msg, max_length=400)+ assert key not in result+ assert "sk-abcdefghij" not in result++ def test_importable_from_security_package(self):+ from local_deep_research.security import (+ sanitize_error_for_client as exported,+ )++ assert exported is sanitize_error_for_client