Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(library): scrub credentials from download-service errors before they reach the client (#4625)

LearningCircuita4757e192ccf
diff --git a/src/local_deep_research/research_library/downloaders/html.py b/src/local_deep_research/research_library/downloaders/html.py
index 70f1262e9d..dd509ba4d8 100644
--- a/src/local_deep_research/research_library/downloaders/html.py
+++ b/src/local_deep_research/research_library/downloaders/html.py
@@ -13,6 +13,7 @@ from bs4 import BeautifulSoup
from .base import BaseDownloader, ContentType, DownloadResult
from .extraction.pipeline import extract_content_with_metadata
from ...constants import BROWSER_USER_AGENT
+from ...security import sanitize_error_for_client
class HTMLDownloader(BaseDownloader):
@@ -106,7 +107,12 @@ class HTMLDownloader(BaseDownloader):
except Exception as e:
logger.exception(f"Failed to download HTML from {url}")
- return DownloadResult(skip_reason=f"Error: {str(e)}")
+ # skip_reason propagates to the browser via the download SSE
+ # stream; the fetch URL can carry credentials — scrub before
+ # returning (full detail stays in the server log above).
+ return DownloadResult(
+ skip_reason=sanitize_error_for_client(f"Error: {str(e)}")
+ )
def _fetch_html(self, url: str) -> Optional[str]:
"""Fetch raw HTML content from URL."""
diff --git a/src/local_deep_research/research_library/services/download_service.py b/src/local_deep_research/research_library/services/download_service.py
index e27bbb54bf..7e7bcde0b6 100644
--- a/src/local_deep_research/research_library/services/download_service.py
+++ b/src/local_deep_research/research_library/services/download_service.py
@@ -29,7 +29,7 @@ from ...database.models.download_tracker import (
DownloadAttempt,
DownloadTracker,
)
-from ...security import safe_get, sanitize_for_log
+from ...security import safe_get, sanitize_error_for_client
from ...security.path_validator import PathValidator
from ...database.models.library import (
Collection,
@@ -919,7 +919,10 @@ class DownloadService:
logger.exception(f"Download failed for {url}")
attempt.succeeded = False
attempt.error_type = type(e).__name__
- attempt.error_message = sanitize_for_log(str(e), max_length=200)
+ # Surfaced to the browser via the download SSE stream, and
+ # request/network errors can echo a resource URL carrying an
+ # api_key/token or user:pass@host — scrub before returning.
+ attempt.error_message = sanitize_error_for_client(str(e))
tracker.is_accessible = False
# Sanitize error message before returning to API
safe_error = attempt.error_message
@@ -1300,8 +1303,8 @@ class DownloadService:
except Exception as e:
logger.exception(f"Failed to save text for resource {resource.id}")
# Sanitize error message before returning to API
- safe_error = sanitize_for_log(
- f"Failed to save text: {str(e)}", max_length=200
+ safe_error = sanitize_error_for_client(
+ f"Failed to save text: {str(e)}"
)
return False, safe_error
@@ -1381,8 +1384,8 @@ class DownloadService:
except Exception as e:
logger.exception(f"Failed to save text for resource {resource.id}")
# Sanitize error message before returning to API
- safe_error = sanitize_for_log(
- f"Failed to save text: {str(e)}", max_length=200
+ safe_error = sanitize_error_for_client(
+ f"Failed to save text: {str(e)}"
)
return False, safe_error
diff --git a/src/local_deep_research/security/__init__.py b/src/local_deep_research/security/__init__.py
index 2f6ac04e09..5e1f384403 100644
--- a/src/local_deep_research/security/__init__.py
+++ b/src/local_deep_research/security/__init__.py
@@ -35,6 +35,7 @@ from .account_lockout import AccountLockoutManager, get_account_lockout_manager
from .password_validator import PasswordValidator
from .log_sanitizer import (
redact_secrets,
+ sanitize_error_for_client,
sanitize_error_message,
sanitize_for_log,
strip_control_chars,
@@ -100,6 +101,7 @@ __all__ = [
"get_account_lockout_manager",
"PasswordValidator",
"redact_secrets",
+ "sanitize_error_for_client",
"sanitize_error_message",
"sanitize_for_log",
"strip_control_chars",
diff --git a/src/local_deep_research/security/log_sanitizer.py b/src/local_deep_research/security/log_sanitizer.py
index 6bd61091db..5f14e78475 100644
--- a/src/local_deep_research/security/log_sanitizer.py
+++ b/src/local_deep_research/security/log_sanitizer.py
@@ -172,3 +172,20 @@ def sanitize_error_message(message: str) -> str:
for pattern, replacement in _CREDENTIAL_PATTERNS:
message = pattern.sub(replacement, message)
return message
+
+
+def sanitize_error_for_client(message: str, max_length: int = 200) -> str:
+ """Make an exception-derived string safe to return to an HTTP client.
+
+ Composes :func:`sanitize_error_message` (credential redaction) and
+ :func:`sanitize_for_log` (control-char strip + length cap). Credential
+ scrubbing runs FIRST, on the full untruncated string, so a secret near
+ the ``max_length`` boundary cannot be split by truncation and slip past
+ the regexes.
+
+ Use this for any exception text surfaced to the browser (API/JSON/SSE
+ responses); keep the raw exception server-side via ``logger.exception``.
+ """
+ return sanitize_for_log(
+ sanitize_error_message(message), max_length=max_length
+ )
diff --git a/tests/research_library/downloaders/test_html_downloader_deep_coverage.py b/tests/research_library/downloaders/test_html_downloader_deep_coverage.py
index 70c6e6cc48..c93d04bdbe 100644
--- a/tests/research_library/downloaders/test_html_downloader_deep_coverage.py
+++ b/tests/research_library/downloaders/test_html_downloader_deep_coverage.py
@@ -191,6 +191,24 @@ class TestDownloadWithResult:
assert result.is_success is False
assert "Error" in result.skip_reason
+ def test_exception_skip_reason_scrubs_credentials(self, downloader):
+ """The exception path's skip_reason propagates to the browser via the
+ download SSE stream, so a credential echoed in the error (e.g. a fetch
+ URL with an api_key) must be redacted, not leaked. Regression guard
+ for the download-service credential-leak fix."""
+ secret = "supersecret1234567890"
+ boom = ValueError(
+ f"fetch failed for https://api.example.com/doc?api_key={secret}"
+ )
+ with patch.object(downloader, "_fetch_html", side_effect=boom):
+ result = downloader.download_with_result(
+ "https://api.example.com/doc"
+ )
+ assert result.is_success is False
+ assert secret not in result.skip_reason
+ # Still an error message, just with the credential masked.
+ assert result.skip_reason.startswith("Error:")
+
# ---------------------------------------------------------------------------
# _fetch_html
diff --git a/tests/research_library/services/test_download_service_coverage.py b/tests/research_library/services/test_download_service_coverage.py
index 732f7bace3..d9a1ac34c1 100644
--- a/tests/research_library/services/test_download_service_coverage.py
+++ b/tests/research_library/services/test_download_service_coverage.py
@@ -1162,7 +1162,9 @@ class TestTryApiTextExtraction:
patch.object(
svc, "_save_text_with_db", side_effect=RuntimeError("db error")
),
- patch(f"{MODULE}.sanitize_for_log", return_value="safe error"),
+ patch(
+ f"{MODULE}.sanitize_error_for_client", return_value="safe error"
+ ),
):
result = svc._try_api_text_extraction(session, resource)
assert result[0] is False
@@ -1271,7 +1273,7 @@ class TestFallbackPdfExtraction:
patch.object(
svc, "_save_text_with_db", side_effect=RuntimeError("db")
),
- patch(f"{MODULE}.sanitize_for_log", return_value="safe"),
+ patch(f"{MODULE}.sanitize_error_for_client", return_value="safe"),
):
success, msg = svc._fallback_pdf_extraction(session, resource)
assert success is False
@@ -2072,7 +2074,9 @@ class TestDownloadPdf:
downloader.__class__.__name__ = "DirectPDFDownloader"
svc.downloaders = [downloader]
- with patch(f"{MODULE}.sanitize_for_log", return_value="safe error"):
+ with patch(
+ f"{MODULE}.sanitize_error_for_client", return_value="safe error"
+ ):
success, reason, _status_code = svc._download_pdf(
resource, tracker, session
)
diff --git a/tests/research_library/services/test_download_service_deep_coverage.py b/tests/research_library/services/test_download_service_deep_coverage.py
index 4f4fe127cf..9452a76c77 100644
--- a/tests/research_library/services/test_download_service_deep_coverage.py
+++ b/tests/research_library/services/test_download_service_deep_coverage.py
@@ -367,7 +367,9 @@ class TestDownloadPdfDeep:
side_effect=RuntimeError("db unavailable"),
),
patch(f"{MODULE}.PDFStorageManager", return_value=mock_storage),
- patch(f"{MODULE}.sanitize_for_log", return_value="safe error"),
+ patch(
+ f"{MODULE}.sanitize_error_for_client", return_value="safe error"
+ ),
):
success, reason, _status_code = svc._download_pdf(
resource, tracker, session
@@ -469,7 +471,7 @@ class TestDownloadPdfDeep:
assert success is True # text extraction failure does not fail
def test_outer_exception_caught_and_sanitized(self, svc):
- """Exceptions inside the try block are caught and sanitized via sanitize_for_log."""
+ """Exceptions inside the try block are caught and sanitized via sanitize_error_for_client."""
session = MagicMock()
resource = MagicMock()
resource.id = 16
@@ -493,7 +495,10 @@ class TestDownloadPdfDeep:
f"{MODULE}.get_document_for_resource",
side_effect=RuntimeError("DB exploded secret_token"),
),
- patch(f"{MODULE}.sanitize_for_log", return_value="redacted error"),
+ patch(
+ f"{MODULE}.sanitize_error_for_client",
+ return_value="redacted error",
+ ),
):
success, reason, _status_code = svc._download_pdf(
resource, tracker, session
diff --git a/tests/security/test_log_sanitizer.py b/tests/security/test_log_sanitizer.py
index e6a44e278d..6601f50afb 100644
--- a/tests/security/test_log_sanitizer.py
+++ b/tests/security/test_log_sanitizer.py
@@ -4,6 +4,7 @@ import pytest
from local_deep_research.security.log_sanitizer import (
redact_secrets,
+ sanitize_error_for_client,
sanitize_for_log,
strip_control_chars,
)
@@ -203,3 +204,42 @@ class TestRedactSecrets:
# to scrub.
result = redact_secrets("encoded=%2Bsk-abc12345", "+sk-abc12345")
assert "%2Bsk-abc12345" in result
+
+
+class TestSanitizeErrorForClient:
+ """The client-facing composition used for exception text returned over
+ HTTP/SSE (e.g. the library download stream)."""
+
+ def test_redacts_api_key_in_url(self):
+ msg = (
+ "HTTPError for https://api.example.com/doc?api_key=secret1234567890"
+ )
+ result = sanitize_error_for_client(msg)
+ assert "secret1234567890" not in result
+
+ def test_redacts_url_embedded_credentials(self):
+ msg = "ConnectionError: https://alice:hunter2pass@host/file.pdf failed"
+ result = sanitize_error_for_client(msg)
+ assert "hunter2pass" not in result
+
+ def test_truncates_to_max_length(self):
+ result = sanitize_error_for_client("e" * 500, max_length=200)
+ assert len(result) <= 200
+
+ def test_credential_scrubbed_before_truncation(self):
+ # The whole point of scrub-before-truncate: a key sitting past the
+ # max_length boundary must still be redacted, not merely cut off
+ # (a truncate-first order would leave the leading chars exposed if
+ # the cut landed mid-token).
+ key = "sk-abcdefghijklmnopqrstuvwxyz0123456789"
+ msg = "x" * 190 + " " + key
+ result = sanitize_error_for_client(msg, max_length=400)
+ assert key not in result
+ assert "sk-abcdefghij" not in result
+
+ def test_importable_from_security_package(self):
+ from local_deep_research.security import (
+ sanitize_error_for_client as exported,
+ )
+
+ assert exported is sanitize_error_for_client