~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(search): scrub residual exception outputs (#5370)
Ziyang Guo03533e6ebe96
diff --git a/changelog.d/+web-search-exception-scrubbing.security.md b/changelog.d/+web-search-exception-scrubbing.security.mdnew file mode 100644index 0000000000..0ca0a2ce5e--- /dev/null+++ b/changelog.d/+web-search-exception-scrubbing.security.md@@ -0,0 +1 @@+Scrub credential-shaped data from residual web-search exception logs and engine-loading error messages.diff --git a/src/local_deep_research/web_search_engines/rate_limiting/llm/detection.py b/src/local_deep_research/web_search_engines/rate_limiting/llm/detection.pyindex 7dcceeb71a..48ad2d41a0 100644--- a/src/local_deep_research/web_search_engines/rate_limiting/llm/detection.py+++ b/src/local_deep_research/web_search_engines/rate_limiting/llm/detection.py@@ -2,6 +2,7 @@LLM-specific rate limit error detection."""+from ....security.log_sanitizer import scrub_errorfrom ....security.secure_logging import logger@@ -21,7 +22,9 @@ def is_llm_rate_limit_error(error: BaseException) -> bool:# Check for explicit HTTP 429 status codesif hasattr(error, "response") and hasattr(error.response, "status_code"):if error.response.status_code == 429:- logger.debug(f"Detected HTTP 429 rate limit error: {error}")+ logger.debug(+ f"Detected HTTP 429 rate limit error: {scrub_error(error)}"+ )return True# Check for common rate limit error messages@@ -40,7 +43,9 @@ def is_llm_rate_limit_error(error: BaseException) -> bool:]if any(indicator in error_str for indicator in rate_limit_indicators):- logger.debug(f"Detected rate limit error from message: {error}")+ logger.debug(+ f"Detected rate limit error from message: {scrub_error(error)}"+ )return True# Check for specific provider error types@@ -51,7 +56,9 @@ def is_llm_rate_limit_error(error: BaseException) -> bool:# Check for OpenAI specific rate limit errorsif hasattr(error, "__class__") and error.__class__.__module__ == "openai":if error_type in ["ratelimiterror", "apierror"] and "429" in error_str:- logger.debug(f"Detected OpenAI rate limit error: {error}")+ logger.debug(+ f"Detected OpenAI rate limit error: {scrub_error(error)}"+ )return True# Check for Anthropic specific rate limit errors@@ -60,7 +67,9 @@ def is_llm_rate_limit_error(error: BaseException) -> bool:and "anthropic" in error.__class__.__module__):if any(x in error_str for x in ["rate_limit", "429", "too many"]):- logger.debug(f"Detected Anthropic rate limit error: {error}")+ logger.debug(+ f"Detected Anthropic rate limit error: {scrub_error(error)}"+ )return Truereturn Falsediff --git a/src/local_deep_research/web_search_engines/search_engine_base.py b/src/local_deep_research/web_search_engines/search_engine_base.pyindex 63e3fff8c6..b3c77cbbaa 100644--- a/src/local_deep_research/web_search_engines/search_engine_base.py+++ b/src/local_deep_research/web_search_engines/search_engine_base.py@@ -269,10 +269,15 @@ class BaseSearchEngine(ABC):return (False,None,- f"Security error loading engine class for {name}: {e}",+ f"Security error loading engine class for {name}: "+ f"{scrub_error(e)}",)except Exception as e:- return False, None, f"Could not load engine class for {name}: {e}"+ return (+ False,+ None,+ f"Could not load engine class for {name}: {scrub_error(e)}",+ )@classmethoddef is_available(diff --git a/src/local_deep_research/web_search_engines/search_engines_config.py b/src/local_deep_research/web_search_engines/search_engines_config.pyindex 20c4b5c18b..7f00d48caa 100644--- a/src/local_deep_research/web_search_engines/search_engines_config.py+++ b/src/local_deep_research/web_search_engines/search_engines_config.py@@ -9,6 +9,7 @@ from sqlalchemy.orm import Sessionfrom ..security.secure_logging import loggerfrom ..config.thread_settings import get_setting_from_snapshot+from ..security.log_sanitizer import scrub_errorfrom ..utilities.db_utils import get_settings_managerfrom .search_engine_base import _is_api_key_placeholder@@ -40,7 +41,9 @@ def _get_setting(key, default_value, settings_snapshot=settings_snapshot)except Exception as e:- logger.debug(f"Could not get setting {key} from snapshot: {e}")+ logger.debug(+ f"Could not get setting {key} from snapshot: {scrub_error(e)}"+ )# Try database session if availableif db_session:@@ -48,7 +51,9 @@ def _get_setting(settings_manager = get_settings_manager(db_session, username)return settings_manager.get_setting(key, default_value)except Exception as e:- logger.debug(f"Could not get setting {key} from db_session: {e}")+ logger.debug(+ f"Could not get setting {key} from db_session: {scrub_error(e)}"+ )# Return default if all methods faillogger.warning(diff --git a/tests/security/test_web_search_exception_scrubbing.py b/tests/security/test_web_search_exception_scrubbing.pynew file mode 100644index 0000000000..63dae07334--- /dev/null+++ b/tests/security/test_web_search_exception_scrubbing.py@@ -0,0 +1,97 @@+"""Regression tests for credential scrubbing in web-search error paths."""++from types import SimpleNamespace+from unittest.mock import patch++import pytest++from local_deep_research.security.module_whitelist import ModuleNotAllowedError+from local_deep_research.web_search_engines.rate_limiting.llm.detection import (+ is_llm_rate_limit_error,+)+from local_deep_research.web_search_engines.search_engine_base import (+ BaseSearchEngine,+)+from local_deep_research.web_search_engines.search_engines_config import (+ _get_setting,+)+++SECRET = "sk-test-secret-123456789"+ERROR_TEXT = f"rate limit Authorization: Bearer {SECRET}"+++def _assert_scrubbed(message):+ assert SECRET not in message+ assert "[REDACTED]" in message++[email protected]("http_429", [False, True])+def test_rate_limit_detection_scrubs_exception_from_debug_log(http_429):+ error = RuntimeError(ERROR_TEXT)+ if http_429:+ error.response = SimpleNamespace(status_code=429)++ with patch(+ "local_deep_research.web_search_engines.rate_limiting.llm.detection.logger"+ ) as logger:+ assert is_llm_rate_limit_error(error) is True++ logger.debug.assert_called_once()+ _assert_scrubbed(logger.debug.call_args.args[0])+++ "error",+ [+ ModuleNotAllowedError(ERROR_TEXT),+ RuntimeError(ERROR_TEXT),+ ],+)+def test_engine_class_loading_scrubs_returned_error(error):+ config = {+ "module_path": ".engines.search_engine_wikipedia",+ "class_name": "WikipediaSearchEngine",+ }++ with patch(+ "local_deep_research.security.module_whitelist.get_safe_module_class",+ side_effect=error,+ ):+ success, engine_class, message = BaseSearchEngine._load_engine_class(+ "wikipedia", config+ )++ assert success is False+ assert engine_class is None+ _assert_scrubbed(message)++[email protected]("source", ["snapshot", "database"])+def test_setting_fallback_scrubs_exception_from_debug_log(source):+ call_kwargs = {}+ patch_target = (+ "local_deep_research.web_search_engines.search_engines_config."+ "get_setting_from_snapshot"+ )+ if source == "snapshot":+ call_kwargs["settings_snapshot"] = {"search.test": "value"}+ else:+ call_kwargs["db_session"] = object()+ patch_target = (+ "local_deep_research.web_search_engines.search_engines_config."+ "get_settings_manager"+ )++ with (+ patch(patch_target, side_effect=RuntimeError(ERROR_TEXT)),+ patch(+ "local_deep_research.web_search_engines.search_engines_config.logger"+ ) as logger,+ ):+ assert (+ _get_setting("search.test", "default", **call_kwargs) == "default"+ )++ logger.debug.assert_called_once()+ _assert_scrubbed(logger.debug.call_args.args[0])