~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(security): pass Google Gemini API key via header (#4184) (#4452)
Dongping Guo (Donny)ee499abf1f01
diff --git a/src/local_deep_research/llm/providers/implementations/google.py b/src/local_deep_research/llm/providers/implementations/google.pyindex a6e3b42612..b359671429 100644--- a/src/local_deep_research/llm/providers/implementations/google.py+++ b/src/local_deep_research/llm/providers/implementations/google.py@@ -51,12 +51,21 @@ class GoogleProvider(OpenAICompatibleProvider):try:from ....security import safe_get- # Use the native Gemini API endpoint (not OpenAI-compatible)- # Note: Google's API requires the key as a query parameter, not in headers- # This is their documented approach: https://ai.google.dev/api/rest- url = f"https://generativelanguage.googleapis.com/v1beta/models?key={api_key}"-- response = safe_get(url, timeout=10)+ # Use the native Gemini API endpoint (not OpenAI-compatible).+ # Pass the key via the x-goog-api-key header rather than a+ # ?key=... query parameter so the secret never appears in the+ # request URL — and therefore cannot leak into requests /+ # urllib3 exception messages, which embed the URL but not+ # headers. This is the prevention-by-construction control+ # called for in issue #4184; the redact_secrets() wrap below+ # remains as defense-in-depth.+ url = "https://generativelanguage.googleapis.com/v1beta/models"++ response = safe_get(+ url,+ headers={"x-goog-api-key": api_key},+ timeout=10,+ )if response.status_code == 200:data = response.json()@@ -92,13 +101,15 @@ class GoogleProvider(OpenAICompatibleProvider):return []except Exception as e:- # Google's API requires the key as a ?key=... query- # parameter, so requests/urllib3 exceptions often embed the- # full URL — and therefore the key — in str(e).- # logger.exception would write that to every loguru sink;- # instead, redact and log via logger.warning so the exception- # chain (which also carries the URL in earlier frames) is- # dropped. The redacted message is captured in a local+ # Defense-in-depth: the primary defense is now the+ # x-goog-api-key header above (prevention by construction —+ # the URL no longer carries the key, so requests/urllib3+ # exception messages cannot embed it). This redact_secrets()+ # wrap stays as a second layer in case a future code path+ # reintroduces the key into a logged string, and we keep+ # logger.warning rather than logger.exception so the+ # exception chain (which carries the URL in earlier frames)+ # is dropped. The redacted message is captured in a local# before the logger call so the check-sensitive-logging# pre-commit hook does not flag the exception variable as# referenced inside the log call.diff --git a/tests/llm_providers/implementations/test_google_provider.py b/tests/llm_providers/implementations/test_google_provider.pyindex 4cea3066e7..c198263866 100644--- a/tests/llm_providers/implementations/test_google_provider.py+++ b/tests/llm_providers/implementations/test_google_provider.py@@ -201,8 +201,14 @@ class TestGoogleListModelsForApi:assert len(result) == 1assert result[0]["value"] == "gemini-1.5-pro"- def test_uses_api_key_in_url(self):- """Passes API key as query parameter in URL."""+ def test_passes_api_key_via_header_not_url(self):+ """Passes API key via the ``x-goog-api-key`` header, not as a+ ``?key=...`` query parameter. See issue #4184: the URL is embedded+ in ``requests``/``urllib3`` exception messages, but headers are+ not — so keeping the key out of the URL is the primary+ prevention-by-construction control against credential leakage+ through logs.+ """mock_response = Mock()mock_response.status_code = 200mock_response.json.return_value = {"models": []}@@ -212,10 +218,17 @@ class TestGoogleListModelsForApi:GoogleProvider.list_models_for_api(api_key="my-test-key")- # Verify API key is in the URL- call_args = mock_get.call_args[0]- url = call_args[0]- assert "key=my-test-key" in url+ url = mock_get.call_args[0][0]+ headers = mock_get.call_args[1].get("headers") or {}++ assert "my-test-key" not in url, (+ "API key must not appear in the URL — it would leak "+ "through HTTP exception messages. See issue #4184."+ )+ assert headers.get("x-goog-api-key") == "my-test-key", (+ f"API key must be passed via the x-goog-api-key header. "+ f"Got headers: {headers!r}"+ )def test_uses_native_gemini_api_endpoint(self):"""Uses native Gemini API endpoint, not OpenAI-compatible."""diff --git a/tests/security/test_api_key_leakage.py b/tests/security/test_api_key_leakage.pyindex b07cbd3ff2..c3853adf10 100644--- a/tests/security/test_api_key_leakage.py+++ b/tests/security/test_api_key_leakage.py@@ -13,13 +13,12 @@ providers, with the Google case being the one that previously failed."""import base64-from unittest.mock import patch+from unittest.mock import Mock, patchfrom urllib.parse import quote, quote_plusimport pytestimport requests-# A recognizable sentinel that should never appear in any logged or# returned text after these tests run. Kept URL-safe so it can be# embedded literally in test URLs without confusing `urllib.parse`; the@@ -46,6 +45,30 @@ def _all_encodings_of(secret: str) -> list:]+def _stub_rate_tracker() -> Mock:+ """Return a minimal rate_tracker stub for tests that exercise the+ request-layer catch block but not rate limiting.++ Bypasses rate limiting without pulling in the real+ ``AdaptiveRateLimitTracker`` (which would trigger DB imports and a+ settings snapshot). The leak tests want the catch block to run, not+ the rate-limit code path.++ - ``apply_rate_limit`` returns ``0`` so the engine's+ ``time.sleep(self._last_wait_time)`` is a no-op (keeps the test+ fast). Forgetting this would have ``Mock`` return a truthy+ ``Mock`` instance and either sleep garbage or raise.+ - ``enabled = False`` so any code path that gates on+ ``self.rate_tracker.enabled`` sees the stub as disabled.+ - ``record_event`` is a no-op (the real implementation writes to+ a DB; the stub has no DB to write to).+ """+ tracker = Mock()+ tracker.enabled = False+ tracker.apply_rate_limit.return_value = 0+ return tracker++@pytest.fixturedef google_provider_module():"""Late-import the Google provider so the test's loguru_caplog@@ -57,21 +80,76 @@ def google_provider_module():class TestGoogleProviderKeyLeakage:- """The Google provider's ``list_models_for_api`` builds a URL with the- API key as a query parameter (Google's documented requirement). If the- upstream request raises with the URL in its exception message,- ``logger.exception`` would write the key to logs verbatim. The fix at+ """The Google provider's ``list_models_for_api`` previously built a+ URL with the API key as a query parameter (Google's then-documented+ requirement). If the upstream request raised with the URL in its+ exception message, ``logger.exception`` would write the key to logs+ verbatim. The fix at:file:`src/local_deep_research/llm/providers/implementations/google.py`- redacts the key from the exception string before logging and uses- ``logger.warning`` (no traceback) to keep the cause-chain off the log.+ is two-layered:++ 1. **Prevention by construction (primary defense, issue #4184):** the+ key is now passed via the ``x-goog-api-key`` header, so the URL+ handed to ``requests`` no longer carries the secret. HTTP+ exception messages embed the URL but never the headers.+ 2. **Log-side redaction (defense-in-depth):** the except handler+ still wraps ``str(e)`` with ``redact_secrets(..., api_key)`` and+ uses ``logger.warning`` (no traceback) so any future code path+ that reintroduces the key into a logged string is still scrubbed.++ Both properties are pinned below — the construction-layer test+ fails if a maintainer ever reverts to ``?key=...``; the log-side+ tests fail if the redaction wrap is removed."""+ def test_api_key_is_passed_via_header_not_url(self, google_provider_module):+ """Prevention-by-construction: the URL handed to ``safe_get`` must+ not contain the api_key under any name, and the ``x-goog-api-key``+ header must carry it instead. This is the primary defense from+ issue #4184 — reverting to the old ``?key=`` form would make this+ test fail before any exception-handling code even runs.+ """+ import local_deep_research.security as sec_pkg++ captured = {}++ class _Resp:+ status_code = 200++ def json(self):+ return {"models": []}++ def _capture(url, *args, **kwargs):+ captured["url"] = url+ captured["headers"] = kwargs.get("headers") or {}+ return _Resp()++ with patch.object(sec_pkg, "safe_get", side_effect=_capture):+ google_provider_module.GoogleProvider.list_models_for_api(+ api_key=_LEAKED_KEY+ )++ assert "url" in captured, "safe_get was not called"+ for encoding in _all_encodings_of(_LEAKED_KEY):+ assert encoding not in captured["url"], (+ "API key found in the URL handed to safe_get (as encoding "+ f"{encoding!r}) - The Google provider must pass the key via the"+ " x-goog-api-key header (see issue #4184) so HTTP exception"+ " messages — which embed the URL but not headers — cannot "+ "carry the secret."+ )+ assert captured["headers"].get("x-goog-api-key") == _LEAKED_KEY, (+ "API key must be passed via the x-goog-api-key header. "+ f"Got headers: {captured['headers']!r}"+ )+def test_no_leak_when_safe_get_raises_with_url_in_message(self, loguru_caplog, google_provider_module):- """``ConnectionError`` from urllib3 typically includes the failing- URL in its message. The key embedded as ``?key=...`` must not- survive to the log output.+ """Defense-in-depth: even if some upstream exception still embeds+ the key (e.g., a future code path reintroduces it, or an+ intermediate proxy echoes it back), the except handler's+ ``redact_secrets`` wrap must scrub it from the logged message."""import local_deep_research.security as sec_pkg@@ -433,19 +511,10 @@ class TestSearchEngineKeyLeakage:engine.base_url = "https://api.example.com"engine.engine_type = "test_engine"engine._search_results = None- # Avoid the rate-limit-tracker dependency: stub apply_rate_limit- # to return 0 and engine_type lookup to noop. Tavily/Exa each call- # ``self.rate_tracker.apply_rate_limit(self.engine_type)`` before- # the HTTP request.- engine.rate_tracker = type(- "_T",- (),- {- "enabled": False,- "record_event": lambda *a, **k: None,- "apply_rate_limit": lambda *a, **k: 0,- },- )()+ # Bypass the rate-limit-tracker dependency — see _stub_rate_tracker.+ # Tavily/Exa each call ``self.rate_tracker.apply_rate_limit(...)``+ # before the HTTP request.+ engine.rate_tracker = _stub_rate_tracker()# ``_raise_if_rate_limit`` is a base-class method that inspects# the response status code; with a non-HTTP exception passed in,# it must not itself raise. Stub it to a no-op for safety.@@ -503,15 +572,7 @@ class TestSearchEngineKeyLeakage:engine.min_citations = 0engine.include_arxiv = Trueengine.engine_type = "test_engine"- engine.rate_tracker = type(- "_T",- (),- {- "enabled": False,- "record_event": lambda *a, **k: None,- "apply_rate_limit": lambda *a, **k: 0,- },- )()+ engine.rate_tracker = _stub_rate_tracker()exc = requests.exceptions.ConnectionError(f"HTTPSConnectionPool: Max retries exceeded with url: "@@ -536,3 +597,211 @@ class TestSearchEngineKeyLeakage:"NASA ADS test did not exercise the except branch. "f"Captured logs: {loguru_caplog_full.text!r}")+++class TestSearchEngineParamsKeyLeakage:+ """Pin no-leak behavior for search engines that pass the API key via+ the ``params=`` dict to ``safe_get``. The underlying ``requests``+ library assembles ``params`` into the request URL before the network+ call; when ``requests`` raises (``ConnectionError``, ``Timeout``,+ etc.), the exception's ``__str__()`` includes that assembled URL —+ and therefore the key.++ Each engine catches the exception in its own try/except and is+ protected today by either ``_sanitize_error_message()``+ (regex-based, defined on the base class) and/or+ ``redact_secrets()`` (literal-value substitution). This contract is+ "safe by convention" — every catch site must remember the wrap. A+ future refactor that consolidates error handling into a base method+ without one of those calls would silently leak. The tests below+ pin the no-leak property so such a regression flips the test.++ Note on Guardian specifically: its key parameter is ``api-key`` (with+ a dash), which the base-class regex at ``search_engine_base.py:984``+ does NOT match (the regex covers ``api_key|apikey|key|token|secret``).+ Guardian therefore relies entirely on ``redact_secrets()`` for the+ URL-leak path — making it the most fragile of the four and the most+ important to pin.+ """++ @staticmethod+ def _mojeek_engine():+ from local_deep_research.web_search_engines.engines import (+ search_engine_mojeek as mod,+ )++ engine = mod.MojeekSearchEngine.__new__(mod.MojeekSearchEngine)+ engine.api_key = _LEAKED_KEY+ engine.search_url = "https://api.mojeek.com/search"+ engine.max_results = 10+ engine.safe_search = True+ engine.language = None+ engine.region = None+ return (+ mod,+ engine,+ "_get_search_results",+ "/search?api_key=",+ "Error when searching using Mojeek",+ )++ @staticmethod+ def _scaleserp_engine():+ from local_deep_research.web_search_engines.engines import (+ search_engine_scaleserp as mod,+ )++ engine = mod.ScaleSerpSearchEngine.__new__(mod.ScaleSerpSearchEngine)+ engine.api_key = _LEAKED_KEY+ engine.base_url = "https://api.scaleserp.com/search"+ engine.max_results = 10+ engine.location = "United States"+ engine.language = "en"+ engine.device = "desktop"+ engine.safe_search = False+ engine.enable_cache = False+ engine.engine_type = "test_engine"+ engine._knowledge_graph = None+ engine._related_searches = None+ engine._related_questions = None+ engine._search_results = None+ engine.rate_tracker = _stub_rate_tracker()+ engine._raise_if_rate_limit = lambda *a, **k: None+ return (+ mod,+ engine,+ "_get_previews",+ "/search?api_key=",+ "Error getting ScaleSerp API results",+ )++ @staticmethod+ def _google_pse_engine():+ from local_deep_research.web_search_engines.engines import (+ search_engine_google_pse as mod,+ )++ engine = mod.GooglePSESearchEngine.__new__(mod.GooglePSESearchEngine)+ engine.api_key = _LEAKED_KEY+ engine.search_engine_id = "test-cx"+ engine.max_results = 10+ engine.safe = "off"+ engine.language = "en"+ engine.region = "us"+ engine.max_retries = 1 # single attempt — no real sleep, no real retry+ engine.retry_delay = 0+ engine.engine_type = "test_engine"+ engine.rate_tracker = _stub_rate_tracker()+ # _make_request will re-raise as RequestException after max_retries+ # — callers must catch that. Return the method name to invoke.+ # The retry loop logs each attempt before re-raising, so the log+ # marker is the per-attempt warning from line ~266.+ return (+ mod,+ engine,+ "_make_request",+ "/customsearch/v1?key=",+ "Request error on attempt",+ )++ @staticmethod+ def _guardian_engine():+ from local_deep_research.web_search_engines.engines import (+ search_engine_guardian as mod,+ )++ engine = mod.GuardianSearchEngine.__new__(mod.GuardianSearchEngine)+ engine.api_key = _LEAKED_KEY+ engine.api_url = "https://content.guardianapis.com/search"+ engine.max_results = 10+ engine.from_date = "2024-01-01"+ engine.to_date = "2024-12-31"+ engine.order_by = "relevance"+ engine.section = None+ engine.engine_type = "test_engine"+ engine.rate_tracker = _stub_rate_tracker()+ engine._raise_if_rate_limit = lambda *a, **k: None+ return (+ mod,+ engine,+ "_get_all_data",+ "/search?api-key=",+ "Error getting data from The Guardian API",+ )++ @pytest.mark.parametrize(+ "engine_factory_name",+ [+ "_mojeek_engine",+ "_scaleserp_engine",+ "_google_pse_engine",+ "_guardian_engine",+ ],+ )+ def test_no_leak_when_safe_get_raises_with_assembled_url(+ self, loguru_caplog_full, engine_factory_name+ ):+ """When ``safe_get`` raises a ``ConnectionError`` whose message+ embeds the assembled URL (base + query string with the key from+ ``params=``), the engine's catch block must scrub the key before+ logging.++ Mutation check: temporarily remove the per-engine+ ``redact_secrets()`` / ``_sanitize_error_message()`` call from+ the catch block and the relevant parametrized case will fail.+ """+ factory = getattr(self, engine_factory_name)+ mod, engine, method_name, url_path_with_key, marker = factory()++ # Simulate the exception that requests raises when the network+ # call fails — its message embeds the *assembled* URL with the+ # query string built from `params=`.+ exc = requests.exceptions.ConnectionError(+ f"HTTPSConnectionPool: Max retries exceeded with url: "+ f"{url_path_with_key}{_LEAKED_KEY}&q=test"+ )++ # No create=True: every engine under test imports safe_get at+ # module level, so a typo'd patch target (or a future refactor+ # that drops the import) must fail loudly rather than silently+ # no-op into a passing test.+ with loguru_caplog_full.at_level("DEBUG"):+ with patch.object(+ mod, "safe_get", side_effect=exc+ ) as mock_safe_get:+ # Only swallow the request-layer exception we injected.+ # Google PSE's _make_request re-raises as+ # RequestException after retries; the other three engines+ # catch internally and return []. AttributeError /+ # TypeError from a bad test stub must propagate so the+ # test fails loudly instead of silently skipping the+ # logging/redaction path.+ try:+ getattr(engine, method_name)("test query")+ except requests.exceptions.RequestException:+ pass++ # Sanity: the SUT must have actually called safe_get.+ assert mock_safe_get.called, (+ f"{engine_factory_name}: safe_get was not invoked — the "+ f"redaction path under test never ran. Check that "+ f"{mod.__name__} still imports safe_get at module level."+ )+ # Sanity: the SUT must have actually logged the catch-block warning.+ # Without this, the leak assertion above passes vacuously if the+ # catch block is removed or refactored to silently re-raise.+ assert marker in loguru_caplog_full.text, (+ f"{engine_factory_name}: catch-block log marker {marker!r} not "+ f"emitted — the redaction path under test never ran. Check that "+ f"{mod.__name__} still wraps the safe_get call with a "+ f"logger.warning(...) that includes the redacted message. "+ f"Captured logs: {loguru_caplog_full.text!r}"+ )++ for encoding in _all_encodings_of(_LEAKED_KEY):+ assert encoding not in loguru_caplog_full.text, (+ f"{engine_factory_name}: api_key leaked as encoding "+ f"{encoding!r}. The catch block must scrub the key via "+ f"redact_secrets(str(e), self.api_key) and/or "+ f"self._sanitize_error_message(str(e)) before logging."+ )