~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(google-pse): share rate limit across engine instances (#5359)
Ziyang Guo86c8f3b4a857
diff --git a/changelog.d/+google-pse-shared-rate-limit.bugfix.md b/changelog.d/+google-pse-shared-rate-limit.bugfix.mdnew file mode 100644index 0000000000..8221066bee--- /dev/null+++ b/changelog.d/+google-pse-shared-rate-limit.bugfix.md@@ -0,0 +1,3 @@+Google Programmable Search now shares its minimum request-spacing state across+engine instances using the same API key and CSE ID, so research-agent tool+calls cannot bypass throttling by constructing a fresh engine per request.diff --git a/src/local_deep_research/web_search_engines/engines/_google_pse_rate_limiter.py b/src/local_deep_research/web_search_engines/engines/_google_pse_rate_limiter.pynew file mode 100644index 0000000000..caebe80ac1--- /dev/null+++ b/src/local_deep_research/web_search_engines/engines/_google_pse_rate_limiter.py@@ -0,0 +1,59 @@+"""Process-wide minimum request spacing for Google PSE engine instances."""++import secrets+import threading+import time++_meta_lock = threading.Lock()+_scope_locks: dict[str, threading.Lock] = {}+_scope_last_request: dict[str, float] = {}+_scope_nonce = secrets.token_urlsafe(32)+++def _scope_key(api_key: str, search_engine_id: str) -> str:+ """Return a process-local opaque ID without retaining raw credentials.++ Python string hashing is process-keyed SipHash. Two independently ordered+ tuple hashes plus a random nonce give the in-memory limiter a compact scope+ identity without treating API credentials as password-verification data.+ """+ mask = (1 << 64) - 1+ first = hash((_scope_nonce, api_key, search_engine_id)) & mask+ second = hash((search_engine_id, api_key, _scope_nonce)) & mask+ return f"{first:016x}{second:016x}"+++def _get_scope_lock(scope: str) -> threading.Lock:+ with _meta_lock:+ lock = _scope_locks.get(scope)+ if lock is None:+ lock = threading.Lock()+ _scope_locks[scope] = lock+ return lock+++def respect_rate_limit(+ api_key: str,+ search_engine_id: str,+ interval_seconds: float,+) -> float:+ """Sleep as needed to reserve a request slot for one PSE configuration."""+ if interval_seconds <= 0:+ return 0.0++ scope = _scope_key(api_key, search_engine_id)+ lock = _get_scope_lock(scope)+ with lock:+ elapsed = time.monotonic() - _scope_last_request.get(scope, 0.0)+ wait_time = max(0.0, interval_seconds - elapsed)+ if wait_time > 0:+ time.sleep(wait_time)+ _scope_last_request[scope] = time.monotonic()+ return wait_time+++def reset_for_tests() -> None:+ """Clear process-wide state. Intended for unit tests only."""+ with _meta_lock:+ _scope_locks.clear()+ _scope_last_request.clear()diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_google_pse.py b/src/local_deep_research/web_search_engines/engines/search_engine_google_pse.pyindex 7ab4cc9d75..2e67e95b1f 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_google_pse.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_google_pse.py@@ -9,6 +9,7 @@ from requests.exceptions import RequestExceptionfrom ...security.safe_requests import safe_getfrom ..rate_limiting import RateLimitErrorfrom ..search_engine_base import BaseSearchEngine, Exposure, Sensitivity+from ._google_pse_rate_limiter import respect_rate_limitclass GooglePSESearchEngine(BaseSearchEngine):@@ -71,8 +72,6 @@ class GooglePSESearchEngine(BaseSearchEngine):self.max_retries = max_retriesself.retry_delay = retry_delay- # Rate limiting - keep track of last request time- self.last_request_time: float = 0.0self.min_request_interval = (0.5 # Minimum time between requests in seconds)@@ -171,19 +170,20 @@ class GooglePSESearchEngine(BaseSearchEngine):)raise type(e)(safe_msg) from None- def _respect_rate_limit(self):- """Ensure we don't exceed rate limits by adding appropriate delay between requests"""- current_time = time.time()- elapsed = current_time - self.last_request_time-- # If we've made a request recently, wait until the minimum interval has passed- if elapsed < self.min_request_interval:- sleep_time = self.min_request_interval - elapsed- logger.debug("Rate limiting: sleeping for {:.2f} s", sleep_time)- time.sleep(sleep_time)-- # Update the last request time- self.last_request_time = time.time()+ def _respect_rate_limit(self) -> float:+ """Enforce minimum spacing across engines using the same credentials."""+ api_key = self.api_key+ search_engine_id = self.search_engine_id+ if api_key is None or search_engine_id is None:+ raise ValueError("Google PSE credentials are required")+ wait_time = respect_rate_limit(+ api_key,+ search_engine_id,+ self.min_request_interval,+ )+ if wait_time > 0:+ logger.debug("Rate limiting: sleeping for {:.2f} s", wait_time)+ return wait_timedef _make_request(self, query: str, start_index: int = 1) -> Dict:"""@@ -243,9 +243,11 @@ class GooglePSESearchEngine(BaseSearchEngine):start_index,)# Apply rate limiting before request- self._last_wait_time = self.rate_tracker.apply_rate_limit(+ adaptive_wait = self.rate_tracker.apply_rate_limit(self.engine_type)+ minimum_interval_wait = self._respect_rate_limit()+ self._last_wait_time = adaptive_wait + minimum_interval_waitresponse = safe_get(url, params=params, timeout=10)diff --git a/tests/search_engines/test_search_engine_google_pse.py b/tests/search_engines/test_search_engine_google_pse.pyindex 60e39c14be..7c03d63104 100644--- a/tests/search_engines/test_search_engine_google_pse.py+++ b/tests/search_engines/test_search_engine_google_pse.py@@ -264,8 +264,8 @@ class TestGooglePSERateLimiting:assert hasattr(engine, "min_request_interval")assert engine.min_request_interval == 0.5- def test_last_request_time_initialized(self):- """Test that last request time is initialized."""+ def test_last_request_time_not_stored_per_instance(self):+ """Rate-limit timestamps live in the process-wide tracker."""from local_deep_research.web_search_engines.engines.search_engine_google_pse import (GooglePSESearchEngine,)@@ -273,8 +273,7 @@ class TestGooglePSERateLimiting:engine = GooglePSESearchEngine(api_key="test_key", search_engine_id="test_id")- assert hasattr(engine, "last_request_time")- assert engine.last_request_time == 0+ assert not hasattr(engine, "last_request_time")class TestGooglePSESearchExecution:diff --git a/tests/security/test_api_key_leakage.py b/tests/security/test_api_key_leakage.pyindex bb8bee0679..fc2c518df0 100644--- a/tests/security/test_api_key_leakage.py+++ b/tests/security/test_api_key_leakage.py@@ -772,6 +772,7 @@ class TestSearchEngineParamsKeyLeakage:engine.region = "us"engine.max_retries = 1 # single attempt — no real sleep, no real retryengine.retry_delay = 0+ engine.min_request_interval = 0engine.engine_type = "test_engine"engine.rate_tracker = _stub_rate_tracker()# _make_request will re-raise as RequestException after max_retriesdiff --git a/tests/web_search_engines/engines/test_search_engine_google_pse_coverage.py b/tests/web_search_engines/engines/test_search_engine_google_pse_coverage.pyindex ee1ccbd153..5c53d6d4b2 100644--- a/tests/web_search_engines/engines/test_search_engine_google_pse_coverage.py+++ b/tests/web_search_engines/engines/test_search_engine_google_pse_coverage.py@@ -11,12 +11,14 @@ Covers:- Init: API key and engine ID resolved from settings_snapshot"""-import timefrom unittest.mock import Mock, call, patchimport pytestfrom requests.exceptions import RequestException+from local_deep_research.web_search_engines.engines import (+ _google_pse_rate_limiter,+)from local_deep_research.web_search_engines.engines.search_engine_google_pse import (GooglePSESearchEngine,)@@ -28,6 +30,13 @@ from local_deep_research.web_search_engines.rate_limiting import RateLimitError# ---------------------------------------------------------------------------[email protected](autouse=True)+def _reset_shared_rate_limiter():+ _google_pse_rate_limiter.reset_for_tests()+ yield+ _google_pse_rate_limiter.reset_for_tests()++def _make_engine(**overrides):"""Build a GooglePSESearchEngine with validation disabled."""defaults = dict(@@ -91,6 +100,29 @@ class TestMakeRequestRetryThenSuccess:assert result == {"searchInformation": {"totalResults": "5"}}+ def test_applies_adaptive_and_shared_rate_limits(self):+ engine = _make_engine()+ engine.rate_tracker = Mock()+ engine.rate_tracker.apply_rate_limit.return_value = 0.1+ response = _ok_response({"items": []})++ with (+ patch.object(+ engine, "_respect_rate_limit", return_value=0.4+ ) as shared_limit,+ patch(+ "local_deep_research.web_search_engines.engines.search_engine_google_pse.safe_get",+ return_value=response,+ ),+ ):+ engine._make_request("query")++ engine.rate_tracker.apply_rate_limit.assert_called_once_with(+ engine.engine_type+ )+ shared_limit.assert_called_once_with()+ assert engine._last_wait_time == pytest.approx(0.5)+# ---------------------------------------------------------------------------# _make_request — all retries fail@@ -234,24 +266,40 @@ class TestMakeRequestGenericExceptionRateLimit:class TestRespectRateLimit:"""Tests for the _respect_rate_limit helper."""- def test_sleeps_when_within_interval(self):- engine = _make_engine()- engine.last_request_time = time.time() # just requested+ def test_shared_across_matching_engine_instances(self):+ first = _make_engine()+ second = _make_engine()with patch("time.sleep") as mock_sleep:- engine._respect_rate_limit()+ assert first._respect_rate_limit() == 0+ waited = second._respect_rate_limit()+mock_sleep.assert_called_once()- slept = mock_sleep.call_args[0][0]- assert 0 < slept <= engine.min_request_interval+ assert waited == mock_sleep.call_args.args[0]+ assert 0 < waited <= first.min_request_interval- def test_no_sleep_when_enough_time_elapsed(self):- engine = _make_engine()- engine.last_request_time = time.time() - 10 # 10 seconds ago+ def test_different_credentials_have_independent_scopes(self):+ first = _make_engine(api_key="first-key")+ second = _make_engine(api_key="second-key")with patch("time.sleep") as mock_sleep:- engine._respect_rate_limit()+ assert first._respect_rate_limit() == 0+ assert second._respect_rate_limit() == 0mock_sleep.assert_not_called()+ def test_process_state_does_not_retain_raw_credentials(self):+ engine = _make_engine(+ api_key="private-api-key",+ search_engine_id="private-cse-id",+ )++ engine._respect_rate_limit()++ stored_scopes = tuple(_google_pse_rate_limiter._scope_locks)+ assert stored_scopes+ assert all("private-api-key" not in scope for scope in stored_scopes)+ assert all("private-cse-id" not in scope for scope in stored_scopes)+# ---------------------------------------------------------------------------# _get_previews — pagination across multiple pages