Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(google-pse): share rate limit across engine instances (#5359)

Ziyang Guo86c8f3b4a857
diff --git a/changelog.d/+google-pse-shared-rate-limit.bugfix.md b/changelog.d/+google-pse-shared-rate-limit.bugfix.md
new file mode 100644
index 0000000000..8221066bee
--- /dev/null
+++ b/changelog.d/+google-pse-shared-rate-limit.bugfix.md
@@ -0,0 +1,3 @@
+Google Programmable Search now shares its minimum request-spacing state across
+engine instances using the same API key and CSE ID, so research-agent tool
+calls cannot bypass throttling by constructing a fresh engine per request.
diff --git a/src/local_deep_research/web_search_engines/engines/_google_pse_rate_limiter.py b/src/local_deep_research/web_search_engines/engines/_google_pse_rate_limiter.py
new file mode 100644
index 0000000000..caebe80ac1
--- /dev/null
+++ b/src/local_deep_research/web_search_engines/engines/_google_pse_rate_limiter.py
@@ -0,0 +1,59 @@
+"""Process-wide minimum request spacing for Google PSE engine instances."""
+
+import secrets
+import threading
+import time
+
+_meta_lock = threading.Lock()
+_scope_locks: dict[str, threading.Lock] = {}
+_scope_last_request: dict[str, float] = {}
+_scope_nonce = secrets.token_urlsafe(32)
+
+
+def _scope_key(api_key: str, search_engine_id: str) -> str:
+ """Return a process-local opaque ID without retaining raw credentials.
+
+ Python string hashing is process-keyed SipHash. Two independently ordered
+ tuple hashes plus a random nonce give the in-memory limiter a compact scope
+ identity without treating API credentials as password-verification data.
+ """
+ mask = (1 << 64) - 1
+ first = hash((_scope_nonce, api_key, search_engine_id)) & mask
+ second = hash((search_engine_id, api_key, _scope_nonce)) & mask
+ return f"{first:016x}{second:016x}"
+
+
+def _get_scope_lock(scope: str) -> threading.Lock:
+ with _meta_lock:
+ lock = _scope_locks.get(scope)
+ if lock is None:
+ lock = threading.Lock()
+ _scope_locks[scope] = lock
+ return lock
+
+
+def respect_rate_limit(
+ api_key: str,
+ search_engine_id: str,
+ interval_seconds: float,
+) -> float:
+ """Sleep as needed to reserve a request slot for one PSE configuration."""
+ if interval_seconds <= 0:
+ return 0.0
+
+ scope = _scope_key(api_key, search_engine_id)
+ lock = _get_scope_lock(scope)
+ with lock:
+ elapsed = time.monotonic() - _scope_last_request.get(scope, 0.0)
+ wait_time = max(0.0, interval_seconds - elapsed)
+ if wait_time > 0:
+ time.sleep(wait_time)
+ _scope_last_request[scope] = time.monotonic()
+ return wait_time
+
+
+def reset_for_tests() -> None:
+ """Clear process-wide state. Intended for unit tests only."""
+ with _meta_lock:
+ _scope_locks.clear()
+ _scope_last_request.clear()
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_google_pse.py b/src/local_deep_research/web_search_engines/engines/search_engine_google_pse.py
index 7ab4cc9d75..2e67e95b1f 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_google_pse.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_google_pse.py
@@ -9,6 +9,7 @@ from requests.exceptions import RequestException
from ...security.safe_requests import safe_get
from ..rate_limiting import RateLimitError
from ..search_engine_base import BaseSearchEngine, Exposure, Sensitivity
+from ._google_pse_rate_limiter import respect_rate_limit
class GooglePSESearchEngine(BaseSearchEngine):
@@ -71,8 +72,6 @@ class GooglePSESearchEngine(BaseSearchEngine):
self.max_retries = max_retries
self.retry_delay = retry_delay
- # Rate limiting - keep track of last request time
- self.last_request_time: float = 0.0
self.min_request_interval = (
0.5 # Minimum time between requests in seconds
)
@@ -171,19 +170,20 @@ class GooglePSESearchEngine(BaseSearchEngine):
)
raise type(e)(safe_msg) from None
- def _respect_rate_limit(self):
- """Ensure we don't exceed rate limits by adding appropriate delay between requests"""
- current_time = time.time()
- elapsed = current_time - self.last_request_time
-
- # If we've made a request recently, wait until the minimum interval has passed
- if elapsed < self.min_request_interval:
- sleep_time = self.min_request_interval - elapsed
- logger.debug("Rate limiting: sleeping for {:.2f} s", sleep_time)
- time.sleep(sleep_time)
-
- # Update the last request time
- self.last_request_time = time.time()
+ def _respect_rate_limit(self) -> float:
+ """Enforce minimum spacing across engines using the same credentials."""
+ api_key = self.api_key
+ search_engine_id = self.search_engine_id
+ if api_key is None or search_engine_id is None:
+ raise ValueError("Google PSE credentials are required")
+ wait_time = respect_rate_limit(
+ api_key,
+ search_engine_id,
+ self.min_request_interval,
+ )
+ if wait_time > 0:
+ logger.debug("Rate limiting: sleeping for {:.2f} s", wait_time)
+ return wait_time
def _make_request(self, query: str, start_index: int = 1) -> Dict:
"""
@@ -243,9 +243,11 @@ class GooglePSESearchEngine(BaseSearchEngine):
start_index,
)
# Apply rate limiting before request
- self._last_wait_time = self.rate_tracker.apply_rate_limit(
+ adaptive_wait = self.rate_tracker.apply_rate_limit(
self.engine_type
)
+ minimum_interval_wait = self._respect_rate_limit()
+ self._last_wait_time = adaptive_wait + minimum_interval_wait
response = safe_get(url, params=params, timeout=10)
diff --git a/tests/search_engines/test_search_engine_google_pse.py b/tests/search_engines/test_search_engine_google_pse.py
index 60e39c14be..7c03d63104 100644
--- a/tests/search_engines/test_search_engine_google_pse.py
+++ b/tests/search_engines/test_search_engine_google_pse.py
@@ -264,8 +264,8 @@ class TestGooglePSERateLimiting:
assert hasattr(engine, "min_request_interval")
assert engine.min_request_interval == 0.5
- def test_last_request_time_initialized(self):
- """Test that last request time is initialized."""
+ def test_last_request_time_not_stored_per_instance(self):
+ """Rate-limit timestamps live in the process-wide tracker."""
from local_deep_research.web_search_engines.engines.search_engine_google_pse import (
GooglePSESearchEngine,
)
@@ -273,8 +273,7 @@ class TestGooglePSERateLimiting:
engine = GooglePSESearchEngine(
api_key="test_key", search_engine_id="test_id"
)
- assert hasattr(engine, "last_request_time")
- assert engine.last_request_time == 0
+ assert not hasattr(engine, "last_request_time")
class TestGooglePSESearchExecution:
diff --git a/tests/security/test_api_key_leakage.py b/tests/security/test_api_key_leakage.py
index bb8bee0679..fc2c518df0 100644
--- a/tests/security/test_api_key_leakage.py
+++ b/tests/security/test_api_key_leakage.py
@@ -772,6 +772,7 @@ class TestSearchEngineParamsKeyLeakage:
engine.region = "us"
engine.max_retries = 1 # single attempt — no real sleep, no real retry
engine.retry_delay = 0
+ engine.min_request_interval = 0
engine.engine_type = "test_engine"
engine.rate_tracker = _stub_rate_tracker()
# _make_request will re-raise as RequestException after max_retries
diff --git a/tests/web_search_engines/engines/test_search_engine_google_pse_coverage.py b/tests/web_search_engines/engines/test_search_engine_google_pse_coverage.py
index ee1ccbd153..5c53d6d4b2 100644
--- a/tests/web_search_engines/engines/test_search_engine_google_pse_coverage.py
+++ b/tests/web_search_engines/engines/test_search_engine_google_pse_coverage.py
@@ -11,12 +11,14 @@ Covers:
- Init: API key and engine ID resolved from settings_snapshot
"""
-import time
from unittest.mock import Mock, call, patch
import pytest
from requests.exceptions import RequestException
+from local_deep_research.web_search_engines.engines import (
+ _google_pse_rate_limiter,
+)
from local_deep_research.web_search_engines.engines.search_engine_google_pse import (
GooglePSESearchEngine,
)
@@ -28,6 +30,13 @@ from local_deep_research.web_search_engines.rate_limiting import RateLimitError
# ---------------------------------------------------------------------------
[email protected](autouse=True)
+def _reset_shared_rate_limiter():
+ _google_pse_rate_limiter.reset_for_tests()
+ yield
+ _google_pse_rate_limiter.reset_for_tests()
+
+
def _make_engine(**overrides):
"""Build a GooglePSESearchEngine with validation disabled."""
defaults = dict(
@@ -91,6 +100,29 @@ class TestMakeRequestRetryThenSuccess:
assert result == {"searchInformation": {"totalResults": "5"}}
+ def test_applies_adaptive_and_shared_rate_limits(self):
+ engine = _make_engine()
+ engine.rate_tracker = Mock()
+ engine.rate_tracker.apply_rate_limit.return_value = 0.1
+ response = _ok_response({"items": []})
+
+ with (
+ patch.object(
+ engine, "_respect_rate_limit", return_value=0.4
+ ) as shared_limit,
+ patch(
+ "local_deep_research.web_search_engines.engines.search_engine_google_pse.safe_get",
+ return_value=response,
+ ),
+ ):
+ engine._make_request("query")
+
+ engine.rate_tracker.apply_rate_limit.assert_called_once_with(
+ engine.engine_type
+ )
+ shared_limit.assert_called_once_with()
+ assert engine._last_wait_time == pytest.approx(0.5)
+
# ---------------------------------------------------------------------------
# _make_request — all retries fail
@@ -234,24 +266,40 @@ class TestMakeRequestGenericExceptionRateLimit:
class TestRespectRateLimit:
"""Tests for the _respect_rate_limit helper."""
- def test_sleeps_when_within_interval(self):
- engine = _make_engine()
- engine.last_request_time = time.time() # just requested
+ def test_shared_across_matching_engine_instances(self):
+ first = _make_engine()
+ second = _make_engine()
with patch("time.sleep") as mock_sleep:
- engine._respect_rate_limit()
+ assert first._respect_rate_limit() == 0
+ waited = second._respect_rate_limit()
+
mock_sleep.assert_called_once()
- slept = mock_sleep.call_args[0][0]
- assert 0 < slept <= engine.min_request_interval
+ assert waited == mock_sleep.call_args.args[0]
+ assert 0 < waited <= first.min_request_interval
- def test_no_sleep_when_enough_time_elapsed(self):
- engine = _make_engine()
- engine.last_request_time = time.time() - 10 # 10 seconds ago
+ def test_different_credentials_have_independent_scopes(self):
+ first = _make_engine(api_key="first-key")
+ second = _make_engine(api_key="second-key")
with patch("time.sleep") as mock_sleep:
- engine._respect_rate_limit()
+ assert first._respect_rate_limit() == 0
+ assert second._respect_rate_limit() == 0
mock_sleep.assert_not_called()
+ def test_process_state_does_not_retain_raw_credentials(self):
+ engine = _make_engine(
+ api_key="private-api-key",
+ search_engine_id="private-cse-id",
+ )
+
+ engine._respect_rate_limit()
+
+ stored_scopes = tuple(_google_pse_rate_limiter._scope_locks)
+ assert stored_scopes
+ assert all("private-api-key" not in scope for scope in stored_scopes)
+ assert all("private-cse-id" not in scope for scope in stored_scopes)
+
# ---------------------------------------------------------------------------
# _get_previews — pagination across multiple pages