Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(rate-limit): eliminate latent multi-connection sites in tracker.py (#6562) (#6566)

Sc3f5b767be9f
diff --git a/changelog.d/6562.bugfix.md b/changelog.d/6562.bugfix.md
new file mode 100644
index 0000000000..1dd3cac3d3
--- /dev/null
+++ b/changelog.d/6562.bugfix.md
@@ -0,0 +1 @@
+Eliminate separate database connection checkouts across `AdaptiveRateLimitTracker` (`_ensure_estimates_loaded`, `get_stats`, `_update_estimate`, `reset_engine`, and `cleanup_old_data`) by reusing in-scope thread-local sessions with savepoint (`begin_nested()`) isolation for writes and `session.no_autoflush` for reads, preventing SQLite `database is locked` deadlocks and preserving outer transaction atomicity under writer contention. Read sites can no longer autoflush the caller's pending ORM rows before the SELECT under a competing SQLite writer; write sites still detach pending state (including `cascade="all, delete-orphan"` children such as `ResearchHistory.resources`) before SAVEPOINT so a savepoint-entry flush cannot invalidate the outer transaction. The detach is exception-safe so a half-failed tracker write cannot leak detached caller objects into the outer transaction.
diff --git a/src/local_deep_research/web_search_engines/rate_limiting/tracker.py b/src/local_deep_research/web_search_engines/rate_limiting/tracker.py
index 067cd38504..711b22057d 100644
--- a/src/local_deep_research/web_search_engines/rate_limiting/tracker.py
+++ b/src/local_deep_research/web_search_engines/rate_limiting/tracker.py
@@ -6,8 +6,17 @@ import random
import threading
import time
from collections import deque
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
-
+from contextlib import AbstractContextManager, contextmanager
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ Dict,
+ Iterator,
+ List,
+ Literal,
+ Optional,
+ Tuple,
+)
from cachetools import LRUCache
if TYPE_CHECKING:
@@ -33,11 +42,13 @@ def _get_db_imports():
try:
from ...database.models import RateLimitAttempt, RateLimitEstimate
from ...database.session_context import get_user_db_session
+ from ...database.thread_local_session import thread_session_manager
_db_imports = {
"RateLimitAttempt": RateLimitAttempt,
"RateLimitEstimate": RateLimitEstimate,
"get_user_db_session": get_user_db_session,
+ "thread_session_manager": thread_session_manager,
}
except (ImportError, RuntimeError):
# Database not available - programmatic mode
@@ -45,6 +56,30 @@ def _get_db_imports():
return _db_imports
+class _InScopeSessionContext:
+ """Context manager wrapping an in-scope session to shield outer transactions.
+
+ Calls the underlying context manager's __exit__ with (None, None, None)
+ so internal exceptions do not trigger safe_rollback on the caller's
+ shared session, while propagating the exception to the caller.
+ """
+
+ def __init__(self, cm: Any) -> None:
+ self._cm = cm
+
+ def __enter__(self) -> Any:
+ return self._cm.__enter__()
+
+ def __exit__(
+ self,
+ exc_type: Optional[type[BaseException]],
+ exc_val: Optional[BaseException],
+ exc_tb: Optional[Any],
+ ) -> Literal[False]:
+ self._cm.__exit__(None, None, None)
+ return False
+
+
class AdaptiveRateLimitTracker:
"""
Tracks and learns optimal retry wait times for each search engine.
@@ -159,6 +194,155 @@ class AdaptiveRateLimitTracker:
"Rate limit estimates will be loaded on-demand when user context is available"
)
+ def _resolve_session(
+ self, username: str, password: str
+ ) -> AbstractContextManager["Session"]:
+ """Unified session resolution helper for database operations.
+
+ Reuses the thread-local session when an enclosing scope is active
+ (avoiding a second QueuePool connection and SQLite busy-timeout deadlock);
+ otherwise falls back to a dedicated session that commits and closes on exit.
+ """
+ db_imports = _get_db_imports() or {}
+ thread_session_manager = db_imports.get("thread_session_manager")
+ if thread_session_manager is None:
+ try:
+ from ...database.thread_local_session import (
+ thread_session_manager,
+ )
+ except (ImportError, RuntimeError):
+ thread_session_manager = None
+
+ get_user_db_session = db_imports.get("get_user_db_session")
+ if get_user_db_session is None:
+ try:
+ from ...database.session_context import get_user_db_session
+ except (ImportError, RuntimeError):
+ get_user_db_session = None
+
+ if (
+ thread_session_manager is not None
+ and thread_session_manager.in_scope()
+ and get_user_db_session is not None
+ ):
+ return _InScopeSessionContext(
+ get_user_db_session(username, password)
+ )
+
+ from ...database.thread_metrics import metrics_writer
+
+ metrics_writer.set_user_password(username, password)
+ return metrics_writer.get_session(username)
+
+ @contextmanager
+ def _isolated_begin_nested(self, session: "Session") -> Iterator[None]:
+ """Open a SAVEPOINT without flushing pre-existing pending ORM state.
+
+ ``Session.begin_nested()`` flushes every pending change before it
+ issues SAVEPOINT. If another connection holds SQLite's writer lock,
+ that flush fails outside the savepoint and SQLAlchemy invalidates
+ the caller's transaction. Detach already-pending instances first so
+ tracker writes cannot flush them until after the isolation boundary
+ exists; restore them after the savepoint settles.
+
+ Both expunge and restore are exception-safe: cascade-expelled
+ children (``cascade="all, delete-orphan"`` such as
+ ``ResearchHistory.resources``) are skipped instead of raising
+ ``InvalidRequestError``, and any failure during expunge or restore
+ is contained so a half-failed tracker write cannot leave caller
+ objects detached from the session.
+ """
+ pending_new = list(getattr(session, "new", ()) or ())
+ pending_deleted = list(getattr(session, "deleted", ()) or ())
+ skip_ids = {id(obj) for obj in pending_new}
+ skip_ids.update(id(obj) for obj in pending_deleted)
+ pending_dirty = [
+ obj
+ for obj in list(getattr(session, "dirty", ()) or ())
+ if id(obj) not in skip_ids
+ ]
+
+ # Dedupe by identity and skip objects already detached (e.g. by an
+ # earlier cascade expunge of a parent). ``expunge`` cascades along
+ # ``cascade="all, delete-orphan"`` relationships, so a parent +
+ # child both pending must not be expunged twice.
+ ordered: list[Any] = []
+ seen: set[int] = set()
+ for obj in (*pending_new, *pending_dirty, *pending_deleted):
+ if id(obj) in seen:
+ continue
+ seen.add(id(obj))
+ ordered.append(obj)
+
+ for obj in ordered:
+ if obj not in session:
+ # Cascade already detached this object when its parent was
+ # expunged; it remains in pending_new/dirty/deleted by
+ # Python identity, but not by session membership.
+ continue
+ try:
+ session.expunge(obj)
+ except Exception:
+ # ``expunge`` can raise for already-detached instances or
+ # for objects the session has lost track of. Swallow and
+ # continue so any later objects can still be detached and
+ # later we still attempt full restoration below.
+ logger.debug(
+ "Tracker isolation: expunge failed during pre-flush detach",
+ exc_info=True,
+ )
+
+ try:
+ with session.begin_nested():
+ yield
+ finally:
+ self._restore_pending_state(
+ session, pending_new, pending_dirty, pending_deleted
+ )
+
+ def _restore_pending_state(
+ self,
+ session: "Session",
+ pending_new: list[Any],
+ pending_dirty: list[Any],
+ pending_deleted: list[Any],
+ ) -> None:
+ """Best-effort restoration of expunged caller objects.
+
+ Adds new/dirty objects back to the session and re-stages deleted
+ objects. ``add(parent)`` cascades to children for relationships
+ declared with ``cascade="all, delete-orphan"``, so we skip any
+ child already re-attached by cascade to avoid double-add errors.
+ All restoration steps are individually exception-safe: a failure
+ here must never propagate out and detach caller state.
+ """
+ for obj in (*pending_new, *pending_dirty):
+ try:
+ if obj not in session:
+ session.add(obj)
+ except Exception:
+ logger.debug(
+ "Tracker isolation: failed to re-attach caller object",
+ exc_info=True,
+ )
+ for obj in pending_deleted:
+ try:
+ if obj not in session:
+ session.add(obj)
+ except Exception:
+ logger.debug(
+ "Tracker isolation: failed to re-add deleted caller object",
+ exc_info=True,
+ )
+ continue
+ try:
+ session.delete(obj)
+ except Exception:
+ logger.debug(
+ "Tracker isolation: failed to re-stage deleted caller object",
+ exc_info=True,
+ )
+
def _ensure_estimates_loaded(self) -> None:
"""Load estimates from user's encrypted database if not already loaded."""
# Early return if already loaded or should skip
@@ -191,34 +375,38 @@ class AdaptiveRateLimitTracker:
if username and password:
try:
- # Use thread-safe metrics writer to read from user's encrypted database
- from ...database.thread_metrics import metrics_writer
-
- # Set password for this thread
- metrics_writer.set_user_password(username, password)
-
session: "Session"
- with metrics_writer.get_session(username) as session:
- estimates: list[Any] = session.query(
- RateLimitEstimate
- ).all()
-
- for estimate in estimates:
- # Apply decay for old estimates
- age_hours = (time.time() - estimate.last_updated) / 3600
- decay = self.decay_per_day ** (age_hours / 24)
-
- with self._cache_lock:
- self.current_estimates[estimate.engine_type] = {
- "base": estimate.base_wait_seconds,
- "min": estimate.min_wait_seconds,
- "max": estimate.max_wait_seconds,
- "confidence": decay,
- }
-
- logger.debug(
- f"Loaded estimate for {estimate.engine_type}: base={estimate.base_wait_seconds:.2f}s, confidence={decay:.2f}"
- )
+ with self._resolve_session(username, password) as session:
+ # The production sessionmaker defaults to
+ # ``autoflush=True``, so a bare ``query()`` could flush
+ # the caller's pending ORM writes before the SELECT
+ # runs. Under a competing SQLite writer that flush
+ # fails and SQLAlchemy invalidates the outer
+ # transaction; ``no_autoflush`` keeps the read from
+ # touching unrelated caller state.
+ with session.no_autoflush:
+ estimates: list[Any] = session.query(
+ RateLimitEstimate
+ ).all()
+
+ for estimate in estimates:
+ # Apply decay for old estimates
+ age_hours = (
+ time.time() - estimate.last_updated
+ ) / 3600
+ decay = self.decay_per_day ** (age_hours / 24)
+
+ with self._cache_lock:
+ self.current_estimates[estimate.engine_type] = {
+ "base": estimate.base_wait_seconds,
+ "min": estimate.min_wait_seconds,
+ "max": estimate.max_wait_seconds,
+ "confidence": decay,
+ }
+
+ logger.debug(
+ f"Loaded estimate for {estimate.engine_type}: base={estimate.base_wait_seconds:.2f}s, confidence={decay:.2f}"
+ )
self._estimates_loaded = True
logger.info(
@@ -480,44 +668,47 @@ class AdaptiveRateLimitTracker:
if username and password:
try:
- # Use thread-safe metrics writer to save to user's encrypted database
- from ...database.thread_metrics import metrics_writer
-
- # Set password for this thread if not already set
- metrics_writer.set_user_password(username, password)
-
db_imports = _get_db_imports()
- RateLimitEstimate = db_imports.get("RateLimitEstimate")
+ RateLimitEstimate = (
+ db_imports.get("RateLimitEstimate")
+ if db_imports
+ else None
+ )
+ if not RateLimitEstimate:
+ return
session_inner: "Session"
- with metrics_writer.get_session(username) as session_inner:
- # Check if estimate exists
- estimate: Any = (
- session_inner.query(RateLimitEstimate)
- .filter_by(engine_type=engine_type)
- .first()
- )
-
- if estimate:
- # Update existing estimate
- estimate.base_wait_seconds = new_base
- estimate.min_wait_seconds = min_wait
- estimate.max_wait_seconds = max_wait
- estimate.last_updated = time.time()
- estimate.total_attempts = len(attempts)
- estimate.success_rate = success_rate
- else:
- # Create new estimate
- estimate = RateLimitEstimate(
- engine_type=engine_type,
- base_wait_seconds=new_base,
- min_wait_seconds=min_wait,
- max_wait_seconds=max_wait,
- last_updated=time.time(),
- total_attempts=len(attempts),
- success_rate=success_rate,
+ with self._resolve_session(
+ username, password
+ ) as session_inner:
+ with self._isolated_begin_nested(session_inner):
+ # Check if estimate exists
+ estimate: Any = (
+ session_inner.query(RateLimitEstimate)
+ .filter_by(engine_type=engine_type)
+ .first()
)
- session_inner.add(estimate)
+
+ if estimate:
+ # Update existing estimate
+ estimate.base_wait_seconds = new_base
+ estimate.min_wait_seconds = min_wait
+ estimate.max_wait_seconds = max_wait
+ estimate.last_updated = time.time()
+ estimate.total_attempts = len(attempts)
+ estimate.success_rate = success_rate
+ else:
+ # Create new estimate
+ estimate = RateLimitEstimate(
+ engine_type=engine_type,
+ base_wait_seconds=new_base,
+ min_wait_seconds=min_wait,
+ max_wait_seconds=max_wait,
+ last_updated=time.time(),
+ total_attempts=len(attempts),
+ success_rate=success_rate,
+ )
+ session_inner.add(estimate)
except Exception:
logger.warning("Failed to persist rate limit estimate")
@@ -594,24 +785,27 @@ class AdaptiveRateLimitTracker:
db_imports = _get_db_imports()
RateLimitEstimate = db_imports.get("RateLimitEstimate")
- from ...database.thread_metrics import metrics_writer
-
- metrics_writer.set_user_password(username, password)
-
session_est: "Session"
- with metrics_writer.get_session(username) as session_est:
- if engine_type:
- estimates: list[Any] = (
- session_est.query(RateLimitEstimate)
- .filter_by(engine_type=engine_type)
- .all()
- )
- else:
- estimates = (
- session_est.query(RateLimitEstimate)
- .order_by(RateLimitEstimate.engine_type)
- .all()
- )
+ with self._resolve_session(username, password) as session_est:
+ # See ``_ensure_estimates_loaded``: the production
+ # sessionmaker defaults to ``autoflush=True`` and a cold
+ # ``get_stats`` would otherwise flush caller's pending
+ # writes before the SELECT, which under a competing
+ # SQLite writer fails and invalidates the outer
+ # transaction.
+ with session_est.no_autoflush:
+ if engine_type:
+ estimates: list[Any] = (
+ session_est.query(RateLimitEstimate)
+ .filter_by(engine_type=engine_type)
+ .all()
+ )
+ else:
+ estimates = (
+ session_est.query(RateLimitEstimate)
+ .order_by(RateLimitEstimate.engine_type)
+ .all()
+ )
return [
(
@@ -675,26 +869,27 @@ class AdaptiveRateLimitTracker:
return
db_imports = _get_db_imports()
- RateLimitAttempt = db_imports.get("RateLimitAttempt")
- RateLimitEstimate = db_imports.get("RateLimitEstimate")
-
- from ...database.thread_metrics import metrics_writer
-
- metrics_writer.set_user_password(username, password)
+ RateLimitAttempt = (
+ db_imports.get("RateLimitAttempt") if db_imports else None
+ )
+ RateLimitEstimate = (
+ db_imports.get("RateLimitEstimate") if db_imports else None
+ )
+ if not RateLimitAttempt or not RateLimitEstimate:
+ return
session_reset: "Session"
- with metrics_writer.get_session(username) as session_reset:
- # Delete historical attempts
- session_reset.query(RateLimitAttempt).filter_by(
- engine_type=engine_type
- ).delete()
-
- # Delete estimates
- session_reset.query(RateLimitEstimate).filter_by(
- engine_type=engine_type
- ).delete()
-
- session_reset.commit()
+ with self._resolve_session(username, password) as session_reset:
+ with self._isolated_begin_nested(session_reset):
+ # Delete historical attempts
+ session_reset.query(RateLimitAttempt).filter_by(
+ engine_type=engine_type
+ ).delete()
+
+ # Delete estimates
+ session_reset.query(RateLimitEstimate).filter_by(
+ engine_type=engine_type
+ ).delete()
logger.info(f"Reset rate limit data for {engine_type}")
@@ -809,22 +1004,22 @@ class AdaptiveRateLimitTracker:
return
db_imports = _get_db_imports()
- RateLimitAttempt = db_imports.get("RateLimitAttempt")
-
- from ...database.thread_metrics import metrics_writer
-
- metrics_writer.set_user_password(username, password)
+ RateLimitAttempt = (
+ db_imports.get("RateLimitAttempt") if db_imports else None
+ )
+ if not RateLimitAttempt:
+ return
session_clean: "Session"
- with metrics_writer.get_session(username) as session_clean:
- # Count and delete old attempts
- old_attempts: Any = session_clean.query(
- RateLimitAttempt
- ).filter(RateLimitAttempt.timestamp < cutoff_time)
- deleted_count = old_attempts.count()
- old_attempts.delete()
-
- session_clean.commit()
+ deleted_count = 0
+ with self._resolve_session(username, password) as session_clean:
+ with self._isolated_begin_nested(session_clean):
+ # Count and delete old attempts
+ old_attempts: Any = session_clean.query(
+ RateLimitAttempt
+ ).filter(RateLimitAttempt.timestamp < cutoff_time)
+ deleted_count = old_attempts.count()
+ old_attempts.delete()
if deleted_count > 0:
logger.info(f"Cleaned up {deleted_count} old retry attempts")
diff --git a/tests/rate_limiting/test_rate_limiting.py b/tests/rate_limiting/test_rate_limiting.py
index bf76ef746d..48635382c9 100644
--- a/tests/rate_limiting/test_rate_limiting.py
+++ b/tests/rate_limiting/test_rate_limiting.py
@@ -2,9 +2,11 @@
Tests for the adaptive rate limiting system.
"""
+from collections import deque
import os
+import time
import unittest
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
import pytest
@@ -302,8 +304,6 @@ class TestAdaptiveRateLimitTracker(unittest.TestCase):
"max": 10.0,
"confidence": 0.8,
}
- from collections import deque
-
self.tracker.recent_attempts[engine_type] = deque(maxlen=100)
# Verify data exists
@@ -696,5 +696,1693 @@ class TestProgrammaticMode(unittest.TestCase):
self.assertFalse(tracker.enabled)
+class TestLatentSessionResolution(unittest.TestCase):
+ """Regression coverage for unified session resolution and savepoint isolation across DB operations.
+
+ Issue #6562: When the caller already holds a thread-local session (in_scope()),
+ _ensure_estimates_loaded, get_stats, _update_estimate, reset_engine, and cleanup_old_data
+ must reuse the thread-local session instead of checking out a separate connection via
+ metrics_writer.get_session(), which commits on exit and self-deadlocks on SQLite locks.
+ Furthermore, write operations must use savepoint discipline (begin_nested()) and never commit
+ the caller's shared session, preserving outer transaction rollback atomicity.
+ """
+
+ TRACKER_MOD = "local_deep_research.web_search_engines.rate_limiting.tracker"
+
+ def _build_tracker(self):
+ settings_snapshot = {
+ "rate_limiting.enabled": {"value": True, "ui_element": "checkbox"}
+ }
+ return AdaptiveRateLimitTracker(
+ settings_snapshot=settings_snapshot, programmatic_mode=False
+ )
+
+ # --- 1. Direct _resolve_session helper tests ---
+
+ def test_resolve_session_in_scope_uses_get_user_db_session(self):
+ """When in_scope() is True, _resolve_session yields from get_user_db_session."""
+ tracker = self._build_tracker()
+ mock_session = MagicMock()
+ mock_get_user_db = MagicMock()
+ mock_get_user_db.return_value.__enter__.return_value = mock_session
+ mock_get_user_db.return_value.__exit__.return_value = False
+ mock_writer = MagicMock()
+ mock_tsm = MagicMock()
+ mock_tsm.in_scope.return_value = True
+
+ with (
+ patch(
+ f"{self.TRACKER_MOD}._get_db_imports",
+ return_value={
+ "thread_session_manager": mock_tsm,
+ "get_user_db_session": mock_get_user_db,
+ },
+ ),
+ patch(
+ "local_deep_research.database.thread_metrics.metrics_writer",
+ mock_writer,
+ ),
+ ):
+ with tracker._resolve_session("alice", "secret") as sess:
+ self.assertIs(sess, mock_session)
+
+ mock_get_user_db.assert_called_once_with("alice", "secret")
+ self.assertFalse(mock_writer.get_session.called)
+ self.assertFalse(mock_writer.set_user_password.called)
+
+ def test_resolve_session_not_in_scope_uses_metrics_writer(self):
+ """When in_scope() is False, _resolve_session falls back to metrics_writer."""
+ tracker = self._build_tracker()
+ mock_session = MagicMock()
+ mock_get_user_db = MagicMock()
+ mock_writer = MagicMock()
+ mock_writer.get_session.return_value.__enter__.return_value = (
+ mock_session
+ )
+ mock_writer.get_session.return_value.__exit__.return_value = False
+ mock_tsm = MagicMock()
+ mock_tsm.in_scope.return_value = False
+
+ with (
+ patch(
+ f"{self.TRACKER_MOD}._get_db_imports",
+ return_value={
+ "thread_session_manager": mock_tsm,
+ "get_user_db_session": mock_get_user_db,
+ },
+ ),
+ patch(
+ "local_deep_research.database.thread_metrics.metrics_writer",
+ mock_writer,
+ ),
+ ):
+ with tracker._resolve_session("alice", "secret") as sess:
+ self.assertIs(sess, mock_session)
+
+ mock_writer.set_user_password.assert_called_once_with("alice", "secret")
+ mock_writer.get_session.assert_called_once_with("alice")
+ self.assertFalse(mock_get_user_db.called)
+
+ def test_resolve_session_fallback_when_db_imports_missing_helpers(self):
+ """When thread_session_manager or get_user_db_session is None, fallback works."""
+ tracker = self._build_tracker()
+ mock_session = MagicMock()
+ mock_writer = MagicMock()
+ mock_writer.get_session.return_value.__enter__.return_value = (
+ mock_session
+ )
+ mock_writer.get_session.return_value.__exit__.return_value = False
+
+ with (
+ patch(f"{self.TRACKER_MOD}._get_db_imports", return_value={}),
+ patch(
+ "local_deep_research.database.thread_local_session.thread_session_manager.in_scope",
+ return_value=False,
+ ),
+ patch(
+ "local_deep_research.database.thread_metrics.metrics_writer",
+ mock_writer,
+ ),
+ ):
+ with tracker._resolve_session("alice", "secret") as sess:
+ self.assertIs(sess, mock_session)
+
+ mock_writer.set_user_password.assert_called_once_with("alice", "secret")
+ mock_writer.get_session.assert_called_once_with("alice")
+
+ def test_resolve_session_exception_contained_from_get_user_db_session(self):
+ """Exceptions inside in-scope session must be re-raised after get_user_db_session exits cleanly."""
+ tracker = self._build_tracker()
+ mock_session = MagicMock()
+ mock_cm = MagicMock()
+ mock_cm.__enter__.return_value = mock_session
+ mock_cm.__exit__.return_value = False
+ mock_get_user_db = MagicMock(return_value=mock_cm)
+ mock_tsm = MagicMock()
+ mock_tsm.in_scope.return_value = True
+
+ with (
+ patch(
+ f"{self.TRACKER_MOD}._get_db_imports",
+ return_value={
+ "thread_session_manager": mock_tsm,
+ "get_user_db_session": mock_get_user_db,
+ },
+ ),
+ ):
+ with self.assertRaises(RuntimeError):
+ with tracker._resolve_session("alice", "secret"):
+ raise RuntimeError("inner failure")
+
+ # __exit__ must have received (None, None, None) so get_user_db_session
+ # did not trigger safe_rollback on the shared session
+ mock_cm.__exit__.assert_called_once_with(None, None, None)
+
+ # --- 2. _ensure_estimates_loaded tests ---
+
+ def test_ensure_estimates_loaded_in_scope_reuses_session_without_commit(
+ self,
+ ):
+ """Cold cache miss during active write session must not checkout separate connection or commit."""
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ tracker = self._build_tracker()
+ tracker._estimates_loaded = False
+
+ mock_session = MagicMock()
+ mock_estimate = MagicMock()
+ mock_estimate.engine_type = "bing"
+ mock_estimate.base_wait_seconds = 2.5
+ mock_estimate.min_wait_seconds = 1.0
+ mock_estimate.max_wait_seconds = 5.0
+ mock_estimate.last_updated = 100000.0
+ mock_session.query.return_value.all.return_value = [mock_estimate]
+
+ mock_get_user_db = MagicMock()
+ mock_get_user_db.return_value.__enter__.return_value = mock_session
+ mock_get_user_db.return_value.__exit__.return_value = False
+ mock_writer = MagicMock()
+ mock_tsm = MagicMock()
+ mock_tsm.in_scope.return_value = True
+
+ set_search_context({"username": "testuser", "user_password": "secret"})
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}._get_db_imports",
+ return_value={
+ "thread_session_manager": mock_tsm,
+ "get_user_db_session": mock_get_user_db,
+ "RateLimitEstimate": MagicMock(),
+ },
+ ),
+ patch(
+ "local_deep_research.database.thread_metrics.metrics_writer",
+ mock_writer,
+ ),
+ ):
+ tracker._ensure_estimates_loaded()
+
+ self.assertTrue(tracker._estimates_loaded)
+ self.assertIn("bing", tracker.current_estimates)
+ self.assertEqual(tracker.current_estimates["bing"]["base"], 2.5)
+ mock_get_user_db.assert_called_once_with("testuser", "secret")
+ self.assertFalse(mock_writer.get_session.called)
+ # Must NOT call commit() — reader cannot commit enclosing caller's transaction
+ self.assertFalse(mock_session.commit.called)
+ finally:
+ clear_search_context()
+
+ def test_ensure_estimates_loaded_not_in_scope_uses_metrics_writer(self):
+ """Cold cache miss outside active session scope uses metrics_writer."""
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ tracker = self._build_tracker()
+ tracker._estimates_loaded = False
+
+ mock_session = MagicMock()
+ mock_session.query.return_value.all.return_value = []
+ mock_get_user_db = MagicMock()
+ mock_writer = MagicMock()
+ mock_writer.get_session.return_value.__enter__.return_value = (
+ mock_session
+ )
+ mock_writer.get_session.return_value.__exit__.return_value = False
+ mock_tsm = MagicMock()
+ mock_tsm.in_scope.return_value = False
+
+ set_search_context({"username": "testuser", "user_password": "secret"})
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}._get_db_imports",
+ return_value={
+ "thread_session_manager": mock_tsm,
+ "get_user_db_session": mock_get_user_db,
+ "RateLimitEstimate": MagicMock(),
+ },
+ ),
+ patch(
+ "local_deep_research.database.thread_metrics.metrics_writer",
+ mock_writer,
+ ),
+ ):
+ tracker._ensure_estimates_loaded()
+
+ self.assertTrue(tracker._estimates_loaded)
+ mock_writer.set_user_password.assert_called_once_with(
+ "testuser", "secret"
+ )
+ mock_writer.get_session.assert_called_once_with("testuser")
+ self.assertFalse(mock_get_user_db.called)
+ finally:
+ clear_search_context()
+
+ def test_ensure_estimates_loaded_in_scope_disables_autoflush(self):
+ """Cold-cache read in scope must run inside session.no_autoflush.
+
+ The production sessionmaker uses ``autoflush=True``. A bare
+ ``query()`` would otherwise flush any pending caller writes before
+ the SELECT runs; under a competing SQLite writer that flush fails
+ and SQLAlchemy invalidates the caller's transaction. Pin the
+ no-autoflush contract here.
+ """
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ tracker = self._build_tracker()
+ tracker._estimates_loaded = False
+ mock_session = MagicMock()
+ mock_session.query.return_value.all.return_value = []
+ mock_get_user_db = MagicMock()
+ mock_get_user_db.return_value.__enter__.return_value = mock_session
+ mock_get_user_db.return_value.__exit__.return_value = False
+ mock_writer = MagicMock()
+ mock_tsm = MagicMock()
+ mock_tsm.in_scope.return_value = True
+
+ set_search_context({"username": "testuser", "user_password": "secret"})
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}._get_db_imports",
+ return_value={
+ "thread_session_manager": mock_tsm,
+ "get_user_db_session": mock_get_user_db,
+ "RateLimitEstimate": MagicMock(),
+ },
+ ),
+ patch(
+ "local_deep_research.database.thread_metrics.metrics_writer",
+ mock_writer,
+ ),
+ ):
+ tracker._ensure_estimates_loaded()
+
+ # no_autoflush was entered and exited; query ran inside it
+ self.assertTrue(mock_session.no_autoflush.__enter__.called)
+ self.assertTrue(mock_session.no_autoflush.__exit__.called)
+ self.assertTrue(mock_session.query.called)
+ finally:
+ clear_search_context()
+
+ # --- 3. _update_estimate tests ---
+
+ def test_update_estimate_in_scope_reuses_session_with_savepoint(self):
+ """_update_estimate inside active session scope reuses thread-local session with savepoint."""
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ tracker = self._build_tracker()
+ tracker.recent_attempts["searxng"] = deque(
+ [
+ {
+ "wait_time": 1.0,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ {
+ "wait_time": 1.5,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ {
+ "wait_time": 2.0,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ ]
+ )
+ mock_session = MagicMock()
+ mock_session.query.return_value.filter_by.return_value.first.return_value = None
+ mock_get_user_db = MagicMock()
+ mock_get_user_db.return_value.__enter__.return_value = mock_session
+ mock_get_user_db.return_value.__exit__.return_value = False
+ mock_writer = MagicMock()
+ mock_tsm = MagicMock()
+ mock_tsm.in_scope.return_value = True
+
+ set_search_context({"username": "testuser", "user_password": "secret"})
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}.is_ci_environment",
+ return_value=False,
+ ),
+ patch(
+ f"{self.TRACKER_MOD}._get_db_imports",
+ return_value={
+ "thread_session_manager": mock_tsm,
+ "get_user_db_session": mock_get_user_db,
+ "RateLimitEstimate": MagicMock(),
+ },
+ ),
+ patch(
+ "local_deep_research.database.thread_metrics.metrics_writer",
+ mock_writer,
+ ),
+ ):
+ tracker._update_estimate("searxng")
+ mock_get_user_db.assert_called_once_with("testuser", "secret")
+ self.assertFalse(mock_writer.get_session.called)
+ mock_session.begin_nested.assert_called_once()
+ mock_session.add.assert_called_once()
+ mock_session.commit.assert_not_called()
+ finally:
+ clear_search_context()
+
+ def test_update_estimate_not_in_scope_uses_metrics_writer(self):
+ """_update_estimate outside active session scope uses metrics_writer."""
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ tracker = self._build_tracker()
+ tracker.recent_attempts["searxng"] = deque(
+ [
+ {
+ "wait_time": 1.0,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ {
+ "wait_time": 1.5,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ {
+ "wait_time": 2.0,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ ]
+ )
+ mock_session = MagicMock()
+ mock_session.query.return_value.filter_by.return_value.first.return_value = None
+ mock_get_user_db = MagicMock()
+ mock_writer = MagicMock()
+ mock_writer.get_session.return_value.__enter__.return_value = (
+ mock_session
+ )
+ mock_writer.get_session.return_value.__exit__.return_value = False
+ mock_tsm = MagicMock()
+ mock_tsm.in_scope.return_value = False
+
+ set_search_context({"username": "testuser", "user_password": "secret"})
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}.is_ci_environment",
+ return_value=False,
+ ),
+ patch(
+ f"{self.TRACKER_MOD}._get_db_imports",
+ return_value={
+ "thread_session_manager": mock_tsm,
+ "get_user_db_session": mock_get_user_db,
+ "RateLimitEstimate": MagicMock(),
+ },
+ ),
+ patch(
+ "local_deep_research.database.thread_metrics.metrics_writer",
+ mock_writer,
+ ),
+ ):
+ tracker._update_estimate("searxng")
+ mock_writer.get_session.assert_called_once_with("testuser")
+ self.assertFalse(mock_get_user_db.called)
+ mock_session.begin_nested.assert_called_once()
+ mock_session.add.assert_called_once()
+ mock_session.commit.assert_not_called()
+ finally:
+ clear_search_context()
+
+ # --- 4. get_stats tests ---
+
+ def test_get_stats_in_scope_reuses_session(self):
+ """get_stats inside active session scope reuses thread-local session."""
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ tracker = self._build_tracker()
+ mock_session = MagicMock()
+ mock_est = MagicMock()
+ mock_est.engine_type = "google"
+ mock_est.base_wait_seconds = 1.2
+ mock_est.min_wait_seconds = 0.5
+ mock_est.max_wait_seconds = 3.0
+ mock_est.last_updated = 12345.0
+ mock_est.total_attempts = 10
+ mock_est.success_rate = 0.9
+ mock_session.query.return_value.filter_by.return_value.all.return_value = [
+ mock_est
+ ]
+
+ mock_get_user_db = MagicMock()
+ mock_get_user_db.return_value.__enter__.return_value = mock_session
+ mock_get_user_db.return_value.__exit__.return_value = False
+ mock_writer = MagicMock()
+ mock_tsm = MagicMock()
+ mock_tsm.in_scope.return_value = True
+
+ set_search_context({"username": "testuser", "user_password": "secret"})
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}.is_ci_environment",
+ return_value=False,
+ ),
+ patch(
+ f"{self.TRACKER_MOD}._get_db_imports",
+ return_value={
+ "thread_session_manager": mock_tsm,
+ "get_user_db_session": mock_get_user_db,
+ "RateLimitEstimate": MagicMock(),
+ },
+ ),
+ patch(
+ "local_deep_research.database.thread_metrics.metrics_writer",
+ mock_writer,
+ ),
+ ):
+ stats = tracker.get_stats("google")
+ self.assertEqual(len(stats), 1)
+ self.assertEqual(stats[0][0], "google")
+ self.assertEqual(stats[0][1], 1.2)
+ mock_get_user_db.assert_called_once_with("testuser", "secret")
+ self.assertFalse(mock_writer.get_session.called)
+ finally:
+ clear_search_context()
+
+ def test_get_stats_in_scope_disables_autoflush(self):
+ """get_stats in scope must run query inside session.no_autoflush.
+
+ Cold ``get_stats`` callers share the production sessionmaker
+ (``autoflush=True``). A bare ``query()`` could flush pending caller
+ writes before the SELECT; under a competing SQLite writer that
+ flush fails and rolls the outer transaction back. Pin the
+ no-autoflush contract here.
+ """
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ tracker = self._build_tracker()
+ mock_session = MagicMock()
+ mock_session.query.return_value.filter_by.return_value.all.return_value = []
+
+ mock_get_user_db = MagicMock()
+ mock_get_user_db.return_value.__enter__.return_value = mock_session
+ mock_get_user_db.return_value.__exit__.return_value = False
+ mock_writer = MagicMock()
+ mock_tsm = MagicMock()
+ mock_tsm.in_scope.return_value = True
+
+ set_search_context({"username": "testuser", "user_password": "secret"})
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}.is_ci_environment",
+ return_value=False,
+ ),
+ patch(
+ f"{self.TRACKER_MOD}._get_db_imports",
+ return_value={
+ "thread_session_manager": mock_tsm,
+ "get_user_db_session": mock_get_user_db,
+ "RateLimitEstimate": MagicMock(),
+ },
+ ),
+ patch(
+ "local_deep_research.database.thread_metrics.metrics_writer",
+ mock_writer,
+ ),
+ ):
+ tracker.get_stats("google")
+
+ self.assertTrue(mock_session.no_autoflush.__enter__.called)
+ self.assertTrue(mock_session.no_autoflush.__exit__.called)
+ self.assertTrue(mock_session.query.called)
+ finally:
+ clear_search_context()
+
+ def test_get_stats_not_in_scope_uses_metrics_writer(self):
+ """get_stats outside active session scope uses metrics_writer."""
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ tracker = self._build_tracker()
+ mock_session = MagicMock()
+ mock_session.query.return_value.order_by.return_value.all.return_value = []
+ mock_get_user_db = MagicMock()
+ mock_writer = MagicMock()
+ mock_writer.get_session.return_value.__enter__.return_value = (
+ mock_session
+ )
+ mock_writer.get_session.return_value.__exit__.return_value = False
+ mock_tsm = MagicMock()
+ mock_tsm.in_scope.return_value = False
+
+ set_search_context({"username": "testuser", "user_password": "secret"})
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}.is_ci_environment",
+ return_value=False,
+ ),
+ patch(
+ f"{self.TRACKER_MOD}._get_db_imports",
+ return_value={
+ "thread_session_manager": mock_tsm,
+ "get_user_db_session": mock_get_user_db,
+ "RateLimitEstimate": MagicMock(),
+ },
+ ),
+ patch(
+ "local_deep_research.database.thread_metrics.metrics_writer",
+ mock_writer,
+ ),
+ ):
+ stats = tracker.get_stats()
+
+ self.assertEqual(stats, [])
+ mock_writer.get_session.assert_called_once_with("testuser")
+ self.assertFalse(mock_get_user_db.called)
+ finally:
+ clear_search_context()
+
+ # --- 5. reset_engine tests ---
+
+ def test_reset_engine_in_scope_reuses_session_with_savepoint(self):
+ """reset_engine inside active session scope reuses session with savepoint and does not commit."""
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ tracker = self._build_tracker()
+ mock_session = MagicMock()
+ mock_get_user_db = MagicMock()
+ mock_get_user_db.return_value.__enter__.return_value = mock_session
+ mock_get_user_db.return_value.__exit__.return_value = False
+ mock_writer = MagicMock()
+ mock_tsm = MagicMock()
+ mock_tsm.in_scope.return_value = True
+
+ set_search_context({"username": "testuser", "user_password": "secret"})
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}.is_ci_environment",
+ return_value=False,
+ ),
+ patch(
+ f"{self.TRACKER_MOD}._get_db_imports",
+ return_value={
+ "thread_session_manager": mock_tsm,
+ "get_user_db_session": mock_get_user_db,
+ "RateLimitAttempt": MagicMock(),
+ "RateLimitEstimate": MagicMock(),
+ },
+ ),
+ patch(
+ "local_deep_research.database.thread_metrics.metrics_writer",
+ mock_writer,
+ ),
+ ):
+ tracker.reset_engine("duckduckgo")
+ mock_get_user_db.assert_called_once_with("testuser", "secret")
+ self.assertFalse(mock_writer.get_session.called)
+ mock_session.begin_nested.assert_called_once()
+ # Explicit commit directly on shared session must NOT be called
+ mock_session.commit.assert_not_called()
+ finally:
+ clear_search_context()
+
+ def test_reset_engine_not_in_scope_uses_metrics_writer(self):
+ """reset_engine outside active session scope uses metrics_writer."""
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ tracker = self._build_tracker()
+ mock_session = MagicMock()
+ mock_get_user_db = MagicMock()
+ mock_writer = MagicMock()
+ mock_writer.get_session.return_value.__enter__.return_value = (
+ mock_session
+ )
+ mock_writer.get_session.return_value.__exit__.return_value = False
+ mock_tsm = MagicMock()
+ mock_tsm.in_scope.return_value = False
+
+ set_search_context({"username": "testuser", "user_password": "secret"})
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}.is_ci_environment",
+ return_value=False,
+ ),
+ patch(
+ f"{self.TRACKER_MOD}._get_db_imports",
+ return_value={
+ "thread_session_manager": mock_tsm,
+ "get_user_db_session": mock_get_user_db,
+ "RateLimitAttempt": MagicMock(),
+ "RateLimitEstimate": MagicMock(),
+ },
+ ),
+ patch(
+ "local_deep_research.database.thread_metrics.metrics_writer",
+ mock_writer,
+ ),
+ ):
+ tracker.reset_engine("duckduckgo")
+ mock_writer.get_session.assert_called_once_with("testuser")
+ self.assertFalse(mock_get_user_db.called)
+ mock_session.begin_nested.assert_called_once()
+ mock_session.commit.assert_not_called()
+ finally:
+ clear_search_context()
+
+ # --- 6. cleanup_old_data tests ---
+
+ def test_cleanup_old_data_in_scope_reuses_session_with_savepoint(self):
+ """cleanup_old_data inside active session scope reuses session with savepoint and does not commit."""
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ tracker = self._build_tracker()
+ mock_session = MagicMock()
+ mock_attempt = MagicMock()
+ mock_attempt.timestamp.__lt__.return_value = MagicMock()
+ mock_query = mock_session.query.return_value.filter.return_value
+ mock_query.count.return_value = 5
+ mock_get_user_db = MagicMock()
+ mock_get_user_db.return_value.__enter__.return_value = mock_session
+ mock_get_user_db.return_value.__exit__.return_value = False
+ mock_writer = MagicMock()
+ mock_tsm = MagicMock()
+ mock_tsm.in_scope.return_value = True
+ set_search_context({"username": "testuser", "user_password": "secret"})
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}.is_ci_environment",
+ return_value=False,
+ ),
+ patch(
+ f"{self.TRACKER_MOD}._get_db_imports",
+ return_value={
+ "thread_session_manager": mock_tsm,
+ "get_user_db_session": mock_get_user_db,
+ "RateLimitAttempt": mock_attempt,
+ },
+ ),
+ patch(
+ "local_deep_research.database.thread_metrics.metrics_writer",
+ mock_writer,
+ ),
+ ):
+ tracker.cleanup_old_data(days=14)
+ mock_get_user_db.assert_called_once_with("testuser", "secret")
+ self.assertFalse(mock_writer.get_session.called)
+ mock_session.begin_nested.assert_called_once()
+ mock_query.delete.assert_called_once()
+ mock_session.commit.assert_not_called()
+ finally:
+ clear_search_context()
+
+ def test_cleanup_old_data_not_in_scope_uses_metrics_writer(self):
+ """cleanup_old_data outside active session scope uses metrics_writer."""
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ tracker = self._build_tracker()
+ mock_session = MagicMock()
+ mock_attempt = MagicMock()
+ mock_attempt.timestamp.__lt__.return_value = MagicMock()
+ mock_query = mock_session.query.return_value.filter.return_value
+ mock_query.count.return_value = 0
+ mock_get_user_db = MagicMock()
+ mock_writer = MagicMock()
+ mock_writer.get_session.return_value.__enter__.return_value = (
+ mock_session
+ )
+ mock_writer.get_session.return_value.__exit__.return_value = False
+ mock_tsm = MagicMock()
+ mock_tsm.in_scope.return_value = False
+ set_search_context({"username": "testuser", "user_password": "secret"})
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}.is_ci_environment",
+ return_value=False,
+ ),
+ patch(
+ f"{self.TRACKER_MOD}._get_db_imports",
+ return_value={
+ "thread_session_manager": mock_tsm,
+ "get_user_db_session": mock_get_user_db,
+ "RateLimitAttempt": mock_attempt,
+ },
+ ),
+ patch(
+ "local_deep_research.database.thread_metrics.metrics_writer",
+ mock_writer,
+ ),
+ ):
+ tracker.cleanup_old_data(days=30)
+ mock_writer.get_session.assert_called_once_with("testuser")
+ self.assertFalse(mock_get_user_db.called)
+ mock_session.begin_nested.assert_called_once()
+ mock_session.commit.assert_not_called()
+ finally:
+ clear_search_context()
+
+ # --- 7. Robustness against errors across all 5 sites ---
+
+ def test_resolve_session_error_handled_in_all_sites(self):
+ """Errors in _resolve_session are caught and handled gracefully across all callsites."""
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ tracker = self._build_tracker()
+ tracker._estimates_loaded = False
+
+ set_search_context({"username": "testuser", "user_password": "secret"})
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}.is_ci_environment",
+ return_value=False,
+ ),
+ patch.object(
+ tracker,
+ "_resolve_session",
+ side_effect=RuntimeError("DB exploded"),
+ ),
+ ):
+ # 1. _ensure_estimates_loaded: sets _estimates_loaded=True, doesn't raise
+ tracker._ensure_estimates_loaded()
+ self.assertTrue(tracker._estimates_loaded)
+
+ # 2. get_stats: falls back to in-memory stats, doesn't raise
+ stats = tracker.get_stats()
+ self.assertEqual(stats, [])
+
+ # 3. _update_estimate: logs warning, doesn't raise
+ tracker.recent_attempts["searxng"] = deque(
+ [
+ {
+ "wait_time": 1.0,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ {
+ "wait_time": 1.5,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ {
+ "wait_time": 2.0,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ ]
+ )
+ tracker._update_estimate("searxng")
+
+ # 4. reset_engine: memory cleared, doesn't raise
+ tracker.reset_engine("searxng")
+
+ # 5. cleanup_old_data: logs warning, doesn't raise
+ tracker.cleanup_old_data(days=30)
+ finally:
+ clear_search_context()
+
+ def test_get_wait_time_cold_cache_inside_in_scope_write_session(self):
+ """get_wait_time triggering _ensure_estimates_loaded on cold cache within active write session."""
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ tracker = self._build_tracker()
+ tracker._estimates_loaded = False
+
+ mock_session = MagicMock()
+ mock_estimate = MagicMock()
+ mock_estimate.engine_type = "searxng"
+ mock_estimate.base_wait_seconds = 3.5
+ mock_estimate.min_wait_seconds = 1.0
+ mock_estimate.max_wait_seconds = 10.0
+ mock_estimate.last_updated = time.time()
+ mock_session.query.return_value.all.return_value = [mock_estimate]
+
+ mock_get_user_db = MagicMock()
+ mock_get_user_db.return_value.__enter__.return_value = mock_session
+ mock_get_user_db.return_value.__exit__.return_value = False
+ mock_writer = MagicMock()
+ mock_tsm = MagicMock()
+ mock_tsm.in_scope.return_value = True
+
+ set_search_context({"username": "testuser", "user_password": "secret"})
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}.is_ci_environment",
+ return_value=False,
+ ),
+ patch(
+ f"{self.TRACKER_MOD}._get_db_imports",
+ return_value={
+ "thread_session_manager": mock_tsm,
+ "get_user_db_session": mock_get_user_db,
+ "RateLimitEstimate": MagicMock(),
+ },
+ ),
+ patch(
+ "local_deep_research.database.thread_metrics.metrics_writer",
+ mock_writer,
+ ),
+ ):
+ wait_time = tracker.get_wait_time("searxng")
+
+ self.assertTrue(tracker._estimates_loaded)
+ self.assertGreaterEqual(wait_time, 1.0)
+ self.assertLessEqual(wait_time, 10.0)
+ mock_get_user_db.assert_called_once_with("testuser", "secret")
+ self.assertFalse(mock_writer.get_session.called)
+ self.assertFalse(mock_session.commit.called)
+ finally:
+ clear_search_context()
+
+ def test_ensure_estimates_loaded_applies_decay(self):
+ """Estimates older than 24 hours decay confidence according to decay_per_day."""
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ tracker = self._build_tracker()
+ tracker._estimates_loaded = False
+ tracker.decay_per_day = 0.5
+
+ now = time.time()
+ mock_session = MagicMock()
+ mock_estimate = MagicMock()
+ mock_estimate.engine_type = "test_engine"
+ mock_estimate.base_wait_seconds = 4.0
+ mock_estimate.min_wait_seconds = 1.0
+ mock_estimate.max_wait_seconds = 8.0
+ mock_estimate.last_updated = now - 86400
+ mock_session.query.return_value.all.return_value = [mock_estimate]
+
+ mock_get_user_db = MagicMock()
+ mock_get_user_db.return_value.__enter__.return_value = mock_session
+ mock_get_user_db.return_value.__exit__.return_value = False
+ mock_tsm = MagicMock()
+ mock_tsm.in_scope.return_value = True
+
+ set_search_context({"username": "testuser", "user_password": "secret"})
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}.is_ci_environment",
+ return_value=False,
+ ),
+ patch(
+ f"{self.TRACKER_MOD}._get_db_imports",
+ return_value={
+ "thread_session_manager": mock_tsm,
+ "get_user_db_session": mock_get_user_db,
+ "RateLimitEstimate": MagicMock(),
+ },
+ ),
+ ):
+ tracker._ensure_estimates_loaded()
+
+ self.assertIn("test_engine", tracker.current_estimates)
+ self.assertAlmostEqual(
+ tracker.current_estimates["test_engine"]["confidence"],
+ 0.5,
+ places=2,
+ )
+ finally:
+ clear_search_context()
+
+ # --- 8. Real SQLite transactional integrity tests ---
+
+ def test_real_sqlite_in_scope_preserves_caller_rollback_atomicity(self):
+ """Caller's uncommitted work must NOT be prematurely committed by tracker writes,
+ allowing the caller to cleanly roll back all changes.
+ """
+ from sqlalchemy import create_engine
+ from sqlalchemy.orm import sessionmaker
+ from local_deep_research.database.models import (
+ Base,
+ RateLimitAttempt,
+ RateLimitEstimate,
+ )
+ from local_deep_research.database.session_context import (
+ get_user_db_session,
+ )
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ engine = create_engine("sqlite:///:memory:")
+ Base.metadata.create_all(engine)
+ TestSession = sessionmaker(bind=engine)
+ real_session = TestSession()
+
+ tracker = self._build_tracker()
+ set_search_context(
+ {"username": "realuser", "user_password": "realpassword"}
+ )
+
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}.is_ci_environment",
+ return_value=False,
+ ),
+ patch(
+ "local_deep_research.database.thread_local_session.get_metrics_session",
+ return_value=real_session,
+ ),
+ ):
+ with get_user_db_session(
+ "realuser", "realpassword"
+ ) as outer_session:
+ caller_attempt = RateLimitAttempt(
+ engine_type="caller_staged_engine",
+ wait_time=1.23,
+ success=True,
+ retry_count=1,
+ timestamp=time.time(),
+ )
+ outer_session.add(caller_attempt)
+
+ # Populate attempts so _update_estimate persists to DB
+ tracker.recent_attempts["searxng"] = deque(
+ [
+ {
+ "wait_time": 1.0,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ {
+ "wait_time": 1.5,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ {
+ "wait_time": 2.0,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ ]
+ )
+ # Trigger writes across tracker write methods
+ tracker._update_estimate("searxng")
+ self.assertIsNotNone(
+ outer_session.query(RateLimitEstimate)
+ .filter_by(engine_type="searxng")
+ .first()
+ )
+ tracker.reset_engine("other_engine")
+ tracker.cleanup_old_data(days=30)
+
+ # Caller rolls back their transaction
+ outer_session.rollback()
+
+ # Caller row must NOT be committed to SQLite
+ persisted_caller = (
+ real_session.query(RateLimitAttempt)
+ .filter_by(engine_type="caller_staged_engine")
+ .first()
+ )
+ self.assertIsNone(persisted_caller)
+ finally:
+ clear_search_context()
+ real_session.close()
+ engine.dispose()
+
+ def test_real_sqlite_tracker_write_failure_does_not_destroy_caller_writes(
+ self,
+ ):
+ """Simulated tracker write failure inside savepoint must not abort or destroy
+ caller's uncommitted writes.
+ """
+ from sqlalchemy import create_engine
+ from sqlalchemy.orm import sessionmaker
+ from local_deep_research.database.models import (
+ Base,
+ RateLimitAttempt,
+ )
+ from local_deep_research.database.session_context import (
+ get_user_db_session,
+ )
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ engine = create_engine("sqlite:///:memory:")
+ Base.metadata.create_all(engine)
+ TestSession = sessionmaker(bind=engine)
+ real_session = TestSession()
+
+ tracker = self._build_tracker()
+ set_search_context(
+ {"username": "realuser", "user_password": "realpassword"}
+ )
+
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}.is_ci_environment",
+ return_value=False,
+ ),
+ patch(
+ "local_deep_research.database.thread_local_session.get_metrics_session",
+ return_value=real_session,
+ ),
+ ):
+ with get_user_db_session(
+ "realuser", "realpassword"
+ ) as outer_session:
+ caller_attempt = RateLimitAttempt(
+ engine_type="caller_surviving_engine",
+ wait_time=2.34,
+ success=True,
+ retry_count=1,
+ timestamp=time.time(),
+ )
+ outer_session.add(caller_attempt)
+
+ # Tracker write failure inside savepoint
+ orig_add = real_session.add
+
+ def failing_add(instance):
+ if (
+ hasattr(instance, "engine_type")
+ and instance.engine_type == "fail_engine"
+ ):
+ raise RuntimeError("Simulated inner write failure")
+ return orig_add(instance)
+
+ with patch.object(
+ real_session, "add", side_effect=failing_add
+ ):
+ tracker.recent_attempts["fail_engine"] = deque(
+ [
+ {
+ "wait_time": 1.0,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ {
+ "wait_time": 1.5,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ {
+ "wait_time": 2.0,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ ]
+ )
+ tracker._update_estimate("fail_engine")
+
+ # Caller cleanly commits their outer transaction
+ outer_session.commit()
+
+ persisted_caller = (
+ real_session.query(RateLimitAttempt)
+ .filter_by(engine_type="caller_surviving_engine")
+ .first()
+ )
+ self.assertIsNotNone(persisted_caller)
+ self.assertEqual(persisted_caller.wait_time, 2.34)
+ finally:
+ clear_search_context()
+ real_session.close()
+ engine.dispose()
+
+ def test_real_sqlite_tracker_read_failure_does_not_destroy_caller_writes(
+ self,
+ ):
+ """Simulated tracker read failure in _ensure_estimates_loaded must not rollback
+ caller's uncommitted outer transaction.
+ """
+ from sqlalchemy import create_engine
+ from sqlalchemy.orm import sessionmaker
+ from local_deep_research.database.models import (
+ Base,
+ RateLimitAttempt,
+ RateLimitEstimate,
+ )
+ from local_deep_research.database.session_context import (
+ get_user_db_session,
+ )
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ engine = create_engine("sqlite:///:memory:")
+ Base.metadata.create_all(engine)
+ TestSession = sessionmaker(bind=engine)
+ real_session = TestSession()
+
+ tracker = self._build_tracker()
+ tracker._estimates_loaded = False
+ set_search_context(
+ {"username": "realuser", "user_password": "realpassword"}
+ )
+
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}.is_ci_environment",
+ return_value=False,
+ ),
+ patch(
+ "local_deep_research.database.thread_local_session.get_metrics_session",
+ return_value=real_session,
+ ),
+ ):
+ with get_user_db_session(
+ "realuser", "realpassword"
+ ) as outer_session:
+ caller_attempt = RateLimitAttempt(
+ engine_type="caller_read_survivor",
+ wait_time=3.45,
+ success=True,
+ retry_count=1,
+ timestamp=time.time(),
+ )
+ outer_session.add(caller_attempt)
+
+ orig_query = real_session.query
+
+ def failing_query(*args, **kwargs):
+ if args and args[0] is RateLimitEstimate:
+ raise RuntimeError("Simulated transient read error")
+ return orig_query(*args, **kwargs)
+
+ with patch.object(
+ real_session, "query", side_effect=failing_query
+ ):
+ tracker._ensure_estimates_loaded()
+
+ # Caller commits outer transaction cleanly
+ outer_session.commit()
+
+ persisted_caller = (
+ real_session.query(RateLimitAttempt)
+ .filter_by(engine_type="caller_read_survivor")
+ .first()
+ )
+ self.assertIsNotNone(persisted_caller)
+ self.assertEqual(persisted_caller.wait_time, 3.45)
+ finally:
+ clear_search_context()
+ real_session.close()
+ engine.dispose()
+
+ def test_real_sqlite_cold_read_under_writer_contention_preserves_caller(
+ self,
+ ):
+ """Cold-cache read in scope must not autoflush caller writes through a competing SQLite writer.
+
+ The production sessionmaker uses ``autoflush=True``. A bare
+ ``query()`` in ``_ensure_estimates_loaded`` or ``get_stats`` would
+ otherwise flush a pending caller INSERT before the SELECT; under a
+ held ``BEGIN IMMEDIATE`` that flush fails and SQLAlchemy invalidates
+ the outer transaction, leaving ``is_active=False`` and the staged
+ row gone. The ``no_autoflush`` guard on both read sites prevents
+ the autoflush and lets the caller commit normally.
+ """
+ import sqlite3
+ import tempfile
+
+ from sqlalchemy import create_engine
+ from sqlalchemy.orm import sessionmaker
+ from local_deep_research.database.models import (
+ Base,
+ RateLimitAttempt,
+ RateLimitEstimate,
+ )
+ from local_deep_research.database.session_context import (
+ get_user_db_session,
+ )
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ fd, db_path = tempfile.mkstemp(suffix=".db")
+ os.close(fd)
+ db_url = "sqlite:///" + db_path.replace("\\", "/")
+ engine = create_engine(
+ db_url, connect_args={"timeout": 0.1, "check_same_thread": False}
+ )
+ Base.metadata.create_all(engine)
+ TestSession = sessionmaker(bind=engine)
+ real_session = TestSession()
+ lock_conn = None
+
+ tracker = self._build_tracker()
+ # Cold cache so ``get_wait_time`` must hit the DB on the cold path.
+ tracker._estimates_loaded = False
+ set_search_context(
+ {"username": "realuser", "user_password": "realpassword"}
+ )
+
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}.is_ci_environment",
+ return_value=False,
+ ),
+ patch(
+ "local_deep_research.database.thread_local_session.get_metrics_session",
+ return_value=real_session,
+ ),
+ ):
+ with get_user_db_session(
+ "realuser", "realpassword"
+ ) as outer_session:
+ caller_attempt = RateLimitAttempt(
+ engine_type="caller_read_contention",
+ wait_time=6.78,
+ success=True,
+ retry_count=1,
+ timestamp=time.time(),
+ )
+ outer_session.add(caller_attempt)
+
+ # Independent connection holds the writer lock so any
+ # autoflush of the caller's pending INSERT fails.
+ lock_conn = sqlite3.connect(db_path, timeout=0.1)
+ lock_conn.execute("BEGIN IMMEDIATE")
+ try:
+ # Pre-fix: this read autoflushed the staged
+ # INSERT through the held lock and rolled back the
+ # outer transaction.
+ tracker._ensure_estimates_loaded()
+ tracker.get_stats()
+ self.assertTrue(outer_session.is_active)
+ self.assertIn(caller_attempt, outer_session.new)
+ finally:
+ lock_conn.rollback()
+
+ outer_session.commit()
+
+ persisted_caller = (
+ real_session.query(RateLimitAttempt)
+ .filter_by(engine_type="caller_read_contention")
+ .first()
+ )
+ self.assertIsNotNone(persisted_caller)
+ self.assertEqual(persisted_caller.wait_time, 6.78)
+ # The estimator read did not flip ``is_active`` either.
+ persisted_estimates = real_session.query(RateLimitEstimate).all()
+ self.assertIsInstance(persisted_estimates, list)
+ finally:
+ clear_search_context()
+ real_session.close()
+ engine.dispose()
+ if lock_conn is not None:
+ lock_conn.close()
+ try:
+ os.unlink(db_path)
+ except OSError:
+ pass
+
+ def test_real_sqlite_savepoint_entry_contention_preserves_caller_transaction(
+ self,
+ ):
+ """Competing writer during savepoint entry must not destroy caller work.
+
+ Session.begin_nested() flushes pending ORM changes before SAVEPOINT.
+ A lock held by another connection makes that flush fail outside the
+ savepoint and invalidates the caller (PendingRollbackError). Staging a
+ row then invoking tracker writes under contention covers the live
+ _download_pdf shape; injecting a failure inside session.add() after
+ entry does not.
+ """
+ import sqlite3
+ import tempfile
+
+ from sqlalchemy import create_engine
+ from sqlalchemy.orm import sessionmaker
+ from local_deep_research.database.models import (
+ Base,
+ RateLimitAttempt,
+ )
+ from local_deep_research.database.session_context import (
+ get_user_db_session,
+ )
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ fd, db_path = tempfile.mkstemp(suffix=".db")
+ os.close(fd)
+ db_url = "sqlite:///" + db_path.replace("\\", "/")
+ engine = create_engine(
+ db_url, connect_args={"timeout": 0.1, "check_same_thread": False}
+ )
+ Base.metadata.create_all(engine)
+ TestSession = sessionmaker(bind=engine)
+ real_session = TestSession()
+ lock_conn = None
+ nested_calls = []
+
+ tracker = self._build_tracker()
+ set_search_context(
+ {"username": "realuser", "user_password": "realpassword"}
+ )
+
+ orig_begin_nested = real_session.begin_nested
+
+ def counting_begin_nested(*args, **kwargs):
+ nested_calls.append(1)
+ return orig_begin_nested(*args, **kwargs)
+
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}.is_ci_environment",
+ return_value=False,
+ ),
+ patch(
+ "local_deep_research.database.thread_local_session.get_metrics_session",
+ return_value=real_session,
+ ),
+ ):
+ with get_user_db_session(
+ "realuser", "realpassword"
+ ) as outer_session:
+ caller_attempt = RateLimitAttempt(
+ engine_type="caller_contention_engine",
+ wait_time=4.56,
+ success=True,
+ retry_count=1,
+ timestamp=time.time(),
+ )
+ outer_session.add(caller_attempt)
+
+ tracker.recent_attempts["searxng"] = deque(
+ [
+ {
+ "wait_time": 1.0,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ {
+ "wait_time": 1.5,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ {
+ "wait_time": 2.0,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ ]
+ )
+
+ real_session.begin_nested = counting_begin_nested
+ lock_conn = sqlite3.connect(db_path, timeout=0.1)
+ lock_conn.execute("BEGIN IMMEDIATE")
+ try:
+ tracker._update_estimate("searxng")
+ tracker.reset_engine("other_engine")
+ tracker.cleanup_old_data(days=30)
+ self.assertTrue(outer_session.is_active)
+ finally:
+ lock_conn.rollback()
+
+ self.assertGreaterEqual(len(nested_calls), 3)
+ outer_session.commit()
+
+ persisted_caller = (
+ real_session.query(RateLimitAttempt)
+ .filter_by(engine_type="caller_contention_engine")
+ .first()
+ )
+ self.assertIsNotNone(persisted_caller)
+ self.assertEqual(persisted_caller.wait_time, 4.56)
+ finally:
+ clear_search_context()
+ real_session.begin_nested = orig_begin_nested
+ real_session.close()
+ engine.dispose()
+ if lock_conn is not None:
+ lock_conn.close()
+ try:
+ os.unlink(db_path)
+ except OSError:
+ pass
+
+ def test_real_sqlite_cascade_pending_parent_child_preserves_caller_transaction(
+ self,
+ ):
+ """Caller-staged parent/child with cascade delete-orphan must survive tracker writes.
+
+ ``Session.expunge(parent)`` cascades to children under
+ ``cascade="all, delete-orphan"`` (e.g. ``ResearchHistory.resources``).
+ Re-expunging the already-detached child raises ``InvalidRequestError``
+ and used to leak caller objects out of the session, so a later
+ ``commit()`` would silently drop them. The cascade-safe expunge
+ must skip the already-detached child and ``add(parent)`` must
+ re-attach both via cascade, so the caller can commit normally.
+ """
+ import tempfile
+ import uuid
+
+ from sqlalchemy import create_engine
+ from sqlalchemy.orm import sessionmaker
+ from local_deep_research.database.models import (
+ Base,
+ ResearchHistory,
+ ResearchResource,
+ )
+ from local_deep_research.database.session_context import (
+ get_user_db_session,
+ )
+ from local_deep_research.utilities.thread_context import (
+ clear_search_context,
+ set_search_context,
+ )
+
+ fd, db_path = tempfile.mkstemp(suffix=".db")
+ os.close(fd)
+ db_url = "sqlite:///" + db_path.replace("\\", "/")
+ engine = create_engine(
+ db_url, connect_args={"check_same_thread": False}
+ )
+ Base.metadata.create_all(engine)
+ TestSession = sessionmaker(bind=engine)
+ real_session = TestSession()
+
+ tracker = self._build_tracker()
+ set_search_context(
+ {"username": "realuser", "user_password": "realpassword"}
+ )
+
+ parent_id = str(uuid.uuid4())
+ try:
+ with (
+ patch(
+ f"{self.TRACKER_MOD}.is_ci_environment",
+ return_value=False,
+ ),
+ patch(
+ "local_deep_research.database.thread_local_session.get_metrics_session",
+ return_value=real_session,
+ ),
+ ):
+ with get_user_db_session(
+ "realuser", "realpassword"
+ ) as outer_session:
+ parent = ResearchHistory(
+ id=parent_id,
+ query="cascade_test_query",
+ status="in_progress",
+ mode="quick",
+ created_at="2026-09-20T00:00:00+00:00",
+ )
+ child = ResearchResource(
+ research=parent,
+ title="cascade_child_title",
+ url="https://example.invalid/cascade",
+ created_at="2026-09-20T00:00:00+00:00",
+ )
+ # Pin the parent-first ordering of ``session.new``: add
+ # the parent first, then attach the child via cascade
+ # (``parent.resources.append`` triggers cascade-add of
+ # the child once the parent is in the session). This
+ # guarantees the parent sits before the child in
+ # ``session.new``, so the regression cannot silently
+ # pass on the pre-fix tree when the child happens to
+ # come first by id order.
+ outer_session.add(parent)
+ parent.resources.append(child)
+
+ self.assertTrue(parent in outer_session)
+ self.assertTrue(child in outer_session)
+ new_list = list(outer_session.new)
+ self.assertGreater(
+ new_list.index(parent),
+ -1,
+ "parent must be in session.new",
+ )
+ self.assertGreater(
+ new_list.index(child),
+ -1,
+ "child must be in session.new",
+ )
+ self.assertLess(
+ new_list.index(parent),
+ new_list.index(child),
+ "parent must be inserted before child in session.new",
+ )
+
+ # Trigger every write site so all three exercise the
+ # cascade-safe expunge path with the parent/child pair
+ # still pending in the outer session.
+ tracker.recent_attempts["searxng"] = deque(
+ [
+ {
+ "wait_time": 1.0,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ {
+ "wait_time": 1.5,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ {
+ "wait_time": 2.0,
+ "success": True,
+ "timestamp": time.time(),
+ "retry_count": 1,
+ "search_result_count": 5,
+ },
+ ]
+ )
+ tracker._update_estimate("searxng")
+ tracker.reset_engine("other_engine")
+ tracker.cleanup_old_data(days=30)
+
+ # Caller's transaction is still active and parent +
+ # child remain pending in the session.
+ self.assertTrue(outer_session.is_active)
+ self.assertIn(parent, outer_session)
+ self.assertIn(child, outer_session)
+ self.assertIn(parent, outer_session.new)
+ self.assertIn(child, outer_session.new)
+
+ outer_session.commit()
+ self.assertTrue(outer_session.is_active)
+
+ persisted_parent = (
+ real_session.query(ResearchHistory)
+ .filter_by(id=parent_id)
+ .one_or_none()
+ )
+ self.assertIsNotNone(persisted_parent)
+ self.assertEqual(persisted_parent.query, "cascade_test_query")
+ persisted_children = (
+ real_session.query(ResearchResource)
+ .filter_by(research_id=parent_id)
+ .all()
+ )
+ self.assertEqual(len(persisted_children), 1)
+ self.assertEqual(persisted_children[0].title, "cascade_child_title")
+ finally:
+ clear_search_context()
+ real_session.close()
+ engine.dispose()
+ try:
+ os.unlink(db_path)
+ except OSError:
+ pass
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/web_search_engines/rate_limiting/test_tracker_coverage.py b/tests/web_search_engines/rate_limiting/test_tracker_coverage.py
index 60f2102329..a1baef7a87 100644
--- a/tests/web_search_engines/rate_limiting/test_tracker_coverage.py
+++ b/tests/web_search_engines/rate_limiting/test_tracker_coverage.py
@@ -132,6 +132,42 @@ class TestEnsureEstimatesLoadedDB:
assert tracker.current_estimates["TE"]["base"] == 2.0
assert abs(tracker.current_estimates["TE"]["confidence"] - 0.95) < 0.01
+ def test_query_runs_inside_no_autoflush(self):
+ """Loads estimates inside ``session.no_autoflush`` to keep the
+ read from flushing the caller's pending ORM writes.
+
+ The production sessionmaker defaults to ``autoflush=True`` and a
+ bare ``query()`` would otherwise flush the caller's pending
+ INSERTs before the SELECT; under a competing SQLite writer that
+ flush fails and rolls back the outer transaction.
+ ``session.no_autoflush`` keeps the read atomic.
+ """
+ tracker = _make_tracker()
+ tracker._estimates_loaded = False
+ ms = MagicMock()
+ ms.query.return_value.all.return_value = []
+ mw = MagicMock()
+ mw.get_session = _fake_ctx_mgr(ms)
+ imports = {
+ "RateLimitEstimate": MagicMock(),
+ "get_user_db_session": MagicMock(),
+ }
+ ctx = {"username": "alice", "user_password": "secret"}
+ with patch(f"{MODULE}._get_db_imports", return_value=imports):
+ with patch(f"{MODULE}.get_search_context", return_value=ctx):
+ with patch.dict(
+ "sys.modules",
+ {
+ "local_deep_research.database.thread_metrics": MagicMock(
+ metrics_writer=mw
+ )
+ },
+ ):
+ tracker._ensure_estimates_loaded()
+ assert ms.no_autoflush.__enter__.called
+ assert ms.no_autoflush.__exit__.called
+ assert ms.query.called
+
def test_db_exception_marks_loaded(self):
"""Database exception during load still marks loaded."""
tracker = _make_tracker()
@@ -341,6 +377,32 @@ class TestGetStatsDB:
assert len(result) == 1
assert result[0][0] == "G"
+ @patch(f"{MODULE}.is_ci_environment", return_value=False)
+ @patch(
+ f"{MODULE}.get_search_context",
+ return_value={"username": "u", "user_password": "p"},
+ )
+ def test_query_runs_inside_no_autoflush(self, _ctx, _ci):
+ """get_stats runs query inside ``session.no_autoflush`` so a
+ cold read cannot flush the caller's pending ORM writes through
+ a competing SQLite writer.
+ """
+ tracker = _make_tracker(programmatic_mode=False)
+ ms = MagicMock()
+ ms.query.return_value.order_by.return_value.all.return_value = []
+ mock_writer = MagicMock()
+ mock_writer.get_session = _fake_ctx_mgr(ms)
+ imports = {"RateLimitEstimate": MagicMock()}
+ with patch(f"{MODULE}._get_db_imports", return_value=imports):
+ with patch(
+ "local_deep_research.database.thread_metrics.metrics_writer",
+ mock_writer,
+ ):
+ tracker.get_stats()
+ assert ms.no_autoflush.__enter__.called
+ assert ms.no_autoflush.__exit__.called
+ assert ms.query.called
+
@patch(f"{MODULE}.is_ci_environment", return_value=False)
@patch(
f"{MODULE}.get_search_context",
@@ -393,7 +455,7 @@ class TestResetEngineDB:
return_value={"username": "u", "user_password": "p"},
)
def test_deletes_from_db(self, _ctx, _ci):
- """Deletes from DB and commits."""
+ """Deletes from DB within savepoint without committing shared transaction."""
tracker = _make_tracker(programmatic_mode=False)
tracker.recent_attempts["Eng"] = deque(
[_attempt(1.0, True)], maxlen=100
@@ -419,7 +481,8 @@ class TestResetEngineDB:
tracker.reset_engine("Eng")
assert "Eng" not in tracker.recent_attempts
assert "Eng" not in tracker.current_estimates
- ms.commit.assert_called_once()
+ ms.begin_nested.assert_called_once()
+ ms.commit.assert_not_called()
@patch(f"{MODULE}.is_ci_environment", return_value=False)
@patch(
@@ -470,7 +533,7 @@ class TestCleanupOldDataDB:
return_value={"username": "u", "user_password": "p"},
)
def test_full_cleanup_deletes_old_rows(self, _ctx, _ci):
- """Deletes old attempts from DB and commits."""
+ """Deletes old attempts from DB within savepoint without committing shared transaction."""
tracker = _make_tracker(programmatic_mode=False)
mq = MagicMock()
mq.count.return_value = 42
@@ -486,7 +549,8 @@ class TestCleanupOldDataDB:
):
tracker.cleanup_old_data(days=30)
mq.delete.assert_called_once()
- ms.commit.assert_called_once()
+ ms.begin_nested.assert_called_once()
+ ms.commit.assert_not_called()
@patch(f"{MODULE}.is_ci_environment", return_value=False)
@patch(
@@ -494,7 +558,7 @@ class TestCleanupOldDataDB:
return_value={"username": "u", "user_password": "p"},
)
def test_no_old_data_still_commits(self, _ctx, _ci):
- """When no old data exists, still commits."""
+ """When no old data exists, still uses savepoint discipline."""
tracker = _make_tracker(programmatic_mode=False)
mq = MagicMock()
mq.count.return_value = 0
@@ -509,7 +573,8 @@ class TestCleanupOldDataDB:
mock_writer,
):
tracker.cleanup_old_data(days=7)
- ms.commit.assert_called_once()
+ ms.begin_nested.assert_called_once()
+ ms.commit.assert_not_called()
@patch(f"{MODULE}.is_ci_environment", return_value=False)
@patch(