~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(rate-limit): eliminate latent multi-connection sites in tracker.py (#6562) (#6566)
Sc3f5b767be9f
diff --git a/changelog.d/6562.bugfix.md b/changelog.d/6562.bugfix.mdnew file mode 100644index 0000000000..1dd3cac3d3--- /dev/null+++ b/changelog.d/6562.bugfix.md@@ -0,0 +1 @@+Eliminate separate database connection checkouts across `AdaptiveRateLimitTracker` (`_ensure_estimates_loaded`, `get_stats`, `_update_estimate`, `reset_engine`, and `cleanup_old_data`) by reusing in-scope thread-local sessions with savepoint (`begin_nested()`) isolation for writes and `session.no_autoflush` for reads, preventing SQLite `database is locked` deadlocks and preserving outer transaction atomicity under writer contention. Read sites can no longer autoflush the caller's pending ORM rows before the SELECT under a competing SQLite writer; write sites still detach pending state (including `cascade="all, delete-orphan"` children such as `ResearchHistory.resources`) before SAVEPOINT so a savepoint-entry flush cannot invalidate the outer transaction. The detach is exception-safe so a half-failed tracker write cannot leak detached caller objects into the outer transaction.diff --git a/src/local_deep_research/web_search_engines/rate_limiting/tracker.py b/src/local_deep_research/web_search_engines/rate_limiting/tracker.pyindex 067cd38504..711b22057d 100644--- a/src/local_deep_research/web_search_engines/rate_limiting/tracker.py+++ b/src/local_deep_research/web_search_engines/rate_limiting/tracker.py@@ -6,8 +6,17 @@ import randomimport threadingimport timefrom collections import deque-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple-+from contextlib import AbstractContextManager, contextmanager+from typing import (+ TYPE_CHECKING,+ Any,+ Dict,+ Iterator,+ List,+ Literal,+ Optional,+ Tuple,+)from cachetools import LRUCacheif TYPE_CHECKING:@@ -33,11 +42,13 @@ def _get_db_imports():try:from ...database.models import RateLimitAttempt, RateLimitEstimatefrom ...database.session_context import get_user_db_session+ from ...database.thread_local_session import thread_session_manager_db_imports = {"RateLimitAttempt": RateLimitAttempt,"RateLimitEstimate": RateLimitEstimate,"get_user_db_session": get_user_db_session,+ "thread_session_manager": thread_session_manager,}except (ImportError, RuntimeError):# Database not available - programmatic mode@@ -45,6 +56,30 @@ def _get_db_imports():return _db_imports+class _InScopeSessionContext:+ """Context manager wrapping an in-scope session to shield outer transactions.++ Calls the underlying context manager's __exit__ with (None, None, None)+ so internal exceptions do not trigger safe_rollback on the caller's+ shared session, while propagating the exception to the caller.+ """++ def __init__(self, cm: Any) -> None:+ self._cm = cm++ def __enter__(self) -> Any:+ return self._cm.__enter__()++ def __exit__(+ self,+ exc_type: Optional[type[BaseException]],+ exc_val: Optional[BaseException],+ exc_tb: Optional[Any],+ ) -> Literal[False]:+ self._cm.__exit__(None, None, None)+ return False++class AdaptiveRateLimitTracker:"""Tracks and learns optimal retry wait times for each search engine.@@ -159,6 +194,155 @@ class AdaptiveRateLimitTracker:"Rate limit estimates will be loaded on-demand when user context is available")+ def _resolve_session(+ self, username: str, password: str+ ) -> AbstractContextManager["Session"]:+ """Unified session resolution helper for database operations.++ Reuses the thread-local session when an enclosing scope is active+ (avoiding a second QueuePool connection and SQLite busy-timeout deadlock);+ otherwise falls back to a dedicated session that commits and closes on exit.+ """+ db_imports = _get_db_imports() or {}+ thread_session_manager = db_imports.get("thread_session_manager")+ if thread_session_manager is None:+ try:+ from ...database.thread_local_session import (+ thread_session_manager,+ )+ except (ImportError, RuntimeError):+ thread_session_manager = None++ get_user_db_session = db_imports.get("get_user_db_session")+ if get_user_db_session is None:+ try:+ from ...database.session_context import get_user_db_session+ except (ImportError, RuntimeError):+ get_user_db_session = None++ if (+ thread_session_manager is not None+ and thread_session_manager.in_scope()+ and get_user_db_session is not None+ ):+ return _InScopeSessionContext(+ get_user_db_session(username, password)+ )++ from ...database.thread_metrics import metrics_writer++ metrics_writer.set_user_password(username, password)+ return metrics_writer.get_session(username)++ @contextmanager+ def _isolated_begin_nested(self, session: "Session") -> Iterator[None]:+ """Open a SAVEPOINT without flushing pre-existing pending ORM state.++ ``Session.begin_nested()`` flushes every pending change before it+ issues SAVEPOINT. If another connection holds SQLite's writer lock,+ that flush fails outside the savepoint and SQLAlchemy invalidates+ the caller's transaction. Detach already-pending instances first so+ tracker writes cannot flush them until after the isolation boundary+ exists; restore them after the savepoint settles.++ Both expunge and restore are exception-safe: cascade-expelled+ children (``cascade="all, delete-orphan"`` such as+ ``ResearchHistory.resources``) are skipped instead of raising+ ``InvalidRequestError``, and any failure during expunge or restore+ is contained so a half-failed tracker write cannot leave caller+ objects detached from the session.+ """+ pending_new = list(getattr(session, "new", ()) or ())+ pending_deleted = list(getattr(session, "deleted", ()) or ())+ skip_ids = {id(obj) for obj in pending_new}+ skip_ids.update(id(obj) for obj in pending_deleted)+ pending_dirty = [+ obj+ for obj in list(getattr(session, "dirty", ()) or ())+ if id(obj) not in skip_ids+ ]++ # Dedupe by identity and skip objects already detached (e.g. by an+ # earlier cascade expunge of a parent). ``expunge`` cascades along+ # ``cascade="all, delete-orphan"`` relationships, so a parent ++ # child both pending must not be expunged twice.+ ordered: list[Any] = []+ seen: set[int] = set()+ for obj in (*pending_new, *pending_dirty, *pending_deleted):+ if id(obj) in seen:+ continue+ seen.add(id(obj))+ ordered.append(obj)++ for obj in ordered:+ if obj not in session:+ # Cascade already detached this object when its parent was+ # expunged; it remains in pending_new/dirty/deleted by+ # Python identity, but not by session membership.+ continue+ try:+ session.expunge(obj)+ except Exception:+ # ``expunge`` can raise for already-detached instances or+ # for objects the session has lost track of. Swallow and+ # continue so any later objects can still be detached and+ # later we still attempt full restoration below.+ logger.debug(+ "Tracker isolation: expunge failed during pre-flush detach",+ exc_info=True,+ )++ try:+ with session.begin_nested():+ yield+ finally:+ self._restore_pending_state(+ session, pending_new, pending_dirty, pending_deleted+ )++ def _restore_pending_state(+ self,+ session: "Session",+ pending_new: list[Any],+ pending_dirty: list[Any],+ pending_deleted: list[Any],+ ) -> None:+ """Best-effort restoration of expunged caller objects.++ Adds new/dirty objects back to the session and re-stages deleted+ objects. ``add(parent)`` cascades to children for relationships+ declared with ``cascade="all, delete-orphan"``, so we skip any+ child already re-attached by cascade to avoid double-add errors.+ All restoration steps are individually exception-safe: a failure+ here must never propagate out and detach caller state.+ """+ for obj in (*pending_new, *pending_dirty):+ try:+ if obj not in session:+ session.add(obj)+ except Exception:+ logger.debug(+ "Tracker isolation: failed to re-attach caller object",+ exc_info=True,+ )+ for obj in pending_deleted:+ try:+ if obj not in session:+ session.add(obj)+ except Exception:+ logger.debug(+ "Tracker isolation: failed to re-add deleted caller object",+ exc_info=True,+ )+ continue+ try:+ session.delete(obj)+ except Exception:+ logger.debug(+ "Tracker isolation: failed to re-stage deleted caller object",+ exc_info=True,+ )+def _ensure_estimates_loaded(self) -> None:"""Load estimates from user's encrypted database if not already loaded."""# Early return if already loaded or should skip@@ -191,34 +375,38 @@ class AdaptiveRateLimitTracker:if username and password:try:- # Use thread-safe metrics writer to read from user's encrypted database- from ...database.thread_metrics import metrics_writer-- # Set password for this thread- metrics_writer.set_user_password(username, password)-session: "Session"- with metrics_writer.get_session(username) as session:- estimates: list[Any] = session.query(- RateLimitEstimate- ).all()-- for estimate in estimates:- # Apply decay for old estimates- age_hours = (time.time() - estimate.last_updated) / 3600- decay = self.decay_per_day ** (age_hours / 24)-- with self._cache_lock:- self.current_estimates[estimate.engine_type] = {- "base": estimate.base_wait_seconds,- "min": estimate.min_wait_seconds,- "max": estimate.max_wait_seconds,- "confidence": decay,- }-- logger.debug(- f"Loaded estimate for {estimate.engine_type}: base={estimate.base_wait_seconds:.2f}s, confidence={decay:.2f}"- )+ with self._resolve_session(username, password) as session:+ # The production sessionmaker defaults to+ # ``autoflush=True``, so a bare ``query()`` could flush+ # the caller's pending ORM writes before the SELECT+ # runs. Under a competing SQLite writer that flush+ # fails and SQLAlchemy invalidates the outer+ # transaction; ``no_autoflush`` keeps the read from+ # touching unrelated caller state.+ with session.no_autoflush:+ estimates: list[Any] = session.query(+ RateLimitEstimate+ ).all()++ for estimate in estimates:+ # Apply decay for old estimates+ age_hours = (+ time.time() - estimate.last_updated+ ) / 3600+ decay = self.decay_per_day ** (age_hours / 24)++ with self._cache_lock:+ self.current_estimates[estimate.engine_type] = {+ "base": estimate.base_wait_seconds,+ "min": estimate.min_wait_seconds,+ "max": estimate.max_wait_seconds,+ "confidence": decay,+ }++ logger.debug(+ f"Loaded estimate for {estimate.engine_type}: base={estimate.base_wait_seconds:.2f}s, confidence={decay:.2f}"+ )self._estimates_loaded = Truelogger.info(@@ -480,44 +668,47 @@ class AdaptiveRateLimitTracker:if username and password:try:- # Use thread-safe metrics writer to save to user's encrypted database- from ...database.thread_metrics import metrics_writer-- # Set password for this thread if not already set- metrics_writer.set_user_password(username, password)-db_imports = _get_db_imports()- RateLimitEstimate = db_imports.get("RateLimitEstimate")+ RateLimitEstimate = (+ db_imports.get("RateLimitEstimate")+ if db_imports+ else None+ )+ if not RateLimitEstimate:+ returnsession_inner: "Session"- with metrics_writer.get_session(username) as session_inner:- # Check if estimate exists- estimate: Any = (- session_inner.query(RateLimitEstimate)- .filter_by(engine_type=engine_type)- .first()- )-- if estimate:- # Update existing estimate- estimate.base_wait_seconds = new_base- estimate.min_wait_seconds = min_wait- estimate.max_wait_seconds = max_wait- estimate.last_updated = time.time()- estimate.total_attempts = len(attempts)- estimate.success_rate = success_rate- else:- # Create new estimate- estimate = RateLimitEstimate(- engine_type=engine_type,- base_wait_seconds=new_base,- min_wait_seconds=min_wait,- max_wait_seconds=max_wait,- last_updated=time.time(),- total_attempts=len(attempts),- success_rate=success_rate,+ with self._resolve_session(+ username, password+ ) as session_inner:+ with self._isolated_begin_nested(session_inner):+ # Check if estimate exists+ estimate: Any = (+ session_inner.query(RateLimitEstimate)+ .filter_by(engine_type=engine_type)+ .first())- session_inner.add(estimate)++ if estimate:+ # Update existing estimate+ estimate.base_wait_seconds = new_base+ estimate.min_wait_seconds = min_wait+ estimate.max_wait_seconds = max_wait+ estimate.last_updated = time.time()+ estimate.total_attempts = len(attempts)+ estimate.success_rate = success_rate+ else:+ # Create new estimate+ estimate = RateLimitEstimate(+ engine_type=engine_type,+ base_wait_seconds=new_base,+ min_wait_seconds=min_wait,+ max_wait_seconds=max_wait,+ last_updated=time.time(),+ total_attempts=len(attempts),+ success_rate=success_rate,+ )+ session_inner.add(estimate)except Exception:logger.warning("Failed to persist rate limit estimate")@@ -594,24 +785,27 @@ class AdaptiveRateLimitTracker:db_imports = _get_db_imports()RateLimitEstimate = db_imports.get("RateLimitEstimate")- from ...database.thread_metrics import metrics_writer-- metrics_writer.set_user_password(username, password)-session_est: "Session"- with metrics_writer.get_session(username) as session_est:- if engine_type:- estimates: list[Any] = (- session_est.query(RateLimitEstimate)- .filter_by(engine_type=engine_type)- .all()- )- else:- estimates = (- session_est.query(RateLimitEstimate)- .order_by(RateLimitEstimate.engine_type)- .all()- )+ with self._resolve_session(username, password) as session_est:+ # See ``_ensure_estimates_loaded``: the production+ # sessionmaker defaults to ``autoflush=True`` and a cold+ # ``get_stats`` would otherwise flush caller's pending+ # writes before the SELECT, which under a competing+ # SQLite writer fails and invalidates the outer+ # transaction.+ with session_est.no_autoflush:+ if engine_type:+ estimates: list[Any] = (+ session_est.query(RateLimitEstimate)+ .filter_by(engine_type=engine_type)+ .all()+ )+ else:+ estimates = (+ session_est.query(RateLimitEstimate)+ .order_by(RateLimitEstimate.engine_type)+ .all()+ )return [(@@ -675,26 +869,27 @@ class AdaptiveRateLimitTracker:returndb_imports = _get_db_imports()- RateLimitAttempt = db_imports.get("RateLimitAttempt")- RateLimitEstimate = db_imports.get("RateLimitEstimate")-- from ...database.thread_metrics import metrics_writer-- metrics_writer.set_user_password(username, password)+ RateLimitAttempt = (+ db_imports.get("RateLimitAttempt") if db_imports else None+ )+ RateLimitEstimate = (+ db_imports.get("RateLimitEstimate") if db_imports else None+ )+ if not RateLimitAttempt or not RateLimitEstimate:+ returnsession_reset: "Session"- with metrics_writer.get_session(username) as session_reset:- # Delete historical attempts- session_reset.query(RateLimitAttempt).filter_by(- engine_type=engine_type- ).delete()-- # Delete estimates- session_reset.query(RateLimitEstimate).filter_by(- engine_type=engine_type- ).delete()-- session_reset.commit()+ with self._resolve_session(username, password) as session_reset:+ with self._isolated_begin_nested(session_reset):+ # Delete historical attempts+ session_reset.query(RateLimitAttempt).filter_by(+ engine_type=engine_type+ ).delete()++ # Delete estimates+ session_reset.query(RateLimitEstimate).filter_by(+ engine_type=engine_type+ ).delete()logger.info(f"Reset rate limit data for {engine_type}")@@ -809,22 +1004,22 @@ class AdaptiveRateLimitTracker:returndb_imports = _get_db_imports()- RateLimitAttempt = db_imports.get("RateLimitAttempt")-- from ...database.thread_metrics import metrics_writer-- metrics_writer.set_user_password(username, password)+ RateLimitAttempt = (+ db_imports.get("RateLimitAttempt") if db_imports else None+ )+ if not RateLimitAttempt:+ returnsession_clean: "Session"- with metrics_writer.get_session(username) as session_clean:- # Count and delete old attempts- old_attempts: Any = session_clean.query(- RateLimitAttempt- ).filter(RateLimitAttempt.timestamp < cutoff_time)- deleted_count = old_attempts.count()- old_attempts.delete()-- session_clean.commit()+ deleted_count = 0+ with self._resolve_session(username, password) as session_clean:+ with self._isolated_begin_nested(session_clean):+ # Count and delete old attempts+ old_attempts: Any = session_clean.query(+ RateLimitAttempt+ ).filter(RateLimitAttempt.timestamp < cutoff_time)+ deleted_count = old_attempts.count()+ old_attempts.delete()if deleted_count > 0:logger.info(f"Cleaned up {deleted_count} old retry attempts")diff --git a/tests/rate_limiting/test_rate_limiting.py b/tests/rate_limiting/test_rate_limiting.pyindex bf76ef746d..48635382c9 100644--- a/tests/rate_limiting/test_rate_limiting.py+++ b/tests/rate_limiting/test_rate_limiting.py@@ -2,9 +2,11 @@Tests for the adaptive rate limiting system."""+from collections import dequeimport os+import timeimport unittest-from unittest.mock import patch+from unittest.mock import MagicMock, patchimport pytest@@ -302,8 +304,6 @@ class TestAdaptiveRateLimitTracker(unittest.TestCase):"max": 10.0,"confidence": 0.8,}- from collections import deque-self.tracker.recent_attempts[engine_type] = deque(maxlen=100)# Verify data exists@@ -696,5 +696,1693 @@ class TestProgrammaticMode(unittest.TestCase):self.assertFalse(tracker.enabled)+class TestLatentSessionResolution(unittest.TestCase):+ """Regression coverage for unified session resolution and savepoint isolation across DB operations.++ Issue #6562: When the caller already holds a thread-local session (in_scope()),+ _ensure_estimates_loaded, get_stats, _update_estimate, reset_engine, and cleanup_old_data+ must reuse the thread-local session instead of checking out a separate connection via+ metrics_writer.get_session(), which commits on exit and self-deadlocks on SQLite locks.+ Furthermore, write operations must use savepoint discipline (begin_nested()) and never commit+ the caller's shared session, preserving outer transaction rollback atomicity.+ """++ TRACKER_MOD = "local_deep_research.web_search_engines.rate_limiting.tracker"++ def _build_tracker(self):+ settings_snapshot = {+ "rate_limiting.enabled": {"value": True, "ui_element": "checkbox"}+ }+ return AdaptiveRateLimitTracker(+ settings_snapshot=settings_snapshot, programmatic_mode=False+ )++ # --- 1. Direct _resolve_session helper tests ---++ def test_resolve_session_in_scope_uses_get_user_db_session(self):+ """When in_scope() is True, _resolve_session yields from get_user_db_session."""+ tracker = self._build_tracker()+ mock_session = MagicMock()+ mock_get_user_db = MagicMock()+ mock_get_user_db.return_value.__enter__.return_value = mock_session+ mock_get_user_db.return_value.__exit__.return_value = False+ mock_writer = MagicMock()+ mock_tsm = MagicMock()+ mock_tsm.in_scope.return_value = True++ with (+ patch(+ f"{self.TRACKER_MOD}._get_db_imports",+ return_value={+ "thread_session_manager": mock_tsm,+ "get_user_db_session": mock_get_user_db,+ },+ ),+ patch(+ "local_deep_research.database.thread_metrics.metrics_writer",+ mock_writer,+ ),+ ):+ with tracker._resolve_session("alice", "secret") as sess:+ self.assertIs(sess, mock_session)++ mock_get_user_db.assert_called_once_with("alice", "secret")+ self.assertFalse(mock_writer.get_session.called)+ self.assertFalse(mock_writer.set_user_password.called)++ def test_resolve_session_not_in_scope_uses_metrics_writer(self):+ """When in_scope() is False, _resolve_session falls back to metrics_writer."""+ tracker = self._build_tracker()+ mock_session = MagicMock()+ mock_get_user_db = MagicMock()+ mock_writer = MagicMock()+ mock_writer.get_session.return_value.__enter__.return_value = (+ mock_session+ )+ mock_writer.get_session.return_value.__exit__.return_value = False+ mock_tsm = MagicMock()+ mock_tsm.in_scope.return_value = False++ with (+ patch(+ f"{self.TRACKER_MOD}._get_db_imports",+ return_value={+ "thread_session_manager": mock_tsm,+ "get_user_db_session": mock_get_user_db,+ },+ ),+ patch(+ "local_deep_research.database.thread_metrics.metrics_writer",+ mock_writer,+ ),+ ):+ with tracker._resolve_session("alice", "secret") as sess:+ self.assertIs(sess, mock_session)++ mock_writer.set_user_password.assert_called_once_with("alice", "secret")+ mock_writer.get_session.assert_called_once_with("alice")+ self.assertFalse(mock_get_user_db.called)++ def test_resolve_session_fallback_when_db_imports_missing_helpers(self):+ """When thread_session_manager or get_user_db_session is None, fallback works."""+ tracker = self._build_tracker()+ mock_session = MagicMock()+ mock_writer = MagicMock()+ mock_writer.get_session.return_value.__enter__.return_value = (+ mock_session+ )+ mock_writer.get_session.return_value.__exit__.return_value = False++ with (+ patch(f"{self.TRACKER_MOD}._get_db_imports", return_value={}),+ patch(+ "local_deep_research.database.thread_local_session.thread_session_manager.in_scope",+ return_value=False,+ ),+ patch(+ "local_deep_research.database.thread_metrics.metrics_writer",+ mock_writer,+ ),+ ):+ with tracker._resolve_session("alice", "secret") as sess:+ self.assertIs(sess, mock_session)++ mock_writer.set_user_password.assert_called_once_with("alice", "secret")+ mock_writer.get_session.assert_called_once_with("alice")++ def test_resolve_session_exception_contained_from_get_user_db_session(self):+ """Exceptions inside in-scope session must be re-raised after get_user_db_session exits cleanly."""+ tracker = self._build_tracker()+ mock_session = MagicMock()+ mock_cm = MagicMock()+ mock_cm.__enter__.return_value = mock_session+ mock_cm.__exit__.return_value = False+ mock_get_user_db = MagicMock(return_value=mock_cm)+ mock_tsm = MagicMock()+ mock_tsm.in_scope.return_value = True++ with (+ patch(+ f"{self.TRACKER_MOD}._get_db_imports",+ return_value={+ "thread_session_manager": mock_tsm,+ "get_user_db_session": mock_get_user_db,+ },+ ),+ ):+ with self.assertRaises(RuntimeError):+ with tracker._resolve_session("alice", "secret"):+ raise RuntimeError("inner failure")++ # __exit__ must have received (None, None, None) so get_user_db_session+ # did not trigger safe_rollback on the shared session+ mock_cm.__exit__.assert_called_once_with(None, None, None)++ # --- 2. _ensure_estimates_loaded tests ---++ def test_ensure_estimates_loaded_in_scope_reuses_session_without_commit(+ self,+ ):+ """Cold cache miss during active write session must not checkout separate connection or commit."""+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ tracker = self._build_tracker()+ tracker._estimates_loaded = False++ mock_session = MagicMock()+ mock_estimate = MagicMock()+ mock_estimate.engine_type = "bing"+ mock_estimate.base_wait_seconds = 2.5+ mock_estimate.min_wait_seconds = 1.0+ mock_estimate.max_wait_seconds = 5.0+ mock_estimate.last_updated = 100000.0+ mock_session.query.return_value.all.return_value = [mock_estimate]++ mock_get_user_db = MagicMock()+ mock_get_user_db.return_value.__enter__.return_value = mock_session+ mock_get_user_db.return_value.__exit__.return_value = False+ mock_writer = MagicMock()+ mock_tsm = MagicMock()+ mock_tsm.in_scope.return_value = True++ set_search_context({"username": "testuser", "user_password": "secret"})+ try:+ with (+ patch(+ f"{self.TRACKER_MOD}._get_db_imports",+ return_value={+ "thread_session_manager": mock_tsm,+ "get_user_db_session": mock_get_user_db,+ "RateLimitEstimate": MagicMock(),+ },+ ),+ patch(+ "local_deep_research.database.thread_metrics.metrics_writer",+ mock_writer,+ ),+ ):+ tracker._ensure_estimates_loaded()++ self.assertTrue(tracker._estimates_loaded)+ self.assertIn("bing", tracker.current_estimates)+ self.assertEqual(tracker.current_estimates["bing"]["base"], 2.5)+ mock_get_user_db.assert_called_once_with("testuser", "secret")+ self.assertFalse(mock_writer.get_session.called)+ # Must NOT call commit() — reader cannot commit enclosing caller's transaction+ self.assertFalse(mock_session.commit.called)+ finally:+ clear_search_context()++ def test_ensure_estimates_loaded_not_in_scope_uses_metrics_writer(self):+ """Cold cache miss outside active session scope uses metrics_writer."""+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ tracker = self._build_tracker()+ tracker._estimates_loaded = False++ mock_session = MagicMock()+ mock_session.query.return_value.all.return_value = []+ mock_get_user_db = MagicMock()+ mock_writer = MagicMock()+ mock_writer.get_session.return_value.__enter__.return_value = (+ mock_session+ )+ mock_writer.get_session.return_value.__exit__.return_value = False+ mock_tsm = MagicMock()+ mock_tsm.in_scope.return_value = False++ set_search_context({"username": "testuser", "user_password": "secret"})+ try:+ with (+ patch(+ f"{self.TRACKER_MOD}._get_db_imports",+ return_value={+ "thread_session_manager": mock_tsm,+ "get_user_db_session": mock_get_user_db,+ "RateLimitEstimate": MagicMock(),+ },+ ),+ patch(+ "local_deep_research.database.thread_metrics.metrics_writer",+ mock_writer,+ ),+ ):+ tracker._ensure_estimates_loaded()++ self.assertTrue(tracker._estimates_loaded)+ mock_writer.set_user_password.assert_called_once_with(+ "testuser", "secret"+ )+ mock_writer.get_session.assert_called_once_with("testuser")+ self.assertFalse(mock_get_user_db.called)+ finally:+ clear_search_context()++ def test_ensure_estimates_loaded_in_scope_disables_autoflush(self):+ """Cold-cache read in scope must run inside session.no_autoflush.++ The production sessionmaker uses ``autoflush=True``. A bare+ ``query()`` would otherwise flush any pending caller writes before+ the SELECT runs; under a competing SQLite writer that flush fails+ and SQLAlchemy invalidates the caller's transaction. Pin the+ no-autoflush contract here.+ """+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ tracker = self._build_tracker()+ tracker._estimates_loaded = False+ mock_session = MagicMock()+ mock_session.query.return_value.all.return_value = []+ mock_get_user_db = MagicMock()+ mock_get_user_db.return_value.__enter__.return_value = mock_session+ mock_get_user_db.return_value.__exit__.return_value = False+ mock_writer = MagicMock()+ mock_tsm = MagicMock()+ mock_tsm.in_scope.return_value = True++ set_search_context({"username": "testuser", "user_password": "secret"})+ try:+ with (+ patch(+ f"{self.TRACKER_MOD}._get_db_imports",+ return_value={+ "thread_session_manager": mock_tsm,+ "get_user_db_session": mock_get_user_db,+ "RateLimitEstimate": MagicMock(),+ },+ ),+ patch(+ "local_deep_research.database.thread_metrics.metrics_writer",+ mock_writer,+ ),+ ):+ tracker._ensure_estimates_loaded()++ # no_autoflush was entered and exited; query ran inside it+ self.assertTrue(mock_session.no_autoflush.__enter__.called)+ self.assertTrue(mock_session.no_autoflush.__exit__.called)+ self.assertTrue(mock_session.query.called)+ finally:+ clear_search_context()++ # --- 3. _update_estimate tests ---++ def test_update_estimate_in_scope_reuses_session_with_savepoint(self):+ """_update_estimate inside active session scope reuses thread-local session with savepoint."""+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ tracker = self._build_tracker()+ tracker.recent_attempts["searxng"] = deque(+ [+ {+ "wait_time": 1.0,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ {+ "wait_time": 1.5,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ {+ "wait_time": 2.0,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ ]+ )+ mock_session = MagicMock()+ mock_session.query.return_value.filter_by.return_value.first.return_value = None+ mock_get_user_db = MagicMock()+ mock_get_user_db.return_value.__enter__.return_value = mock_session+ mock_get_user_db.return_value.__exit__.return_value = False+ mock_writer = MagicMock()+ mock_tsm = MagicMock()+ mock_tsm.in_scope.return_value = True++ set_search_context({"username": "testuser", "user_password": "secret"})+ try:+ with (+ patch(+ f"{self.TRACKER_MOD}.is_ci_environment",+ return_value=False,+ ),+ patch(+ f"{self.TRACKER_MOD}._get_db_imports",+ return_value={+ "thread_session_manager": mock_tsm,+ "get_user_db_session": mock_get_user_db,+ "RateLimitEstimate": MagicMock(),+ },+ ),+ patch(+ "local_deep_research.database.thread_metrics.metrics_writer",+ mock_writer,+ ),+ ):+ tracker._update_estimate("searxng")+ mock_get_user_db.assert_called_once_with("testuser", "secret")+ self.assertFalse(mock_writer.get_session.called)+ mock_session.begin_nested.assert_called_once()+ mock_session.add.assert_called_once()+ mock_session.commit.assert_not_called()+ finally:+ clear_search_context()++ def test_update_estimate_not_in_scope_uses_metrics_writer(self):+ """_update_estimate outside active session scope uses metrics_writer."""+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ tracker = self._build_tracker()+ tracker.recent_attempts["searxng"] = deque(+ [+ {+ "wait_time": 1.0,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ {+ "wait_time": 1.5,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ {+ "wait_time": 2.0,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ ]+ )+ mock_session = MagicMock()+ mock_session.query.return_value.filter_by.return_value.first.return_value = None+ mock_get_user_db = MagicMock()+ mock_writer = MagicMock()+ mock_writer.get_session.return_value.__enter__.return_value = (+ mock_session+ )+ mock_writer.get_session.return_value.__exit__.return_value = False+ mock_tsm = MagicMock()+ mock_tsm.in_scope.return_value = False++ set_search_context({"username": "testuser", "user_password": "secret"})+ try:+ with (+ patch(+ f"{self.TRACKER_MOD}.is_ci_environment",+ return_value=False,+ ),+ patch(+ f"{self.TRACKER_MOD}._get_db_imports",+ return_value={+ "thread_session_manager": mock_tsm,+ "get_user_db_session": mock_get_user_db,+ "RateLimitEstimate": MagicMock(),+ },+ ),+ patch(+ "local_deep_research.database.thread_metrics.metrics_writer",+ mock_writer,+ ),+ ):+ tracker._update_estimate("searxng")+ mock_writer.get_session.assert_called_once_with("testuser")+ self.assertFalse(mock_get_user_db.called)+ mock_session.begin_nested.assert_called_once()+ mock_session.add.assert_called_once()+ mock_session.commit.assert_not_called()+ finally:+ clear_search_context()++ # --- 4. get_stats tests ---++ def test_get_stats_in_scope_reuses_session(self):+ """get_stats inside active session scope reuses thread-local session."""+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ tracker = self._build_tracker()+ mock_session = MagicMock()+ mock_est = MagicMock()+ mock_est.engine_type = "google"+ mock_est.base_wait_seconds = 1.2+ mock_est.min_wait_seconds = 0.5+ mock_est.max_wait_seconds = 3.0+ mock_est.last_updated = 12345.0+ mock_est.total_attempts = 10+ mock_est.success_rate = 0.9+ mock_session.query.return_value.filter_by.return_value.all.return_value = [+ mock_est+ ]++ mock_get_user_db = MagicMock()+ mock_get_user_db.return_value.__enter__.return_value = mock_session+ mock_get_user_db.return_value.__exit__.return_value = False+ mock_writer = MagicMock()+ mock_tsm = MagicMock()+ mock_tsm.in_scope.return_value = True++ set_search_context({"username": "testuser", "user_password": "secret"})+ try:+ with (+ patch(+ f"{self.TRACKER_MOD}.is_ci_environment",+ return_value=False,+ ),+ patch(+ f"{self.TRACKER_MOD}._get_db_imports",+ return_value={+ "thread_session_manager": mock_tsm,+ "get_user_db_session": mock_get_user_db,+ "RateLimitEstimate": MagicMock(),+ },+ ),+ patch(+ "local_deep_research.database.thread_metrics.metrics_writer",+ mock_writer,+ ),+ ):+ stats = tracker.get_stats("google")+ self.assertEqual(len(stats), 1)+ self.assertEqual(stats[0][0], "google")+ self.assertEqual(stats[0][1], 1.2)+ mock_get_user_db.assert_called_once_with("testuser", "secret")+ self.assertFalse(mock_writer.get_session.called)+ finally:+ clear_search_context()++ def test_get_stats_in_scope_disables_autoflush(self):+ """get_stats in scope must run query inside session.no_autoflush.++ Cold ``get_stats`` callers share the production sessionmaker+ (``autoflush=True``). A bare ``query()`` could flush pending caller+ writes before the SELECT; under a competing SQLite writer that+ flush fails and rolls the outer transaction back. Pin the+ no-autoflush contract here.+ """+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ tracker = self._build_tracker()+ mock_session = MagicMock()+ mock_session.query.return_value.filter_by.return_value.all.return_value = []++ mock_get_user_db = MagicMock()+ mock_get_user_db.return_value.__enter__.return_value = mock_session+ mock_get_user_db.return_value.__exit__.return_value = False+ mock_writer = MagicMock()+ mock_tsm = MagicMock()+ mock_tsm.in_scope.return_value = True++ set_search_context({"username": "testuser", "user_password": "secret"})+ try:+ with (+ patch(+ f"{self.TRACKER_MOD}.is_ci_environment",+ return_value=False,+ ),+ patch(+ f"{self.TRACKER_MOD}._get_db_imports",+ return_value={+ "thread_session_manager": mock_tsm,+ "get_user_db_session": mock_get_user_db,+ "RateLimitEstimate": MagicMock(),+ },+ ),+ patch(+ "local_deep_research.database.thread_metrics.metrics_writer",+ mock_writer,+ ),+ ):+ tracker.get_stats("google")++ self.assertTrue(mock_session.no_autoflush.__enter__.called)+ self.assertTrue(mock_session.no_autoflush.__exit__.called)+ self.assertTrue(mock_session.query.called)+ finally:+ clear_search_context()++ def test_get_stats_not_in_scope_uses_metrics_writer(self):+ """get_stats outside active session scope uses metrics_writer."""+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ tracker = self._build_tracker()+ mock_session = MagicMock()+ mock_session.query.return_value.order_by.return_value.all.return_value = []+ mock_get_user_db = MagicMock()+ mock_writer = MagicMock()+ mock_writer.get_session.return_value.__enter__.return_value = (+ mock_session+ )+ mock_writer.get_session.return_value.__exit__.return_value = False+ mock_tsm = MagicMock()+ mock_tsm.in_scope.return_value = False++ set_search_context({"username": "testuser", "user_password": "secret"})+ try:+ with (+ patch(+ f"{self.TRACKER_MOD}.is_ci_environment",+ return_value=False,+ ),+ patch(+ f"{self.TRACKER_MOD}._get_db_imports",+ return_value={+ "thread_session_manager": mock_tsm,+ "get_user_db_session": mock_get_user_db,+ "RateLimitEstimate": MagicMock(),+ },+ ),+ patch(+ "local_deep_research.database.thread_metrics.metrics_writer",+ mock_writer,+ ),+ ):+ stats = tracker.get_stats()++ self.assertEqual(stats, [])+ mock_writer.get_session.assert_called_once_with("testuser")+ self.assertFalse(mock_get_user_db.called)+ finally:+ clear_search_context()++ # --- 5. reset_engine tests ---++ def test_reset_engine_in_scope_reuses_session_with_savepoint(self):+ """reset_engine inside active session scope reuses session with savepoint and does not commit."""+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ tracker = self._build_tracker()+ mock_session = MagicMock()+ mock_get_user_db = MagicMock()+ mock_get_user_db.return_value.__enter__.return_value = mock_session+ mock_get_user_db.return_value.__exit__.return_value = False+ mock_writer = MagicMock()+ mock_tsm = MagicMock()+ mock_tsm.in_scope.return_value = True++ set_search_context({"username": "testuser", "user_password": "secret"})+ try:+ with (+ patch(+ f"{self.TRACKER_MOD}.is_ci_environment",+ return_value=False,+ ),+ patch(+ f"{self.TRACKER_MOD}._get_db_imports",+ return_value={+ "thread_session_manager": mock_tsm,+ "get_user_db_session": mock_get_user_db,+ "RateLimitAttempt": MagicMock(),+ "RateLimitEstimate": MagicMock(),+ },+ ),+ patch(+ "local_deep_research.database.thread_metrics.metrics_writer",+ mock_writer,+ ),+ ):+ tracker.reset_engine("duckduckgo")+ mock_get_user_db.assert_called_once_with("testuser", "secret")+ self.assertFalse(mock_writer.get_session.called)+ mock_session.begin_nested.assert_called_once()+ # Explicit commit directly on shared session must NOT be called+ mock_session.commit.assert_not_called()+ finally:+ clear_search_context()++ def test_reset_engine_not_in_scope_uses_metrics_writer(self):+ """reset_engine outside active session scope uses metrics_writer."""+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ tracker = self._build_tracker()+ mock_session = MagicMock()+ mock_get_user_db = MagicMock()+ mock_writer = MagicMock()+ mock_writer.get_session.return_value.__enter__.return_value = (+ mock_session+ )+ mock_writer.get_session.return_value.__exit__.return_value = False+ mock_tsm = MagicMock()+ mock_tsm.in_scope.return_value = False++ set_search_context({"username": "testuser", "user_password": "secret"})+ try:+ with (+ patch(+ f"{self.TRACKER_MOD}.is_ci_environment",+ return_value=False,+ ),+ patch(+ f"{self.TRACKER_MOD}._get_db_imports",+ return_value={+ "thread_session_manager": mock_tsm,+ "get_user_db_session": mock_get_user_db,+ "RateLimitAttempt": MagicMock(),+ "RateLimitEstimate": MagicMock(),+ },+ ),+ patch(+ "local_deep_research.database.thread_metrics.metrics_writer",+ mock_writer,+ ),+ ):+ tracker.reset_engine("duckduckgo")+ mock_writer.get_session.assert_called_once_with("testuser")+ self.assertFalse(mock_get_user_db.called)+ mock_session.begin_nested.assert_called_once()+ mock_session.commit.assert_not_called()+ finally:+ clear_search_context()++ # --- 6. cleanup_old_data tests ---++ def test_cleanup_old_data_in_scope_reuses_session_with_savepoint(self):+ """cleanup_old_data inside active session scope reuses session with savepoint and does not commit."""+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ tracker = self._build_tracker()+ mock_session = MagicMock()+ mock_attempt = MagicMock()+ mock_attempt.timestamp.__lt__.return_value = MagicMock()+ mock_query = mock_session.query.return_value.filter.return_value+ mock_query.count.return_value = 5+ mock_get_user_db = MagicMock()+ mock_get_user_db.return_value.__enter__.return_value = mock_session+ mock_get_user_db.return_value.__exit__.return_value = False+ mock_writer = MagicMock()+ mock_tsm = MagicMock()+ mock_tsm.in_scope.return_value = True+ set_search_context({"username": "testuser", "user_password": "secret"})+ try:+ with (+ patch(+ f"{self.TRACKER_MOD}.is_ci_environment",+ return_value=False,+ ),+ patch(+ f"{self.TRACKER_MOD}._get_db_imports",+ return_value={+ "thread_session_manager": mock_tsm,+ "get_user_db_session": mock_get_user_db,+ "RateLimitAttempt": mock_attempt,+ },+ ),+ patch(+ "local_deep_research.database.thread_metrics.metrics_writer",+ mock_writer,+ ),+ ):+ tracker.cleanup_old_data(days=14)+ mock_get_user_db.assert_called_once_with("testuser", "secret")+ self.assertFalse(mock_writer.get_session.called)+ mock_session.begin_nested.assert_called_once()+ mock_query.delete.assert_called_once()+ mock_session.commit.assert_not_called()+ finally:+ clear_search_context()++ def test_cleanup_old_data_not_in_scope_uses_metrics_writer(self):+ """cleanup_old_data outside active session scope uses metrics_writer."""+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ tracker = self._build_tracker()+ mock_session = MagicMock()+ mock_attempt = MagicMock()+ mock_attempt.timestamp.__lt__.return_value = MagicMock()+ mock_query = mock_session.query.return_value.filter.return_value+ mock_query.count.return_value = 0+ mock_get_user_db = MagicMock()+ mock_writer = MagicMock()+ mock_writer.get_session.return_value.__enter__.return_value = (+ mock_session+ )+ mock_writer.get_session.return_value.__exit__.return_value = False+ mock_tsm = MagicMock()+ mock_tsm.in_scope.return_value = False+ set_search_context({"username": "testuser", "user_password": "secret"})+ try:+ with (+ patch(+ f"{self.TRACKER_MOD}.is_ci_environment",+ return_value=False,+ ),+ patch(+ f"{self.TRACKER_MOD}._get_db_imports",+ return_value={+ "thread_session_manager": mock_tsm,+ "get_user_db_session": mock_get_user_db,+ "RateLimitAttempt": mock_attempt,+ },+ ),+ patch(+ "local_deep_research.database.thread_metrics.metrics_writer",+ mock_writer,+ ),+ ):+ tracker.cleanup_old_data(days=30)+ mock_writer.get_session.assert_called_once_with("testuser")+ self.assertFalse(mock_get_user_db.called)+ mock_session.begin_nested.assert_called_once()+ mock_session.commit.assert_not_called()+ finally:+ clear_search_context()++ # --- 7. Robustness against errors across all 5 sites ---++ def test_resolve_session_error_handled_in_all_sites(self):+ """Errors in _resolve_session are caught and handled gracefully across all callsites."""+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ tracker = self._build_tracker()+ tracker._estimates_loaded = False++ set_search_context({"username": "testuser", "user_password": "secret"})+ try:+ with (+ patch(+ f"{self.TRACKER_MOD}.is_ci_environment",+ return_value=False,+ ),+ patch.object(+ tracker,+ "_resolve_session",+ side_effect=RuntimeError("DB exploded"),+ ),+ ):+ # 1. _ensure_estimates_loaded: sets _estimates_loaded=True, doesn't raise+ tracker._ensure_estimates_loaded()+ self.assertTrue(tracker._estimates_loaded)++ # 2. get_stats: falls back to in-memory stats, doesn't raise+ stats = tracker.get_stats()+ self.assertEqual(stats, [])++ # 3. _update_estimate: logs warning, doesn't raise+ tracker.recent_attempts["searxng"] = deque(+ [+ {+ "wait_time": 1.0,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ {+ "wait_time": 1.5,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ {+ "wait_time": 2.0,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ ]+ )+ tracker._update_estimate("searxng")++ # 4. reset_engine: memory cleared, doesn't raise+ tracker.reset_engine("searxng")++ # 5. cleanup_old_data: logs warning, doesn't raise+ tracker.cleanup_old_data(days=30)+ finally:+ clear_search_context()++ def test_get_wait_time_cold_cache_inside_in_scope_write_session(self):+ """get_wait_time triggering _ensure_estimates_loaded on cold cache within active write session."""+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ tracker = self._build_tracker()+ tracker._estimates_loaded = False++ mock_session = MagicMock()+ mock_estimate = MagicMock()+ mock_estimate.engine_type = "searxng"+ mock_estimate.base_wait_seconds = 3.5+ mock_estimate.min_wait_seconds = 1.0+ mock_estimate.max_wait_seconds = 10.0+ mock_estimate.last_updated = time.time()+ mock_session.query.return_value.all.return_value = [mock_estimate]++ mock_get_user_db = MagicMock()+ mock_get_user_db.return_value.__enter__.return_value = mock_session+ mock_get_user_db.return_value.__exit__.return_value = False+ mock_writer = MagicMock()+ mock_tsm = MagicMock()+ mock_tsm.in_scope.return_value = True++ set_search_context({"username": "testuser", "user_password": "secret"})+ try:+ with (+ patch(+ f"{self.TRACKER_MOD}.is_ci_environment",+ return_value=False,+ ),+ patch(+ f"{self.TRACKER_MOD}._get_db_imports",+ return_value={+ "thread_session_manager": mock_tsm,+ "get_user_db_session": mock_get_user_db,+ "RateLimitEstimate": MagicMock(),+ },+ ),+ patch(+ "local_deep_research.database.thread_metrics.metrics_writer",+ mock_writer,+ ),+ ):+ wait_time = tracker.get_wait_time("searxng")++ self.assertTrue(tracker._estimates_loaded)+ self.assertGreaterEqual(wait_time, 1.0)+ self.assertLessEqual(wait_time, 10.0)+ mock_get_user_db.assert_called_once_with("testuser", "secret")+ self.assertFalse(mock_writer.get_session.called)+ self.assertFalse(mock_session.commit.called)+ finally:+ clear_search_context()++ def test_ensure_estimates_loaded_applies_decay(self):+ """Estimates older than 24 hours decay confidence according to decay_per_day."""+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ tracker = self._build_tracker()+ tracker._estimates_loaded = False+ tracker.decay_per_day = 0.5++ now = time.time()+ mock_session = MagicMock()+ mock_estimate = MagicMock()+ mock_estimate.engine_type = "test_engine"+ mock_estimate.base_wait_seconds = 4.0+ mock_estimate.min_wait_seconds = 1.0+ mock_estimate.max_wait_seconds = 8.0+ mock_estimate.last_updated = now - 86400+ mock_session.query.return_value.all.return_value = [mock_estimate]++ mock_get_user_db = MagicMock()+ mock_get_user_db.return_value.__enter__.return_value = mock_session+ mock_get_user_db.return_value.__exit__.return_value = False+ mock_tsm = MagicMock()+ mock_tsm.in_scope.return_value = True++ set_search_context({"username": "testuser", "user_password": "secret"})+ try:+ with (+ patch(+ f"{self.TRACKER_MOD}.is_ci_environment",+ return_value=False,+ ),+ patch(+ f"{self.TRACKER_MOD}._get_db_imports",+ return_value={+ "thread_session_manager": mock_tsm,+ "get_user_db_session": mock_get_user_db,+ "RateLimitEstimate": MagicMock(),+ },+ ),+ ):+ tracker._ensure_estimates_loaded()++ self.assertIn("test_engine", tracker.current_estimates)+ self.assertAlmostEqual(+ tracker.current_estimates["test_engine"]["confidence"],+ 0.5,+ places=2,+ )+ finally:+ clear_search_context()++ # --- 8. Real SQLite transactional integrity tests ---++ def test_real_sqlite_in_scope_preserves_caller_rollback_atomicity(self):+ """Caller's uncommitted work must NOT be prematurely committed by tracker writes,+ allowing the caller to cleanly roll back all changes.+ """+ from sqlalchemy import create_engine+ from sqlalchemy.orm import sessionmaker+ from local_deep_research.database.models import (+ Base,+ RateLimitAttempt,+ RateLimitEstimate,+ )+ from local_deep_research.database.session_context import (+ get_user_db_session,+ )+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ engine = create_engine("sqlite:///:memory:")+ Base.metadata.create_all(engine)+ TestSession = sessionmaker(bind=engine)+ real_session = TestSession()++ tracker = self._build_tracker()+ set_search_context(+ {"username": "realuser", "user_password": "realpassword"}+ )++ try:+ with (+ patch(+ f"{self.TRACKER_MOD}.is_ci_environment",+ return_value=False,+ ),+ patch(+ "local_deep_research.database.thread_local_session.get_metrics_session",+ return_value=real_session,+ ),+ ):+ with get_user_db_session(+ "realuser", "realpassword"+ ) as outer_session:+ caller_attempt = RateLimitAttempt(+ engine_type="caller_staged_engine",+ wait_time=1.23,+ success=True,+ retry_count=1,+ timestamp=time.time(),+ )+ outer_session.add(caller_attempt)++ # Populate attempts so _update_estimate persists to DB+ tracker.recent_attempts["searxng"] = deque(+ [+ {+ "wait_time": 1.0,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ {+ "wait_time": 1.5,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ {+ "wait_time": 2.0,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ ]+ )+ # Trigger writes across tracker write methods+ tracker._update_estimate("searxng")+ self.assertIsNotNone(+ outer_session.query(RateLimitEstimate)+ .filter_by(engine_type="searxng")+ .first()+ )+ tracker.reset_engine("other_engine")+ tracker.cleanup_old_data(days=30)++ # Caller rolls back their transaction+ outer_session.rollback()++ # Caller row must NOT be committed to SQLite+ persisted_caller = (+ real_session.query(RateLimitAttempt)+ .filter_by(engine_type="caller_staged_engine")+ .first()+ )+ self.assertIsNone(persisted_caller)+ finally:+ clear_search_context()+ real_session.close()+ engine.dispose()++ def test_real_sqlite_tracker_write_failure_does_not_destroy_caller_writes(+ self,+ ):+ """Simulated tracker write failure inside savepoint must not abort or destroy+ caller's uncommitted writes.+ """+ from sqlalchemy import create_engine+ from sqlalchemy.orm import sessionmaker+ from local_deep_research.database.models import (+ Base,+ RateLimitAttempt,+ )+ from local_deep_research.database.session_context import (+ get_user_db_session,+ )+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ engine = create_engine("sqlite:///:memory:")+ Base.metadata.create_all(engine)+ TestSession = sessionmaker(bind=engine)+ real_session = TestSession()++ tracker = self._build_tracker()+ set_search_context(+ {"username": "realuser", "user_password": "realpassword"}+ )++ try:+ with (+ patch(+ f"{self.TRACKER_MOD}.is_ci_environment",+ return_value=False,+ ),+ patch(+ "local_deep_research.database.thread_local_session.get_metrics_session",+ return_value=real_session,+ ),+ ):+ with get_user_db_session(+ "realuser", "realpassword"+ ) as outer_session:+ caller_attempt = RateLimitAttempt(+ engine_type="caller_surviving_engine",+ wait_time=2.34,+ success=True,+ retry_count=1,+ timestamp=time.time(),+ )+ outer_session.add(caller_attempt)++ # Tracker write failure inside savepoint+ orig_add = real_session.add++ def failing_add(instance):+ if (+ hasattr(instance, "engine_type")+ and instance.engine_type == "fail_engine"+ ):+ raise RuntimeError("Simulated inner write failure")+ return orig_add(instance)++ with patch.object(+ real_session, "add", side_effect=failing_add+ ):+ tracker.recent_attempts["fail_engine"] = deque(+ [+ {+ "wait_time": 1.0,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ {+ "wait_time": 1.5,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ {+ "wait_time": 2.0,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ ]+ )+ tracker._update_estimate("fail_engine")++ # Caller cleanly commits their outer transaction+ outer_session.commit()++ persisted_caller = (+ real_session.query(RateLimitAttempt)+ .filter_by(engine_type="caller_surviving_engine")+ .first()+ )+ self.assertIsNotNone(persisted_caller)+ self.assertEqual(persisted_caller.wait_time, 2.34)+ finally:+ clear_search_context()+ real_session.close()+ engine.dispose()++ def test_real_sqlite_tracker_read_failure_does_not_destroy_caller_writes(+ self,+ ):+ """Simulated tracker read failure in _ensure_estimates_loaded must not rollback+ caller's uncommitted outer transaction.+ """+ from sqlalchemy import create_engine+ from sqlalchemy.orm import sessionmaker+ from local_deep_research.database.models import (+ Base,+ RateLimitAttempt,+ RateLimitEstimate,+ )+ from local_deep_research.database.session_context import (+ get_user_db_session,+ )+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ engine = create_engine("sqlite:///:memory:")+ Base.metadata.create_all(engine)+ TestSession = sessionmaker(bind=engine)+ real_session = TestSession()++ tracker = self._build_tracker()+ tracker._estimates_loaded = False+ set_search_context(+ {"username": "realuser", "user_password": "realpassword"}+ )++ try:+ with (+ patch(+ f"{self.TRACKER_MOD}.is_ci_environment",+ return_value=False,+ ),+ patch(+ "local_deep_research.database.thread_local_session.get_metrics_session",+ return_value=real_session,+ ),+ ):+ with get_user_db_session(+ "realuser", "realpassword"+ ) as outer_session:+ caller_attempt = RateLimitAttempt(+ engine_type="caller_read_survivor",+ wait_time=3.45,+ success=True,+ retry_count=1,+ timestamp=time.time(),+ )+ outer_session.add(caller_attempt)++ orig_query = real_session.query++ def failing_query(*args, **kwargs):+ if args and args[0] is RateLimitEstimate:+ raise RuntimeError("Simulated transient read error")+ return orig_query(*args, **kwargs)++ with patch.object(+ real_session, "query", side_effect=failing_query+ ):+ tracker._ensure_estimates_loaded()++ # Caller commits outer transaction cleanly+ outer_session.commit()++ persisted_caller = (+ real_session.query(RateLimitAttempt)+ .filter_by(engine_type="caller_read_survivor")+ .first()+ )+ self.assertIsNotNone(persisted_caller)+ self.assertEqual(persisted_caller.wait_time, 3.45)+ finally:+ clear_search_context()+ real_session.close()+ engine.dispose()++ def test_real_sqlite_cold_read_under_writer_contention_preserves_caller(+ self,+ ):+ """Cold-cache read in scope must not autoflush caller writes through a competing SQLite writer.++ The production sessionmaker uses ``autoflush=True``. A bare+ ``query()`` in ``_ensure_estimates_loaded`` or ``get_stats`` would+ otherwise flush a pending caller INSERT before the SELECT; under a+ held ``BEGIN IMMEDIATE`` that flush fails and SQLAlchemy invalidates+ the outer transaction, leaving ``is_active=False`` and the staged+ row gone. The ``no_autoflush`` guard on both read sites prevents+ the autoflush and lets the caller commit normally.+ """+ import sqlite3+ import tempfile++ from sqlalchemy import create_engine+ from sqlalchemy.orm import sessionmaker+ from local_deep_research.database.models import (+ Base,+ RateLimitAttempt,+ RateLimitEstimate,+ )+ from local_deep_research.database.session_context import (+ get_user_db_session,+ )+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ fd, db_path = tempfile.mkstemp(suffix=".db")+ os.close(fd)+ db_url = "sqlite:///" + db_path.replace("\\", "/")+ engine = create_engine(+ db_url, connect_args={"timeout": 0.1, "check_same_thread": False}+ )+ Base.metadata.create_all(engine)+ TestSession = sessionmaker(bind=engine)+ real_session = TestSession()+ lock_conn = None++ tracker = self._build_tracker()+ # Cold cache so ``get_wait_time`` must hit the DB on the cold path.+ tracker._estimates_loaded = False+ set_search_context(+ {"username": "realuser", "user_password": "realpassword"}+ )++ try:+ with (+ patch(+ f"{self.TRACKER_MOD}.is_ci_environment",+ return_value=False,+ ),+ patch(+ "local_deep_research.database.thread_local_session.get_metrics_session",+ return_value=real_session,+ ),+ ):+ with get_user_db_session(+ "realuser", "realpassword"+ ) as outer_session:+ caller_attempt = RateLimitAttempt(+ engine_type="caller_read_contention",+ wait_time=6.78,+ success=True,+ retry_count=1,+ timestamp=time.time(),+ )+ outer_session.add(caller_attempt)++ # Independent connection holds the writer lock so any+ # autoflush of the caller's pending INSERT fails.+ lock_conn = sqlite3.connect(db_path, timeout=0.1)+ lock_conn.execute("BEGIN IMMEDIATE")+ try:+ # Pre-fix: this read autoflushed the staged+ # INSERT through the held lock and rolled back the+ # outer transaction.+ tracker._ensure_estimates_loaded()+ tracker.get_stats()+ self.assertTrue(outer_session.is_active)+ self.assertIn(caller_attempt, outer_session.new)+ finally:+ lock_conn.rollback()++ outer_session.commit()++ persisted_caller = (+ real_session.query(RateLimitAttempt)+ .filter_by(engine_type="caller_read_contention")+ .first()+ )+ self.assertIsNotNone(persisted_caller)+ self.assertEqual(persisted_caller.wait_time, 6.78)+ # The estimator read did not flip ``is_active`` either.+ persisted_estimates = real_session.query(RateLimitEstimate).all()+ self.assertIsInstance(persisted_estimates, list)+ finally:+ clear_search_context()+ real_session.close()+ engine.dispose()+ if lock_conn is not None:+ lock_conn.close()+ try:+ os.unlink(db_path)+ except OSError:+ pass++ def test_real_sqlite_savepoint_entry_contention_preserves_caller_transaction(+ self,+ ):+ """Competing writer during savepoint entry must not destroy caller work.++ Session.begin_nested() flushes pending ORM changes before SAVEPOINT.+ A lock held by another connection makes that flush fail outside the+ savepoint and invalidates the caller (PendingRollbackError). Staging a+ row then invoking tracker writes under contention covers the live+ _download_pdf shape; injecting a failure inside session.add() after+ entry does not.+ """+ import sqlite3+ import tempfile++ from sqlalchemy import create_engine+ from sqlalchemy.orm import sessionmaker+ from local_deep_research.database.models import (+ Base,+ RateLimitAttempt,+ )+ from local_deep_research.database.session_context import (+ get_user_db_session,+ )+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ fd, db_path = tempfile.mkstemp(suffix=".db")+ os.close(fd)+ db_url = "sqlite:///" + db_path.replace("\\", "/")+ engine = create_engine(+ db_url, connect_args={"timeout": 0.1, "check_same_thread": False}+ )+ Base.metadata.create_all(engine)+ TestSession = sessionmaker(bind=engine)+ real_session = TestSession()+ lock_conn = None+ nested_calls = []++ tracker = self._build_tracker()+ set_search_context(+ {"username": "realuser", "user_password": "realpassword"}+ )++ orig_begin_nested = real_session.begin_nested++ def counting_begin_nested(*args, **kwargs):+ nested_calls.append(1)+ return orig_begin_nested(*args, **kwargs)++ try:+ with (+ patch(+ f"{self.TRACKER_MOD}.is_ci_environment",+ return_value=False,+ ),+ patch(+ "local_deep_research.database.thread_local_session.get_metrics_session",+ return_value=real_session,+ ),+ ):+ with get_user_db_session(+ "realuser", "realpassword"+ ) as outer_session:+ caller_attempt = RateLimitAttempt(+ engine_type="caller_contention_engine",+ wait_time=4.56,+ success=True,+ retry_count=1,+ timestamp=time.time(),+ )+ outer_session.add(caller_attempt)++ tracker.recent_attempts["searxng"] = deque(+ [+ {+ "wait_time": 1.0,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ {+ "wait_time": 1.5,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ {+ "wait_time": 2.0,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ ]+ )++ real_session.begin_nested = counting_begin_nested+ lock_conn = sqlite3.connect(db_path, timeout=0.1)+ lock_conn.execute("BEGIN IMMEDIATE")+ try:+ tracker._update_estimate("searxng")+ tracker.reset_engine("other_engine")+ tracker.cleanup_old_data(days=30)+ self.assertTrue(outer_session.is_active)+ finally:+ lock_conn.rollback()++ self.assertGreaterEqual(len(nested_calls), 3)+ outer_session.commit()++ persisted_caller = (+ real_session.query(RateLimitAttempt)+ .filter_by(engine_type="caller_contention_engine")+ .first()+ )+ self.assertIsNotNone(persisted_caller)+ self.assertEqual(persisted_caller.wait_time, 4.56)+ finally:+ clear_search_context()+ real_session.begin_nested = orig_begin_nested+ real_session.close()+ engine.dispose()+ if lock_conn is not None:+ lock_conn.close()+ try:+ os.unlink(db_path)+ except OSError:+ pass++ def test_real_sqlite_cascade_pending_parent_child_preserves_caller_transaction(+ self,+ ):+ """Caller-staged parent/child with cascade delete-orphan must survive tracker writes.++ ``Session.expunge(parent)`` cascades to children under+ ``cascade="all, delete-orphan"`` (e.g. ``ResearchHistory.resources``).+ Re-expunging the already-detached child raises ``InvalidRequestError``+ and used to leak caller objects out of the session, so a later+ ``commit()`` would silently drop them. The cascade-safe expunge+ must skip the already-detached child and ``add(parent)`` must+ re-attach both via cascade, so the caller can commit normally.+ """+ import tempfile+ import uuid++ from sqlalchemy import create_engine+ from sqlalchemy.orm import sessionmaker+ from local_deep_research.database.models import (+ Base,+ ResearchHistory,+ ResearchResource,+ )+ from local_deep_research.database.session_context import (+ get_user_db_session,+ )+ from local_deep_research.utilities.thread_context import (+ clear_search_context,+ set_search_context,+ )++ fd, db_path = tempfile.mkstemp(suffix=".db")+ os.close(fd)+ db_url = "sqlite:///" + db_path.replace("\\", "/")+ engine = create_engine(+ db_url, connect_args={"check_same_thread": False}+ )+ Base.metadata.create_all(engine)+ TestSession = sessionmaker(bind=engine)+ real_session = TestSession()++ tracker = self._build_tracker()+ set_search_context(+ {"username": "realuser", "user_password": "realpassword"}+ )++ parent_id = str(uuid.uuid4())+ try:+ with (+ patch(+ f"{self.TRACKER_MOD}.is_ci_environment",+ return_value=False,+ ),+ patch(+ "local_deep_research.database.thread_local_session.get_metrics_session",+ return_value=real_session,+ ),+ ):+ with get_user_db_session(+ "realuser", "realpassword"+ ) as outer_session:+ parent = ResearchHistory(+ id=parent_id,+ query="cascade_test_query",+ status="in_progress",+ mode="quick",+ created_at="2026-09-20T00:00:00+00:00",+ )+ child = ResearchResource(+ research=parent,+ title="cascade_child_title",+ url="https://example.invalid/cascade",+ created_at="2026-09-20T00:00:00+00:00",+ )+ # Pin the parent-first ordering of ``session.new``: add+ # the parent first, then attach the child via cascade+ # (``parent.resources.append`` triggers cascade-add of+ # the child once the parent is in the session). This+ # guarantees the parent sits before the child in+ # ``session.new``, so the regression cannot silently+ # pass on the pre-fix tree when the child happens to+ # come first by id order.+ outer_session.add(parent)+ parent.resources.append(child)++ self.assertTrue(parent in outer_session)+ self.assertTrue(child in outer_session)+ new_list = list(outer_session.new)+ self.assertGreater(+ new_list.index(parent),+ -1,+ "parent must be in session.new",+ )+ self.assertGreater(+ new_list.index(child),+ -1,+ "child must be in session.new",+ )+ self.assertLess(+ new_list.index(parent),+ new_list.index(child),+ "parent must be inserted before child in session.new",+ )++ # Trigger every write site so all three exercise the+ # cascade-safe expunge path with the parent/child pair+ # still pending in the outer session.+ tracker.recent_attempts["searxng"] = deque(+ [+ {+ "wait_time": 1.0,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ {+ "wait_time": 1.5,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ {+ "wait_time": 2.0,+ "success": True,+ "timestamp": time.time(),+ "retry_count": 1,+ "search_result_count": 5,+ },+ ]+ )+ tracker._update_estimate("searxng")+ tracker.reset_engine("other_engine")+ tracker.cleanup_old_data(days=30)++ # Caller's transaction is still active and parent ++ # child remain pending in the session.+ self.assertTrue(outer_session.is_active)+ self.assertIn(parent, outer_session)+ self.assertIn(child, outer_session)+ self.assertIn(parent, outer_session.new)+ self.assertIn(child, outer_session.new)++ outer_session.commit()+ self.assertTrue(outer_session.is_active)++ persisted_parent = (+ real_session.query(ResearchHistory)+ .filter_by(id=parent_id)+ .one_or_none()+ )+ self.assertIsNotNone(persisted_parent)+ self.assertEqual(persisted_parent.query, "cascade_test_query")+ persisted_children = (+ real_session.query(ResearchResource)+ .filter_by(research_id=parent_id)+ .all()+ )+ self.assertEqual(len(persisted_children), 1)+ self.assertEqual(persisted_children[0].title, "cascade_child_title")+ finally:+ clear_search_context()+ real_session.close()+ engine.dispose()+ try:+ os.unlink(db_path)+ except OSError:+ pass++if __name__ == "__main__":unittest.main()diff --git a/tests/web_search_engines/rate_limiting/test_tracker_coverage.py b/tests/web_search_engines/rate_limiting/test_tracker_coverage.pyindex 60f2102329..a1baef7a87 100644--- a/tests/web_search_engines/rate_limiting/test_tracker_coverage.py+++ b/tests/web_search_engines/rate_limiting/test_tracker_coverage.py@@ -132,6 +132,42 @@ class TestEnsureEstimatesLoadedDB:assert tracker.current_estimates["TE"]["base"] == 2.0assert abs(tracker.current_estimates["TE"]["confidence"] - 0.95) < 0.01+ def test_query_runs_inside_no_autoflush(self):+ """Loads estimates inside ``session.no_autoflush`` to keep the+ read from flushing the caller's pending ORM writes.++ The production sessionmaker defaults to ``autoflush=True`` and a+ bare ``query()`` would otherwise flush the caller's pending+ INSERTs before the SELECT; under a competing SQLite writer that+ flush fails and rolls back the outer transaction.+ ``session.no_autoflush`` keeps the read atomic.+ """+ tracker = _make_tracker()+ tracker._estimates_loaded = False+ ms = MagicMock()+ ms.query.return_value.all.return_value = []+ mw = MagicMock()+ mw.get_session = _fake_ctx_mgr(ms)+ imports = {+ "RateLimitEstimate": MagicMock(),+ "get_user_db_session": MagicMock(),+ }+ ctx = {"username": "alice", "user_password": "secret"}+ with patch(f"{MODULE}._get_db_imports", return_value=imports):+ with patch(f"{MODULE}.get_search_context", return_value=ctx):+ with patch.dict(+ "sys.modules",+ {+ "local_deep_research.database.thread_metrics": MagicMock(+ metrics_writer=mw+ )+ },+ ):+ tracker._ensure_estimates_loaded()+ assert ms.no_autoflush.__enter__.called+ assert ms.no_autoflush.__exit__.called+ assert ms.query.called+def test_db_exception_marks_loaded(self):"""Database exception during load still marks loaded."""tracker = _make_tracker()@@ -341,6 +377,32 @@ class TestGetStatsDB:assert len(result) == 1assert result[0][0] == "G"+ @patch(f"{MODULE}.is_ci_environment", return_value=False)+ @patch(+ f"{MODULE}.get_search_context",+ return_value={"username": "u", "user_password": "p"},+ )+ def test_query_runs_inside_no_autoflush(self, _ctx, _ci):+ """get_stats runs query inside ``session.no_autoflush`` so a+ cold read cannot flush the caller's pending ORM writes through+ a competing SQLite writer.+ """+ tracker = _make_tracker(programmatic_mode=False)+ ms = MagicMock()+ ms.query.return_value.order_by.return_value.all.return_value = []+ mock_writer = MagicMock()+ mock_writer.get_session = _fake_ctx_mgr(ms)+ imports = {"RateLimitEstimate": MagicMock()}+ with patch(f"{MODULE}._get_db_imports", return_value=imports):+ with patch(+ "local_deep_research.database.thread_metrics.metrics_writer",+ mock_writer,+ ):+ tracker.get_stats()+ assert ms.no_autoflush.__enter__.called+ assert ms.no_autoflush.__exit__.called+ assert ms.query.called+@patch(f"{MODULE}.is_ci_environment", return_value=False)@patch(f"{MODULE}.get_search_context",@@ -393,7 +455,7 @@ class TestResetEngineDB:return_value={"username": "u", "user_password": "p"},)def test_deletes_from_db(self, _ctx, _ci):- """Deletes from DB and commits."""+ """Deletes from DB within savepoint without committing shared transaction."""tracker = _make_tracker(programmatic_mode=False)tracker.recent_attempts["Eng"] = deque([_attempt(1.0, True)], maxlen=100@@ -419,7 +481,8 @@ class TestResetEngineDB:tracker.reset_engine("Eng")assert "Eng" not in tracker.recent_attemptsassert "Eng" not in tracker.current_estimates- ms.commit.assert_called_once()+ ms.begin_nested.assert_called_once()+ ms.commit.assert_not_called()@patch(f"{MODULE}.is_ci_environment", return_value=False)@patch(@@ -470,7 +533,7 @@ class TestCleanupOldDataDB:return_value={"username": "u", "user_password": "p"},)def test_full_cleanup_deletes_old_rows(self, _ctx, _ci):- """Deletes old attempts from DB and commits."""+ """Deletes old attempts from DB within savepoint without committing shared transaction."""tracker = _make_tracker(programmatic_mode=False)mq = MagicMock()mq.count.return_value = 42@@ -486,7 +549,8 @@ class TestCleanupOldDataDB:):tracker.cleanup_old_data(days=30)mq.delete.assert_called_once()- ms.commit.assert_called_once()+ ms.begin_nested.assert_called_once()+ ms.commit.assert_not_called()@patch(f"{MODULE}.is_ci_environment", return_value=False)@patch(@@ -494,7 +558,7 @@ class TestCleanupOldDataDB:return_value={"username": "u", "user_password": "p"},)def test_no_old_data_still_commits(self, _ctx, _ci):- """When no old data exists, still commits."""+ """When no old data exists, still uses savepoint discipline."""tracker = _make_tracker(programmatic_mode=False)mq = MagicMock()mq.count.return_value = 0@@ -509,7 +573,8 @@ class TestCleanupOldDataDB:mock_writer,):tracker.cleanup_old_data(days=7)- ms.commit.assert_called_once()+ ms.begin_nested.assert_called_once()+ ms.commit.assert_not_called()@patch(f"{MODULE}.is_ci_environment", return_value=False)@patch(