~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(benchmarks): push request-user context in the benchmark thread (#7363)
LearningCircuit8ab28bcba864
diff --git a/changelog.d/7362.bugfix.md b/changelog.d/7362.bugfix.mdnew file mode 100644index 0000000000..fa1a3ceb47--- /dev/null+++ b/changelog.d/7362.bugfix.md@@ -0,0 +1 @@+Fix the web benchmark failing every task since the FastAPI migration: the background thread now pushes the run's owner via `request_user(...)` (nothing is pushed for a missing owner, matching scheduler jobs) so `quick_summary`'s `@no_db_settings` resolves the user's settings instead of tripping the background-thread database guard. The benchmark thread also closes its cached per-thread DB sessions after each task instead of accumulating them for the whole run.diff --git a/src/local_deep_research/benchmarks/web_api/benchmark_service.py b/src/local_deep_research/benchmarks/web_api/benchmark_service.pyindex 6b4bd14246..d84e3ec04f 100644--- a/src/local_deep_research/benchmarks/web_api/benchmark_service.py+++ b/src/local_deep_research/benchmarks/web_api/benchmark_service.py@@ -4,6 +4,7 @@ import hashlibimport jsonimport threadingimport time+from contextlib import nullcontextfrom datetime import UTC, datetimefrom enum import Enumfrom typing import Any, Dict, List, Optional, Tuple@@ -26,6 +27,7 @@ from ..datasets import load_datasetfrom ..graders import extract_answer_from_response, grade_single_resultfrom ..runners import format_queryfrom ...database.thread_local_session import thread_cleanup+from ...utilities.request_context import request_user# Generic failure message surfaced to API clients in place of the raw benchmark# exception. ``benchmark_run.error_message`` is set from ``str(e)`` of the@@ -472,6 +474,22 @@ class BenchmarkService:self, username: Optional[str], benchmark_run_id: int):"""Main benchmark execution thread."""+ # A plain threading.Thread does not inherit the request context+ # (contextvars), so quick_summary's @no_db_settings wrapper — which+ # resolves get_settings_manager() before the wrapped function runs —+ # finds no user here and trips the background-thread DB guard:+ # every task failed with "Research error" and 0% accuracy. Push the+ # owner for the whole run, the same way scheduler jobs do+ # (scheduler/background.py). A missing owner pushes nothing (as in+ # scheduler/background.py) rather than a "benchmark_user" fallback,+ # which could resolve an unrelated account of that name.+ user_ctx = request_user(username) if username else nullcontext()+ with user_ctx:+ self._run_benchmark_thread_impl(username, benchmark_run_id)++ def _run_benchmark_thread_impl(+ self, username: Optional[str], benchmark_run_id: int+ ):# IMPORTANT: This runs in a background thread, so we cannot access the user database# Using in-memory queue tracker for benchmark status tracking@@ -626,6 +644,23 @@ class BenchmarkService:"Failed to clear benchmark task flag ""(non-critical)")+ # Close the DB sessions this task opened through+ # db_utils' per-thread cache (quick_summary's ambient+ # settings manager). The run's thread lives for every+ # task, so without this they would accumulate until+ # @thread_cleanup at thread exit; the next task+ # re-opens one lazily on a cache miss.+ try:+ from ...utilities.db_utils import (+ cleanup_cached_user_sessions_current_thread,+ )++ cleanup_cached_user_sessions_current_thread()+ except Exception:+ logger.debug(+ "Failed to clean up benchmark task DB "+ "sessions (non-critical)"+ )# Store result in memory for now (will be saved later)if "results" not in self.active_runs[run_key]:diff --git a/tests/benchmarks/test_benchmark_service_thread_context.py b/tests/benchmarks/test_benchmark_service_thread_context.pynew file mode 100644index 0000000000..08cb87926c--- /dev/null+++ b/tests/benchmarks/test_benchmark_service_thread_context.py@@ -0,0 +1,165 @@+"""Tests for the benchmark thread's request-user context.++The benchmark runs in a plain ``threading.Thread``, which does NOT inherit+the request context (contextvars). ``quick_summary`` is decorated with+``@no_db_settings``, whose wrapper eagerly resolves ``get_settings_manager()``+before the wrapped function runs — with no username and no request context,+the background-thread DB guard raises RuntimeError and every benchmark task+fails ("Research error" in the logs, 0% accuracy).++The thread must push its owner via ``request_user(...)`` for the duration of+the run, the same way scheduler jobs do (scheduler/background.py).+"""++import threading+from unittest.mock import Mock+++def _run_thread_and_capture(service, username="testuser", run_id=1):+ """Run _run_benchmark_thread in a real background thread.++ ``_process_benchmark_task`` is stubbed to reproduce what+ ``@no_db_settings`` does inside quick_summary: resolve the settings+ manager from the calling thread's context. Returns what the stub saw.+ """+ captured = {}++ def fake_process(task, search_config, evaluation_config):+ from local_deep_research.utilities.db_utils import get_settings_manager+ from local_deep_research.utilities.request_context import (+ get_current_username,+ )++ captured["context_username"] = get_current_username()+ # This is the exact call that raised RuntimeError from the+ # background thread before the fix.+ captured["manager"] = get_settings_manager()+ return {"example_id": task["example_id"], "success": True}++ service._create_task_queue = Mock(+ return_value=[{"example_id": "e0", "dataset": "simpleqa"}]+ )+ service._process_benchmark_task = fake_process+ service._sync_results_to_database = Mock(return_value=None)+ service.queue_tracker = Mock()++ thread = threading.Thread(+ target=service._run_benchmark_thread, args=(username, run_id)+ )+ thread.start()+ thread.join(timeout=30)++ assert not thread.is_alive(), "benchmark thread did not finish"+ return captured+++def _make_service_with_run(username="testuser", run_id=1):+ from local_deep_research.benchmarks.web_api.benchmark_service import (+ BenchmarkService,+ )++ service = BenchmarkService(socket_service=Mock())+ service.active_runs[(username, run_id)] = {+ "data": {+ "benchmark_run_id": run_id,+ "username": username,+ "user_password": None,+ "config_hash": "abc12345",+ "datasets_config": {"simpleqa": {"count": 1}},+ "search_config": {},+ "evaluation_config": {},+ "settings_snapshot": {},+ },+ "status": "running",+ "results": [],+ }+ return service+++def test_benchmark_thread_resolves_settings_manager_from_owner_context():+ """quick_summary's @no_db_settings must see the run's owner.++ Reproduces the web-benchmark failure where every task died with+ ``RuntimeError: Database access attempted from background thread ...``+ before any research ran.+ """+ service = _make_service_with_run()+ captured = _run_thread_and_capture(service)++ assert captured["context_username"] == "testuser"+ assert captured["manager"] is not None+++def _stub_service_for_sync_run(service, captured):+ """Stub the task pipeline so _run_benchmark_thread can run inline."""++ def fake_process(task, search_config, evaluation_config):+ from local_deep_research.utilities.request_context import (+ get_current_username,+ )++ captured["context_username"] = get_current_username()+ return {"example_id": task["example_id"], "success": True}++ service._create_task_queue = Mock(+ return_value=[{"example_id": "e0", "dataset": "simpleqa"}]+ )+ service._process_benchmark_task = fake_process+ service._sync_results_to_database = Mock(return_value=None)+ service.queue_tracker = Mock()+++def test_benchmark_thread_context_restores_after_run():+ """The pushed owner must be scoped to the run, not leak afterwards.++ A separate ``threading.Thread`` never writes its contextvars back to the+ parent, so observing the parent after a join proves nothing. Instead run+ ``_run_benchmark_thread`` synchronously in this thread: the owner must be+ visible inside the run and gone (back to None) once it returns.+ """+ from local_deep_research.utilities.request_context import (+ get_current_username,+ )++ assert get_current_username() is None+ service = _make_service_with_run()+ captured = {}+ _stub_service_for_sync_run(service, captured)++ service._run_benchmark_thread("testuser", 1)++ assert captured["context_username"] == "testuser"+ assert get_current_username() is None+++def test_benchmark_thread_context_restores_outer_user_after_run():+ """A pre-existing outer request user is restored, not cleared."""+ from local_deep_research.utilities.request_context import (+ get_current_username,+ request_user,+ )++ service = _make_service_with_run()+ captured = {}+ _stub_service_for_sync_run(service, captured)++ with request_user("outer_user"):+ service._run_benchmark_thread("testuser", 1)+ assert captured["context_username"] == "testuser"+ assert get_current_username() == "outer_user"++ assert get_current_username() is None+++def test_benchmark_thread_completes_run_with_context_pushed():+ """End-to-end thread health: with the context pushed the run completes+ (not FAILED) and the task result is recorded."""+ from local_deep_research.database.models import BenchmarkStatus++ service = _make_service_with_run()+ _run_thread_and_capture(service)++ completion = service.active_runs[("testuser", 1)]["completion_info"]+ assert completion["status"] == BenchmarkStatus.COMPLETED+ assert completion["completed_examples"] == 1+ assert completion["failed_examples"] == 0diff --git a/tests/web/test_research_state_registry.py b/tests/web/test_research_state_registry.pyindex baaaa42973..1a8aa0edb2 100644--- a/tests/web/test_research_state_registry.py+++ b/tests/web/test_research_state_registry.py@@ -763,7 +763,7 @@ def test_clear_termination_flag_call_sites_are_pinned():``benchmark_service._process_benchmark_task``), and both callersbelow operate only on that scope:- * ``_run_benchmark_thread`` clears it in a per-task ``finally``+ * ``_run_benchmark_thread_impl`` clears it in a per-task ``finally``so one finished task's cancellation state can never leak intothe next task's LLM calls;* ``cancel_benchmark`` reclaims it only on the cancel-vs-finish@@ -776,6 +776,15 @@ def test_clear_termination_flag_call_sites_are_pinned():re-deriving (not just extending) this list is required if it everchanges, because a further caller needs the same argument or itreintroduces cross-run kills.++ Premise updated by PR #7363: the thread body -- including this+ per-task ``finally`` -- moved verbatim into+ ``_run_benchmark_thread_impl``, behind a new thin+ ``_run_benchmark_thread`` wrapper that only pushes the+ request-user context before delegating. That refactor changed+ the enclosing function of the clear call, so the pinned qualname+ below was updated to match; the call sites themselves, their+ arguments, and the safety argument above are unchanged."""sites, _ = _census()callers = [@@ -786,7 +795,7 @@ def test_clear_termination_flag_call_sites_are_pinned():assert callers == [("benchmarks/web_api/benchmark_service.py",- "_run_benchmark_thread",+ "_run_benchmark_thread_impl",),("benchmarks/web_api/benchmark_service.py",