Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(benchmarks): push request-user context in the benchmark thread (#7363)

LearningCircuit8ab28bcba864
diff --git a/changelog.d/7362.bugfix.md b/changelog.d/7362.bugfix.md
new file mode 100644
index 0000000000..fa1a3ceb47
--- /dev/null
+++ b/changelog.d/7362.bugfix.md
@@ -0,0 +1 @@
+Fix the web benchmark failing every task since the FastAPI migration: the background thread now pushes the run's owner via `request_user(...)` (nothing is pushed for a missing owner, matching scheduler jobs) so `quick_summary`'s `@no_db_settings` resolves the user's settings instead of tripping the background-thread database guard. The benchmark thread also closes its cached per-thread DB sessions after each task instead of accumulating them for the whole run.
diff --git a/src/local_deep_research/benchmarks/web_api/benchmark_service.py b/src/local_deep_research/benchmarks/web_api/benchmark_service.py
index 6b4bd14246..d84e3ec04f 100644
--- a/src/local_deep_research/benchmarks/web_api/benchmark_service.py
+++ b/src/local_deep_research/benchmarks/web_api/benchmark_service.py
@@ -4,6 +4,7 @@ import hashlib
import json
import threading
import time
+from contextlib import nullcontext
from datetime import UTC, datetime
from enum import Enum
from typing import Any, Dict, List, Optional, Tuple
@@ -26,6 +27,7 @@ from ..datasets import load_dataset
from ..graders import extract_answer_from_response, grade_single_result
from ..runners import format_query
from ...database.thread_local_session import thread_cleanup
+from ...utilities.request_context import request_user
# Generic failure message surfaced to API clients in place of the raw benchmark
# exception. ``benchmark_run.error_message`` is set from ``str(e)`` of the
@@ -472,6 +474,22 @@ class BenchmarkService:
self, username: Optional[str], benchmark_run_id: int
):
"""Main benchmark execution thread."""
+ # A plain threading.Thread does not inherit the request context
+ # (contextvars), so quick_summary's @no_db_settings wrapper — which
+ # resolves get_settings_manager() before the wrapped function runs —
+ # finds no user here and trips the background-thread DB guard:
+ # every task failed with "Research error" and 0% accuracy. Push the
+ # owner for the whole run, the same way scheduler jobs do
+ # (scheduler/background.py). A missing owner pushes nothing (as in
+ # scheduler/background.py) rather than a "benchmark_user" fallback,
+ # which could resolve an unrelated account of that name.
+ user_ctx = request_user(username) if username else nullcontext()
+ with user_ctx:
+ self._run_benchmark_thread_impl(username, benchmark_run_id)
+
+ def _run_benchmark_thread_impl(
+ self, username: Optional[str], benchmark_run_id: int
+ ):
# IMPORTANT: This runs in a background thread, so we cannot access the user database
# Using in-memory queue tracker for benchmark status tracking
@@ -626,6 +644,23 @@ class BenchmarkService:
"Failed to clear benchmark task flag "
"(non-critical)"
)
+ # Close the DB sessions this task opened through
+ # db_utils' per-thread cache (quick_summary's ambient
+ # settings manager). The run's thread lives for every
+ # task, so without this they would accumulate until
+ # @thread_cleanup at thread exit; the next task
+ # re-opens one lazily on a cache miss.
+ try:
+ from ...utilities.db_utils import (
+ cleanup_cached_user_sessions_current_thread,
+ )
+
+ cleanup_cached_user_sessions_current_thread()
+ except Exception:
+ logger.debug(
+ "Failed to clean up benchmark task DB "
+ "sessions (non-critical)"
+ )
# Store result in memory for now (will be saved later)
if "results" not in self.active_runs[run_key]:
diff --git a/tests/benchmarks/test_benchmark_service_thread_context.py b/tests/benchmarks/test_benchmark_service_thread_context.py
new file mode 100644
index 0000000000..08cb87926c
--- /dev/null
+++ b/tests/benchmarks/test_benchmark_service_thread_context.py
@@ -0,0 +1,165 @@
+"""Tests for the benchmark thread's request-user context.
+
+The benchmark runs in a plain ``threading.Thread``, which does NOT inherit
+the request context (contextvars). ``quick_summary`` is decorated with
+``@no_db_settings``, whose wrapper eagerly resolves ``get_settings_manager()``
+before the wrapped function runs — with no username and no request context,
+the background-thread DB guard raises RuntimeError and every benchmark task
+fails ("Research error" in the logs, 0% accuracy).
+
+The thread must push its owner via ``request_user(...)`` for the duration of
+the run, the same way scheduler jobs do (scheduler/background.py).
+"""
+
+import threading
+from unittest.mock import Mock
+
+
+def _run_thread_and_capture(service, username="testuser", run_id=1):
+ """Run _run_benchmark_thread in a real background thread.
+
+ ``_process_benchmark_task`` is stubbed to reproduce what
+ ``@no_db_settings`` does inside quick_summary: resolve the settings
+ manager from the calling thread's context. Returns what the stub saw.
+ """
+ captured = {}
+
+ def fake_process(task, search_config, evaluation_config):
+ from local_deep_research.utilities.db_utils import get_settings_manager
+ from local_deep_research.utilities.request_context import (
+ get_current_username,
+ )
+
+ captured["context_username"] = get_current_username()
+ # This is the exact call that raised RuntimeError from the
+ # background thread before the fix.
+ captured["manager"] = get_settings_manager()
+ return {"example_id": task["example_id"], "success": True}
+
+ service._create_task_queue = Mock(
+ return_value=[{"example_id": "e0", "dataset": "simpleqa"}]
+ )
+ service._process_benchmark_task = fake_process
+ service._sync_results_to_database = Mock(return_value=None)
+ service.queue_tracker = Mock()
+
+ thread = threading.Thread(
+ target=service._run_benchmark_thread, args=(username, run_id)
+ )
+ thread.start()
+ thread.join(timeout=30)
+
+ assert not thread.is_alive(), "benchmark thread did not finish"
+ return captured
+
+
+def _make_service_with_run(username="testuser", run_id=1):
+ from local_deep_research.benchmarks.web_api.benchmark_service import (
+ BenchmarkService,
+ )
+
+ service = BenchmarkService(socket_service=Mock())
+ service.active_runs[(username, run_id)] = {
+ "data": {
+ "benchmark_run_id": run_id,
+ "username": username,
+ "user_password": None,
+ "config_hash": "abc12345",
+ "datasets_config": {"simpleqa": {"count": 1}},
+ "search_config": {},
+ "evaluation_config": {},
+ "settings_snapshot": {},
+ },
+ "status": "running",
+ "results": [],
+ }
+ return service
+
+
+def test_benchmark_thread_resolves_settings_manager_from_owner_context():
+ """quick_summary's @no_db_settings must see the run's owner.
+
+ Reproduces the web-benchmark failure where every task died with
+ ``RuntimeError: Database access attempted from background thread ...``
+ before any research ran.
+ """
+ service = _make_service_with_run()
+ captured = _run_thread_and_capture(service)
+
+ assert captured["context_username"] == "testuser"
+ assert captured["manager"] is not None
+
+
+def _stub_service_for_sync_run(service, captured):
+ """Stub the task pipeline so _run_benchmark_thread can run inline."""
+
+ def fake_process(task, search_config, evaluation_config):
+ from local_deep_research.utilities.request_context import (
+ get_current_username,
+ )
+
+ captured["context_username"] = get_current_username()
+ return {"example_id": task["example_id"], "success": True}
+
+ service._create_task_queue = Mock(
+ return_value=[{"example_id": "e0", "dataset": "simpleqa"}]
+ )
+ service._process_benchmark_task = fake_process
+ service._sync_results_to_database = Mock(return_value=None)
+ service.queue_tracker = Mock()
+
+
+def test_benchmark_thread_context_restores_after_run():
+ """The pushed owner must be scoped to the run, not leak afterwards.
+
+ A separate ``threading.Thread`` never writes its contextvars back to the
+ parent, so observing the parent after a join proves nothing. Instead run
+ ``_run_benchmark_thread`` synchronously in this thread: the owner must be
+ visible inside the run and gone (back to None) once it returns.
+ """
+ from local_deep_research.utilities.request_context import (
+ get_current_username,
+ )
+
+ assert get_current_username() is None
+ service = _make_service_with_run()
+ captured = {}
+ _stub_service_for_sync_run(service, captured)
+
+ service._run_benchmark_thread("testuser", 1)
+
+ assert captured["context_username"] == "testuser"
+ assert get_current_username() is None
+
+
+def test_benchmark_thread_context_restores_outer_user_after_run():
+ """A pre-existing outer request user is restored, not cleared."""
+ from local_deep_research.utilities.request_context import (
+ get_current_username,
+ request_user,
+ )
+
+ service = _make_service_with_run()
+ captured = {}
+ _stub_service_for_sync_run(service, captured)
+
+ with request_user("outer_user"):
+ service._run_benchmark_thread("testuser", 1)
+ assert captured["context_username"] == "testuser"
+ assert get_current_username() == "outer_user"
+
+ assert get_current_username() is None
+
+
+def test_benchmark_thread_completes_run_with_context_pushed():
+ """End-to-end thread health: with the context pushed the run completes
+ (not FAILED) and the task result is recorded."""
+ from local_deep_research.database.models import BenchmarkStatus
+
+ service = _make_service_with_run()
+ _run_thread_and_capture(service)
+
+ completion = service.active_runs[("testuser", 1)]["completion_info"]
+ assert completion["status"] == BenchmarkStatus.COMPLETED
+ assert completion["completed_examples"] == 1
+ assert completion["failed_examples"] == 0
diff --git a/tests/web/test_research_state_registry.py b/tests/web/test_research_state_registry.py
index baaaa42973..1a8aa0edb2 100644
--- a/tests/web/test_research_state_registry.py
+++ b/tests/web/test_research_state_registry.py
@@ -763,7 +763,7 @@ def test_clear_termination_flag_call_sites_are_pinned():
``benchmark_service._process_benchmark_task``), and both callers
below operate only on that scope:
- * ``_run_benchmark_thread`` clears it in a per-task ``finally``
+ * ``_run_benchmark_thread_impl`` clears it in a per-task ``finally``
so one finished task's cancellation state can never leak into
the next task's LLM calls;
* ``cancel_benchmark`` reclaims it only on the cancel-vs-finish
@@ -776,6 +776,15 @@ def test_clear_termination_flag_call_sites_are_pinned():
re-deriving (not just extending) this list is required if it ever
changes, because a further caller needs the same argument or it
reintroduces cross-run kills.
+
+ Premise updated by PR #7363: the thread body -- including this
+ per-task ``finally`` -- moved verbatim into
+ ``_run_benchmark_thread_impl``, behind a new thin
+ ``_run_benchmark_thread`` wrapper that only pushes the
+ request-user context before delegating. That refactor changed
+ the enclosing function of the clear call, so the pinned qualname
+ below was updated to match; the call sites themselves, their
+ arguments, and the safety argument above are unchanged.
"""
sites, _ = _census()
callers = [
@@ -786,7 +795,7 @@ def test_clear_termination_flag_call_sites_are_pinned():
assert callers == [
(
"benchmarks/web_api/benchmark_service.py",
- "_run_benchmark_thread",
+ "_run_benchmark_thread_impl",
),
(
"benchmarks/web_api/benchmark_service.py",