~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Tests for the benchmark thread's request-user context.
The benchmark runs in a plain ``threading.Thread``, which does NOT inherit
the request context (contextvars). ``quick_summary`` is decorated with
``@no_db_settings``, whose wrapper eagerly resolves ``get_settings_manager()``
before the wrapped function runs — with no username and no request context,
the background-thread DB guard raises RuntimeError and every benchmark task
fails ("Research error" in the logs, 0% accuracy).
The thread must push its owner via ``request_user(...)`` for the duration of
the run, the same way scheduler jobs do (scheduler/background.py).
"""
import threading
from unittest.mock import Mock
def _run_thread_and_capture(service, username="testuser", run_id=1):
"""Run _run_benchmark_thread in a real background thread.
``_process_benchmark_task`` is stubbed to reproduce what
``@no_db_settings`` does inside quick_summary: resolve the settings
manager from the calling thread's context. Returns what the stub saw.
"""
captured = {}
def fake_process(task, search_config, evaluation_config):
from local_deep_research.utilities.db_utils import get_settings_manager
from local_deep_research.utilities.request_context import (
get_current_username,
)
captured["context_username"] = get_current_username()
# This is the exact call that raised RuntimeError from the
# background thread before the fix.
captured["manager"] = get_settings_manager()
return {"example_id": task["example_id"], "success": True}
service._create_task_queue = Mock(
return_value=[{"example_id": "e0", "dataset": "simpleqa"}]
)
service._process_benchmark_task = fake_process
service._sync_results_to_database = Mock(return_value=None)
service.queue_tracker = Mock()
thread = threading.Thread(
target=service._run_benchmark_thread, args=(username, run_id)
)
thread.start()
thread.join(timeout=30)
assert not thread.is_alive(), "benchmark thread did not finish"
return captured
def _make_service_with_run(username="testuser", run_id=1):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkService,
)
service = BenchmarkService(socket_service=Mock())
service.active_runs[(username, run_id)] = {
"data": {
"benchmark_run_id": run_id,
"username": username,
"user_password": None,
"config_hash": "abc12345",
"datasets_config": {"simpleqa": {"count": 1}},
"search_config": {},
"evaluation_config": {},
"settings_snapshot": {},
},
"status": "running",
"results": [],
}
return service
def test_benchmark_thread_resolves_settings_manager_from_owner_context():
"""quick_summary's @no_db_settings must see the run's owner.
Reproduces the web-benchmark failure where every task died with
``RuntimeError: Database access attempted from background thread ...``
before any research ran.
"""
service = _make_service_with_run()
captured = _run_thread_and_capture(service)
assert captured["context_username"] == "testuser"
assert captured["manager"] is not None
def _stub_service_for_sync_run(service, captured):
"""Stub the task pipeline so _run_benchmark_thread can run inline."""
def fake_process(task, search_config, evaluation_config):
from local_deep_research.utilities.request_context import (
get_current_username,
)
captured["context_username"] = get_current_username()
return {"example_id": task["example_id"], "success": True}
service._create_task_queue = Mock(
return_value=[{"example_id": "e0", "dataset": "simpleqa"}]
)
service._process_benchmark_task = fake_process
service._sync_results_to_database = Mock(return_value=None)
service.queue_tracker = Mock()
def test_benchmark_thread_context_restores_after_run():
"""The pushed owner must be scoped to the run, not leak afterwards.
A separate ``threading.Thread`` never writes its contextvars back to the
parent, so observing the parent after a join proves nothing. Instead run
``_run_benchmark_thread`` synchronously in this thread: the owner must be
visible inside the run and gone (back to None) once it returns.
"""
from local_deep_research.utilities.request_context import (
get_current_username,
)
assert get_current_username() is None
service = _make_service_with_run()
captured = {}
_stub_service_for_sync_run(service, captured)
service._run_benchmark_thread("testuser", 1)
assert captured["context_username"] == "testuser"
assert get_current_username() is None
def test_benchmark_thread_context_restores_outer_user_after_run():
"""A pre-existing outer request user is restored, not cleared."""
from local_deep_research.utilities.request_context import (
get_current_username,
request_user,
)
service = _make_service_with_run()
captured = {}
_stub_service_for_sync_run(service, captured)
with request_user("outer_user"):
service._run_benchmark_thread("testuser", 1)
assert captured["context_username"] == "testuser"
assert get_current_username() == "outer_user"
assert get_current_username() is None
def test_benchmark_thread_completes_run_with_context_pushed():
"""End-to-end thread health: with the context pushed the run completes
(not FAILED) and the task result is recorded."""
from local_deep_research.database.models import BenchmarkStatus
service = _make_service_with_run()
_run_thread_and_capture(service)
completion = service.active_runs[("testuser", 1)]["completion_info"]
assert completion["status"] == BenchmarkStatus.COMPLETED
assert completion["completed_examples"] == 1
assert completion["failed_examples"] == 0