Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

5.9 KB
Raw Blame
"""Tests for the benchmark thread's request-user context.

The benchmark runs in a plain ``threading.Thread``, which does NOT inherit
the request context (contextvars). ``quick_summary`` is decorated with
``@no_db_settings``, whose wrapper eagerly resolves ``get_settings_manager()``
before the wrapped function runs — with no username and no request context,
the background-thread DB guard raises RuntimeError and every benchmark task
fails ("Research error" in the logs, 0% accuracy).

The thread must push its owner via ``request_user(...)`` for the duration of
the run, the same way scheduler jobs do (scheduler/background.py).
"""

import threading
from unittest.mock import Mock


def _run_thread_and_capture(service, username="testuser", run_id=1):
    """Run _run_benchmark_thread in a real background thread.

    ``_process_benchmark_task`` is stubbed to reproduce what
    ``@no_db_settings`` does inside quick_summary: resolve the settings
    manager from the calling thread's context. Returns what the stub saw.
    """
    captured = {}

    def fake_process(task, search_config, evaluation_config):
        from local_deep_research.utilities.db_utils import get_settings_manager
        from local_deep_research.utilities.request_context import (
            get_current_username,
        )

        captured["context_username"] = get_current_username()
        # This is the exact call that raised RuntimeError from the
        # background thread before the fix.
        captured["manager"] = get_settings_manager()
        return {"example_id": task["example_id"], "success": True}

    service._create_task_queue = Mock(
        return_value=[{"example_id": "e0", "dataset": "simpleqa"}]
    )
    service._process_benchmark_task = fake_process
    service._sync_results_to_database = Mock(return_value=None)
    service.queue_tracker = Mock()

    thread = threading.Thread(
        target=service._run_benchmark_thread, args=(username, run_id)
    )
    thread.start()
    thread.join(timeout=30)

    assert not thread.is_alive(), "benchmark thread did not finish"
    return captured


def _make_service_with_run(username="testuser", run_id=1):
    from local_deep_research.benchmarks.web_api.benchmark_service import (
        BenchmarkService,
    )

    service = BenchmarkService(socket_service=Mock())
    service.active_runs[(username, run_id)] = {
        "data": {
            "benchmark_run_id": run_id,
            "username": username,
            "user_password": None,
            "config_hash": "abc12345",
            "datasets_config": {"simpleqa": {"count": 1}},
            "search_config": {},
            "evaluation_config": {},
            "settings_snapshot": {},
        },
        "status": "running",
        "results": [],
    }
    return service


def test_benchmark_thread_resolves_settings_manager_from_owner_context():
    """quick_summary's @no_db_settings must see the run's owner.

    Reproduces the web-benchmark failure where every task died with
    ``RuntimeError: Database access attempted from background thread ...``
    before any research ran.
    """
    service = _make_service_with_run()
    captured = _run_thread_and_capture(service)

    assert captured["context_username"] == "testuser"
    assert captured["manager"] is not None


def _stub_service_for_sync_run(service, captured):
    """Stub the task pipeline so _run_benchmark_thread can run inline."""

    def fake_process(task, search_config, evaluation_config):
        from local_deep_research.utilities.request_context import (
            get_current_username,
        )

        captured["context_username"] = get_current_username()
        return {"example_id": task["example_id"], "success": True}

    service._create_task_queue = Mock(
        return_value=[{"example_id": "e0", "dataset": "simpleqa"}]
    )
    service._process_benchmark_task = fake_process
    service._sync_results_to_database = Mock(return_value=None)
    service.queue_tracker = Mock()


def test_benchmark_thread_context_restores_after_run():
    """The pushed owner must be scoped to the run, not leak afterwards.

    A separate ``threading.Thread`` never writes its contextvars back to the
    parent, so observing the parent after a join proves nothing. Instead run
    ``_run_benchmark_thread`` synchronously in this thread: the owner must be
    visible inside the run and gone (back to None) once it returns.
    """
    from local_deep_research.utilities.request_context import (
        get_current_username,
    )

    assert get_current_username() is None
    service = _make_service_with_run()
    captured = {}
    _stub_service_for_sync_run(service, captured)

    service._run_benchmark_thread("testuser", 1)

    assert captured["context_username"] == "testuser"
    assert get_current_username() is None


def test_benchmark_thread_context_restores_outer_user_after_run():
    """A pre-existing outer request user is restored, not cleared."""
    from local_deep_research.utilities.request_context import (
        get_current_username,
        request_user,
    )

    service = _make_service_with_run()
    captured = {}
    _stub_service_for_sync_run(service, captured)

    with request_user("outer_user"):
        service._run_benchmark_thread("testuser", 1)
        assert captured["context_username"] == "testuser"
        assert get_current_username() == "outer_user"

    assert get_current_username() is None


def test_benchmark_thread_completes_run_with_context_pushed():
    """End-to-end thread health: with the context pushed the run completes
    (not FAILED) and the task result is recorded."""
    from local_deep_research.database.models import BenchmarkStatus

    service = _make_service_with_run()
    _run_thread_and_capture(service)

    completion = service.active_runs[("testuser", 1)]["completion_info"]
    assert completion["status"] == BenchmarkStatus.COMPLETED
    assert completion["completed_examples"] == 1
    assert completion["failed_examples"] == 0