~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Comprehensive coverage tests for benchmarks/web_api/benchmark_service.py.
Focuses on uncovered code paths: complex branching in _run_benchmark_thread,
_process_benchmark_task, _sync_results_to_database, get_benchmark_status,
sync_pending_results, cancel_benchmark, update_benchmark_status, and more.
"""
import json
from datetime import UTC, datetime, timedelta
from unittest.mock import MagicMock, Mock, patch
import pytest
# ---------------------------------------------------------------------------
# Helpers to import from the module under test
# ---------------------------------------------------------------------------
MODULE = "local_deep_research.benchmarks.web_api.benchmark_service"
SETTINGS_CTX_MODULE = "local_deep_research.config.thread_settings"
def _import_service():
"""Import BenchmarkService with SocketIOService mocked."""
with patch(f"{MODULE}.SocketIOService"):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkService,
)
return BenchmarkService
def _make_service(socket=None):
"""Create a BenchmarkService with a mock socket service."""
cls = _import_service()
svc = cls(socket_service=socket or MagicMock())
return svc
# ============================================================
# BenchmarkQueueTracker – deeper coverage
# ============================================================
class TestQueueTrackerCleanup:
"""Cover cleanup_completed_tasks edge cases."""
def test_cleanup_removes_old_completed(self):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkQueueTracker,
BenchmarkTaskStatus,
)
tracker = BenchmarkQueueTracker()
tracker.add_task("old1", "user")
tracker.update_task_status("old1", BenchmarkTaskStatus.COMPLETED)
# Manually set updated_at to the past
tracker.tasks["old1"]["updated_at"] = datetime.now(UTC) - timedelta(
seconds=7200
)
tracker.cleanup_completed_tasks(max_age_seconds=3600)
assert "old1" not in tracker.tasks
def test_cleanup_keeps_recent_completed(self):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkQueueTracker,
BenchmarkTaskStatus,
)
tracker = BenchmarkQueueTracker()
tracker.add_task("new1", "user")
tracker.update_task_status("new1", BenchmarkTaskStatus.COMPLETED)
tracker.cleanup_completed_tasks(max_age_seconds=3600)
assert "new1" in tracker.tasks
def test_cleanup_removes_old_failed(self):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkQueueTracker,
BenchmarkTaskStatus,
)
tracker = BenchmarkQueueTracker()
tracker.add_task("fail1", "user")
tracker.update_task_status("fail1", BenchmarkTaskStatus.FAILED)
tracker.tasks["fail1"]["updated_at"] = datetime.now(UTC) - timedelta(
seconds=7200
)
tracker.cleanup_completed_tasks(max_age_seconds=3600)
assert "fail1" not in tracker.tasks
def test_cleanup_removes_old_cancelled(self):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkQueueTracker,
BenchmarkTaskStatus,
)
tracker = BenchmarkQueueTracker()
tracker.add_task("canc1", "user")
tracker.update_task_status("canc1", BenchmarkTaskStatus.CANCELLED)
tracker.tasks["canc1"]["updated_at"] = datetime.now(UTC) - timedelta(
seconds=7200
)
tracker.cleanup_completed_tasks(max_age_seconds=3600)
assert "canc1" not in tracker.tasks
def test_cleanup_keeps_processing_tasks(self):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkQueueTracker,
BenchmarkTaskStatus,
)
tracker = BenchmarkQueueTracker()
tracker.add_task("proc1", "user")
tracker.update_task_status("proc1", BenchmarkTaskStatus.PROCESSING)
tracker.tasks["proc1"]["updated_at"] = datetime.now(UTC) - timedelta(
seconds=7200
)
tracker.cleanup_completed_tasks(max_age_seconds=3600)
assert "proc1" in tracker.tasks
def test_cleanup_uses_created_at_when_no_updated_at(self):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkQueueTracker,
BenchmarkTaskStatus,
)
tracker = BenchmarkQueueTracker()
tracker.add_task("t1", "user")
# Set status directly without going through update (no updated_at)
tracker.tasks["t1"]["status"] = BenchmarkTaskStatus.COMPLETED.value
tracker.tasks["t1"]["created_at"] = datetime.now(UTC) - timedelta(
seconds=7200
)
if "updated_at" in tracker.tasks["t1"]:
del tracker.tasks["t1"]["updated_at"]
tracker.cleanup_completed_tasks(max_age_seconds=3600)
assert "t1" not in tracker.tasks
def test_update_nonexistent_task_logs_warning(self):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkQueueTracker,
BenchmarkTaskStatus,
)
tracker = BenchmarkQueueTracker()
# Should not raise; logs a warning
tracker.update_task_status("nonexistent", BenchmarkTaskStatus.COMPLETED)
def test_remove_task(self):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkQueueTracker,
)
tracker = BenchmarkQueueTracker()
tracker.add_task("rm1", "user")
tracker.remove_task("rm1")
assert tracker.get_task_status("rm1") is None
def test_remove_nonexistent_task(self):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkQueueTracker,
)
tracker = BenchmarkQueueTracker()
# Should not raise
tracker.remove_task("doesnotexist")
# ============================================================
# BenchmarkService – initialization
# ============================================================
class TestBenchmarkServiceInit:
def test_init_with_provided_socket(self):
mock_socket = MagicMock()
svc = _make_service(socket=mock_socket)
assert svc.socket_service is mock_socket
def test_init_default_socket_emit_to_room_delegates(self):
"""With no socket provided, __init__ constructs a real
``SocketIOService`` adapter whose ``emit_to_room`` forwards to the
FastAPI-native ``socketio_asgi`` emit path.
Replaces the old ``test_init_fallback_socket``: pre-FastAPI the
constructor wrapped ``SocketIOService()`` in a try/except that fell
back to a no-op shim when the Flask app was unavailable. The
migration (commit dec8e1452) dropped that fallback because the new
``SocketIOService`` is a default-constructible adapter that cannot
fail at construction — so the obsolete fallback assertion is replaced
with the live delegation contract.
"""
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkService,
)
svc = BenchmarkService()
with patch(
"local_deep_research.web.services.socketio_asgi.emit_socket_event",
return_value=True,
) as emit:
result = svc.socket_service.emit_to_room("event", {}, room="room")
assert result is True
emit.assert_called_once_with("event", {}, room="room")
# ============================================================
# generate_config_hash / generate_query_hash
# ============================================================
class TestHashGeneration:
def test_config_hash_deterministic(self):
svc = _make_service()
cfg = {"iterations": 5, "search_tool": "searxng", "model_name": "gpt-4"}
h1 = svc.generate_config_hash(cfg)
h2 = svc.generate_config_hash(cfg)
assert h1 == h2
assert len(h1) == 8
def test_config_hash_ignores_none_values(self):
svc = _make_service()
cfg1 = {"iterations": 5, "search_tool": None}
cfg2 = {"iterations": 5}
assert svc.generate_config_hash(cfg1) == svc.generate_config_hash(cfg2)
def test_config_hash_differs_for_different_configs(self):
svc = _make_service()
h1 = svc.generate_config_hash({"iterations": 5})
h2 = svc.generate_config_hash({"iterations": 10})
assert h1 != h2
def test_query_hash_deterministic(self):
svc = _make_service()
h1 = svc.generate_query_hash("What is AI?", "simpleqa", "ex1")
h2 = svc.generate_query_hash("What is AI?", "simpleqa", "ex1")
assert h1 == h2
def test_query_hash_strips_whitespace(self):
svc = _make_service()
h1 = svc.generate_query_hash(" What is AI? ", "simpleqa", "ex1")
h2 = svc.generate_query_hash("What is AI?", "simpleqa", "ex1")
assert h1 == h2
def test_query_hash_case_insensitive_dataset(self):
svc = _make_service()
h1 = svc.generate_query_hash("What is AI?", "SimpleQA", "ex1")
h2 = svc.generate_query_hash("What is AI?", "simpleqa", "ex1")
assert h1 == h2
def test_query_hash_keys_on_example_id(self):
# #4860: distinct examples, identical question -> distinct hashes.
svc = _make_service()
h1 = svc.generate_query_hash("What is AI?", "simpleqa", "ex1")
h2 = svc.generate_query_hash("What is AI?", "simpleqa", "ex2")
assert h1 != h2
# ============================================================
# create_benchmark_run
# ============================================================
class TestCreateBenchmarkRun:
@patch(f"{MODULE}.BenchmarkRun")
def test_create_benchmark_run_success(self, mock_run_cls):
svc = _make_service()
mock_session = MagicMock()
mock_run_instance = MagicMock()
mock_run_instance.id = 42
mock_run_cls.return_value = mock_run_instance
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
result = svc.create_benchmark_run(
run_name="test",
search_config={"iterations": 5},
evaluation_config={},
datasets_config={"simpleqa": {"count": 10}},
username="user1",
)
assert result == 42
mock_session.add.assert_called_once_with(mock_run_instance)
mock_session.commit.assert_called_once()
@patch(f"{MODULE}.BenchmarkRun")
def test_create_benchmark_run_db_error(self, mock_run_cls):
svc = _make_service()
mock_session = MagicMock()
mock_session.commit.side_effect = RuntimeError("db error")
mock_run_cls.return_value = MagicMock()
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
with pytest.raises(RuntimeError, match="db error"):
svc.create_benchmark_run(
run_name="test",
search_config={},
evaluation_config={},
datasets_config={},
)
mock_session.rollback.assert_called_once()
# ============================================================
# _create_task_queue
# ============================================================
class TestCreateTaskQueue:
@patch(f"{MODULE}.load_dataset")
def test_creates_tasks_from_dataset(self, mock_load):
svc = _make_service()
mock_load.return_value = [
{"id": "ex1", "problem": "Q1?", "answer": "A1"},
{"id": "ex2", "problem": "Q2?", "answer": "A2"},
]
tasks = svc._create_task_queue(
{"simpleqa": {"count": 2}},
benchmark_run_id=1,
)
assert len(tasks) == 2
assert tasks[0]["question"] == "Q1?"
assert tasks[0]["dataset_type"] == "simpleqa"
@patch(f"{MODULE}.load_dataset")
def test_passes_seed_from_dataset_config(self, mock_load):
"""A "seed" in the dataset config drives reproducible sampling."""
svc = _make_service()
mock_load.return_value = [
{"id": "ex1", "problem": "Q1?", "answer": "A1"},
]
svc._create_task_queue(
{"simpleqa": {"count": 1, "seed": 7}},
benchmark_run_id=1,
)
mock_load.assert_called_once_with(
dataset_type="simpleqa", num_examples=1, seed=7
)
@patch(f"{MODULE}.load_dataset")
def test_no_seed_means_random_sampling(self, mock_load):
"""Without a configured seed the sample stays random (seed=None)."""
svc = _make_service()
mock_load.return_value = [
{"id": "ex1", "problem": "Q1?", "answer": "A1"},
]
svc._create_task_queue(
{"simpleqa": {"count": 1}},
benchmark_run_id=1,
)
mock_load.assert_called_once_with(
dataset_type="simpleqa", num_examples=1, seed=None
)
@patch(f"{MODULE}.load_dataset")
def test_queues_every_sampled_question(self, mock_load):
"""Regression test for #4498: every sampled question becomes a task.
The removed cross-run reuse feature silently skipped questions that
had results in previous compatible runs, which broke the
completed/total accounting (#4451). A run must always process its
full sample, even if identical questions were answered before."""
svc = _make_service()
examples = [
{"id": f"ex{i}", "problem": f"Q{i}?", "answer": f"A{i}"}
for i in range(5)
]
mock_load.return_value = examples
tasks = svc._create_task_queue(
{"simpleqa": {"count": 5}},
benchmark_run_id=1,
)
assert len(tasks) == len(examples)
assert [t["question"] for t in tasks] == [
e["problem"] for e in examples
]
assert [t["task_index"] for t in tasks] == list(range(5))
@patch(f"{MODULE}.load_dataset")
def test_skips_zero_count_datasets(self, mock_load):
svc = _make_service()
tasks = svc._create_task_queue(
{"simpleqa": {"count": 0}},
benchmark_run_id=1,
)
assert len(tasks) == 0
mock_load.assert_not_called()
@patch(f"{MODULE}.load_dataset")
def test_browsecomp_dataset_type(self, mock_load):
svc = _make_service()
mock_load.return_value = [
{"id": "b1", "problem": "Browse Q?", "answer": "Browse A"},
]
tasks = svc._create_task_queue(
{"browsecomp": {"count": 1}},
benchmark_run_id=1,
)
assert len(tasks) == 1
assert tasks[0]["dataset_type"] == "browsecomp"
@patch(f"{MODULE}.load_dataset")
def test_example_without_id_gets_default(self, mock_load):
svc = _make_service()
mock_load.return_value = [
{"problem": "Q?", "answer": "A"}, # no id
]
tasks = svc._create_task_queue(
{"simpleqa": {"count": 1}},
benchmark_run_id=1,
)
assert tasks[0]["example_id"] == "example_0"
# ============================================================
# _process_benchmark_task
# ============================================================
class TestProcessBenchmarkTask:
def _make_task(self, **overrides):
task = {
"benchmark_run_id": 1,
"example_id": "ex1",
"dataset_type": "simpleqa",
"question": "What is 2+2?",
"correct_answer": "4",
"query_hash": "abc123",
"task_index": 0,
"username": "user1",
"user_password": None,
}
task.update(overrides)
return task
@pytest.mark.parametrize(
("findings", "is_failure"),
[
([{"phase": "Error", "content": "Error: synthesis detail"}], True),
(
[
{
"phase": "Final synthesis",
"content": "Error: synthesis detail",
}
],
False,
),
(
[
{"phase": "Error", "content": "earlier step"},
{
"phase": "Final synthesis",
"content": "Error: synthesis detail",
},
],
False,
),
([{"phase": "Error", "content": "different content"}], False),
(None, False),
],
)
@patch(f"{MODULE}.grade_single_result")
@patch(f"{MODULE}.extract_answer_from_response")
@patch(f"{MODULE}.quick_summary")
@patch(f"{MODULE}.format_query")
def test_terminal_research_finding_controls_result_state(
self,
mock_format,
mock_summary,
mock_extract,
mock_grade,
findings,
is_failure,
):
svc = _make_service()
mock_format.return_value = "q"
summary = "Error: synthesis detail"
mock_summary.return_value = {
"summary": summary,
"findings": findings,
"sources": [],
}
mock_extract.return_value = {"extracted_answer": "answer"}
mock_grade.return_value = {"is_correct": True}
context = MagicMock()
context.snapshot = {}
with patch(
f"{SETTINGS_CTX_MODULE}.get_settings_context", return_value=context
):
result = svc._process_benchmark_task(self._make_task(), {}, {})
if is_failure:
assert result["research_error"] == (
"Research failed. Check the server logs for details."
)
assert "response" not in result
assert "extracted_answer" not in result
mock_extract.assert_not_called()
mock_grade.assert_not_called()
else:
assert result["response"] == summary
mock_extract.assert_called_once()
mock_grade.assert_called_once()
@patch(f"{MODULE}.grade_single_result")
@patch(f"{MODULE}.extract_answer_from_response")
@patch(f"{MODULE}.quick_summary")
@patch(f"{MODULE}.format_query")
def test_successful_processing_with_grading(
self, mock_format, mock_summary, mock_extract, mock_grade
):
svc = _make_service()
mock_format.return_value = "formatted query"
mock_summary.return_value = {
"summary": "The answer is 4",
"sources": [{"url": "http://example.com"}],
}
mock_extract.return_value = {
"extracted_answer": "4",
"confidence": "95",
}
mock_grade.return_value = {
"is_correct": True,
"graded_confidence": "98",
"grader_response": "Correct answer",
}
mock_settings = MagicMock()
mock_settings.snapshot = {}
with patch(
f"{SETTINGS_CTX_MODULE}.get_settings_context",
return_value=mock_settings,
):
result = svc._process_benchmark_task(
self._make_task(),
{"iterations": 5},
{},
)
assert result["is_correct"] is True
assert result["extracted_answer"] == "4"
assert result["response"] == "The answer is 4"
@patch(f"{MODULE}.grade_single_result")
@patch(f"{MODULE}.extract_answer_from_response")
@patch(f"{MODULE}.quick_summary")
@patch(f"{MODULE}.format_query")
def test_grading_error_in_result(
self, mock_format, mock_summary, mock_extract, mock_grade
):
svc = _make_service()
mock_format.return_value = "q"
mock_summary.return_value = {"summary": "resp", "sources": []}
mock_extract.return_value = {
"extracted_answer": "ans",
"confidence": "50",
}
mock_grade.return_value = {"grading_error": "model unavailable"}
mock_settings = MagicMock()
mock_settings.snapshot = {}
with patch(
f"{SETTINGS_CTX_MODULE}.get_settings_context",
return_value=mock_settings,
):
result = svc._process_benchmark_task(self._make_task(), {}, {})
assert result["is_correct"] is None
assert "model unavailable" in result["grader_response"]
@patch(f"{MODULE}.grade_single_result")
@patch(f"{MODULE}.extract_answer_from_response")
@patch(f"{MODULE}.quick_summary")
@patch(f"{MODULE}.format_query")
def test_grading_returns_none(
self, mock_format, mock_summary, mock_extract, mock_grade
):
svc = _make_service()
mock_format.return_value = "q"
mock_summary.return_value = {"summary": "resp", "sources": []}
mock_extract.return_value = {"extracted_answer": "ans"}
mock_grade.return_value = None
mock_settings = MagicMock()
mock_settings.snapshot = {}
with patch(
f"{SETTINGS_CTX_MODULE}.get_settings_context",
return_value=mock_settings,
):
result = svc._process_benchmark_task(self._make_task(), {}, {})
assert result["is_correct"] is None
assert "No evaluation results returned" in result["grader_response"]
@patch(f"{MODULE}.grade_single_result")
@patch(f"{MODULE}.extract_answer_from_response")
@patch(f"{MODULE}.quick_summary")
@patch(f"{MODULE}.format_query")
def test_grading_exception(
self, mock_format, mock_summary, mock_extract, mock_grade
):
svc = _make_service()
mock_format.return_value = "q"
mock_summary.return_value = {"summary": "resp", "sources": []}
mock_extract.return_value = {"extracted_answer": "ans"}
mock_grade.side_effect = ValueError("grade fail")
mock_settings = MagicMock()
mock_settings.snapshot = {}
with patch(
f"{SETTINGS_CTX_MODULE}.get_settings_context",
return_value=mock_settings,
):
result = svc._process_benchmark_task(self._make_task(), {}, {})
assert result["is_correct"] is None
assert "grade fail" in result["grader_response"]
@patch(f"{MODULE}.format_query")
def test_research_error(self, mock_format):
svc = _make_service()
mock_format.side_effect = RuntimeError("research crashed")
mock_settings = MagicMock()
mock_settings.snapshot = {}
with patch(
f"{SETTINGS_CTX_MODULE}.get_settings_context",
return_value=mock_settings,
):
result = svc._process_benchmark_task(self._make_task(), {}, {})
assert "research_error" in result
assert "research crashed" in result["research_error"]
@patch(f"{MODULE}.grade_single_result")
@patch(f"{MODULE}.extract_answer_from_response")
@patch(f"{MODULE}.quick_summary")
@patch(f"{MODULE}.format_query")
def test_extract_returns_string(
self, mock_format, mock_summary, mock_extract, mock_grade
):
"""When extract_answer_from_response returns a string instead of dict."""
svc = _make_service()
mock_format.return_value = "q"
mock_summary.return_value = {"summary": "resp", "sources": []}
mock_extract.return_value = "plain string answer"
mock_grade.return_value = {
"is_correct": False,
"graded_confidence": "10",
"grader_response": "Wrong",
}
mock_settings = MagicMock()
mock_settings.snapshot = {}
with patch(
f"{SETTINGS_CTX_MODULE}.get_settings_context",
return_value=mock_settings,
):
result = svc._process_benchmark_task(self._make_task(), {}, {})
assert result["extracted_answer"] == "plain string answer"
assert result["confidence"] == "100"
@patch(f"{MODULE}.grade_single_result")
@patch(f"{MODULE}.extract_answer_from_response")
@patch(f"{MODULE}.quick_summary")
@patch(f"{MODULE}.format_query")
def test_sources_from_all_links(
self, mock_format, mock_summary, mock_extract, mock_grade
):
"""When sources is empty but all_links_of_system is present."""
svc = _make_service()
mock_format.return_value = "q"
mock_summary.return_value = {
"summary": "resp",
"sources": [],
"all_links_of_system": ["http://link1.com"],
}
mock_extract.return_value = {"extracted_answer": "ans"}
mock_grade.return_value = {
"is_correct": True,
"graded_confidence": "90",
"grader_response": "ok",
}
mock_settings = MagicMock()
mock_settings.snapshot = {}
with patch(
f"{SETTINGS_CTX_MODULE}.get_settings_context",
return_value=mock_settings,
):
result = svc._process_benchmark_task(self._make_task(), {}, {})
sources = json.loads(result["sources"])
assert "http://link1.com" in sources
# ============================================================
# _send_progress_update
# ============================================================
class TestSendProgressUpdate:
def test_sends_progress_via_socket(self):
mock_socket = MagicMock()
svc = _make_service(socket=mock_socket)
svc._send_progress_update(1, 5, 10, "user1")
mock_socket.emit_to_subscribers.assert_called_once()
call_args = mock_socket.emit_to_subscribers.call_args
assert call_args[0][0] == "research_progress"
assert call_args[0][2]["progress"] == 50.0
def test_sends_zero_progress_when_total_zero(self):
mock_socket = MagicMock()
svc = _make_service(socket=mock_socket)
svc._send_progress_update(1, 0, 0, "user1")
call_args = mock_socket.emit_to_subscribers.call_args
assert call_args[0][2]["progress"] == 0
def test_exception_in_send_is_caught(self):
mock_socket = MagicMock()
mock_socket.emit_to_subscribers.side_effect = RuntimeError(
"socket error"
)
svc = _make_service(socket=mock_socket)
# Should not raise
svc._send_progress_update(1, 5, 10, "user1")
# ============================================================
# sync_pending_results
# ============================================================
class TestSyncPendingResults:
def test_returns_zero_when_no_active_run(self):
svc = _make_service()
assert svc.sync_pending_results(999, "user1") == 0
def test_returns_zero_when_entry_deleted_during_toctou_window(self):
"""The worker thread's ``del self.active_runs[run_key]`` can land between
the membership check and the subscript in ``sync_pending_results``.
The read must not raise KeyError (which surfaces as a spurious 500 on
/api/results); a run that vanished has nothing pending. See #4859.
"""
class _VanishingRuns(dict):
# Present at the ``in`` check, gone at the read: models the
# concurrent delete firing inside the TOCTOU window.
def __contains__(self, key):
return True
svc = _make_service()
# dict.get on an empty mapping returns None while __contains__ lies True.
svc.active_runs = _VanishingRuns()
assert svc.sync_pending_results(123, "user1") == 0
def test_saves_new_results(self):
svc = _make_service()
svc.active_runs[("user1", 1)] = {
"data": {"username": "user1", "user_password": None},
"results": [
{
"example_id": "ex1",
"query_hash": "h1",
"dataset_type": "simpleqa",
"question": "Q?",
"correct_answer": "A",
"task_index": 0,
},
],
"result_persistence_failed": True,
}
mock_session = MagicMock()
# No rows persisted for this run yet
mock_session.query.return_value.filter.return_value.all.return_value = []
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
count = svc.sync_pending_results(1, "user1")
assert count == 1
mock_session.add.assert_called_once()
mock_session.commit.assert_called_once()
assert svc.get_result_persistence_error(1, "user1") is None
def test_skips_already_saved_indices(self):
svc = _make_service()
svc.active_runs[("user1", 1)] = {
"data": {"username": "user1", "user_password": None},
"results": [
{
"example_id": "ex1",
"query_hash": "h1",
"dataset_type": "simpleqa",
"question": "Q?",
"correct_answer": "A",
"task_index": 0,
},
],
"saved_indices": {0},
}
mock_session = MagicMock()
# No rows persisted for this run yet; idx 0 is skipped via saved_indices.
mock_session.query.return_value.filter.return_value.all.return_value = []
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
count = svc.sync_pending_results(1, "user1")
assert count == 0
def test_skips_existing_db_result(self):
svc = _make_service()
svc.active_runs[("user1", 1)] = {
"data": {"username": "user1", "user_password": None},
"results": [
{
"example_id": "ex1",
"query_hash": "h1",
"dataset_type": "simpleqa",
"question": "Q?",
"correct_answer": "A",
"task_index": 0,
},
],
}
mock_session = MagicMock()
# query_hash "h1" already persisted for this run
mock_session.query.return_value.filter.return_value.all.return_value = [
("h1",)
]
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
count = svc.sync_pending_results(1, "user1")
assert count == 0
mock_session.add.assert_not_called()
def test_handles_db_error(self):
svc = _make_service()
svc.active_runs[("user1", 1)] = {
"data": {"username": "user1", "user_password": None},
"results": [
{
"example_id": "ex1",
"query_hash": "h1",
"dataset_type": "simpleqa",
"question": "Q?",
"correct_answer": "A",
"task_index": 0,
},
],
}
mock_session = MagicMock()
mock_session.query.side_effect = RuntimeError("db error")
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
count = svc.sync_pending_results(1, "user1")
assert count == 0
error = svc.get_result_persistence_error(1, "user1")
assert error is not None
assert error["code"] == "database_write_failed"
assert "db error" not in error["message"]
def test_opens_the_db_for_the_caller_supplied_user(self):
"""``username`` is now an argument, not something recovered from the
run entry: ``active_runs`` is keyed by ``(username, run_id)``, so it
is needed to find the entry at all. Recovering it from the entry was
never safe anyway -- the entry a bare id found could belong to
another user (ADR-0009)."""
svc = _make_service()
svc.active_runs[("fromdata", 1)] = {
"data": {"username": "fromdata", "user_password": "pw"},
"results": [],
}
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_session = MagicMock()
mock_session.query.return_value.filter.return_value.all.return_value = []
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
# active_runs is keyed by (username, id): an internal caller must
# name the owner to reach the run; the stored password still comes
# from run_data.
svc.sync_pending_results(1, "fromdata")
mock_get_session.assert_called_once_with("fromdata", "pw")
def test_another_users_run_with_the_same_id_is_not_found(self):
"""Two users' first benchmark are both id 1. Asking as one user must
not reach the other's entry."""
svc = _make_service()
svc.active_runs[("alice", 1)] = {
"data": {"username": "alice", "user_password": "pw"},
"results": [{"id": "r1"}],
}
assert svc.sync_pending_results(1, "bob") == 0
def test_anonymous_caller_falls_back_to_recorded_owner(self):
"""When no username is passed (anonymous/internal caller), the run stored
under the normalized ``benchmark_user`` key is found and its recorded
owner drives the DB session -- exercising the ``if not username`` owner
fallback that the composite key otherwise makes unnecessary."""
svc = _make_service()
# An anonymous run is stored under the normalized owner "benchmark_user".
svc.active_runs[("benchmark_user", 1)] = {
"data": {"username": "benchmark_user", "user_password": "anon-pw"},
"results": [],
}
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_session = MagicMock()
mock_session.query.return_value.filter.return_value.all.return_value = []
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
# No username -> _run_key(None, 1) == ("benchmark_user", 1) hits the
# anonymous run, then the fallback derives the owner for the session.
svc.sync_pending_results(1)
mock_get_session.assert_called_once_with(
"benchmark_user", "anon-pw"
)
# ============================================================
# _sync_results_to_database
# ============================================================
class TestSyncResultsToDatabase:
def test_returns_early_if_no_active_run(self):
svc = _make_service()
# Should not raise
svc._sync_results_to_database("user1", 999)
def test_returns_early_if_thread_not_complete(self):
svc = _make_service()
svc.active_runs[("user1", 1)] = {"thread_complete": False}
svc._sync_results_to_database("user1", 1)
# No DB calls expected
def test_syncs_completed_run(self):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkStatus,
)
svc = _make_service()
svc.active_runs[("user1", 1)] = {
"thread_complete": True,
"data": {"username": "user1", "user_password": None},
"completion_info": {
"status": BenchmarkStatus.COMPLETED,
"end_time": datetime.now(UTC),
"completed_examples": 2,
"failed_examples": 0,
},
"results": [
{
"example_id": "ex1",
"query_hash": "h1",
"dataset_type": "simpleqa",
"question": "Q?",
"correct_answer": "A",
"is_correct": True,
"processing_time": 10.0,
"task_index": 0,
},
{
"example_id": "ex2",
"query_hash": "h2",
"dataset_type": "simpleqa",
"question": "Q2?",
"correct_answer": "A2",
"is_correct": False,
"processing_time": 20.0,
"task_index": 1,
},
],
}
mock_session = MagicMock()
mock_benchmark_run = MagicMock()
mock_benchmark_run.status = BenchmarkStatus.COMPLETED
mock_session.query.return_value.filter.return_value.first.return_value = mock_benchmark_run
# _persist_unsaved_results reads existing query_hashes for the run.
mock_session.query.return_value.filter.return_value.all.return_value = []
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
svc._sync_results_to_database("user1", 1)
mock_session.commit.assert_called_once()
# Active run should be cleaned up
assert ("user1", 1) not in svc.active_runs
def test_calculates_accuracy_correctly(self):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkStatus,
)
svc = _make_service()
svc.active_runs[("user1", 1)] = {
"thread_complete": True,
"data": {"username": "user1", "user_password": None},
"completion_info": {
"status": BenchmarkStatus.COMPLETED,
"completed_examples": 2,
"failed_examples": 0,
},
"results": [
{
"example_id": "ex1",
"query_hash": "h1",
"dataset_type": "simpleqa",
"question": "Q?",
"correct_answer": "A",
"is_correct": True,
"processing_time": 10.0,
"task_index": 0,
},
{
"example_id": "ex2",
"query_hash": "h2",
"dataset_type": "simpleqa",
"question": "Q2?",
"correct_answer": "A2",
"is_correct": True,
"processing_time": 20.0,
"task_index": 1,
},
],
}
mock_session = MagicMock()
mock_benchmark_run = MagicMock()
mock_benchmark_run.status = BenchmarkStatus.COMPLETED
mock_session.query.return_value.filter.return_value.first.return_value = mock_benchmark_run
# _persist_unsaved_results reads existing query_hashes for the run.
mock_session.query.return_value.filter.return_value.all.return_value = []
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
svc._sync_results_to_database("user1", 1)
# 2 correct out of 2 = 100%
assert mock_benchmark_run.overall_accuracy == 100.0
def test_skips_already_saved_results(self):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkStatus,
)
svc = _make_service()
svc.active_runs[("user1", 1)] = {
"thread_complete": True,
"data": {"username": "user1", "user_password": None},
"completion_info": {
"status": BenchmarkStatus.FAILED,
"error_message": "something failed",
},
"results": [
{
"example_id": "ex1",
"query_hash": "h1",
"dataset_type": "simpleqa",
"question": "Q?",
"correct_answer": "A",
"task_index": 0,
},
],
"saved_indices": {0},
}
mock_session = MagicMock()
mock_benchmark_run = MagicMock()
mock_session.query.return_value.filter.return_value.first.return_value = mock_benchmark_run
# _persist_unsaved_results reads existing query_hashes for the run.
mock_session.query.return_value.filter.return_value.all.return_value = []
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
svc._sync_results_to_database("user1", 1)
# Only the run status update, no result adds
assert mock_session.add.call_count == 0
def test_handles_db_exception(self):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkStatus,
)
svc = _make_service()
svc.active_runs[("user1", 1)] = {
"thread_complete": True,
"data": {"username": "user1", "user_password": None},
"completion_info": {
"status": BenchmarkStatus.COMPLETED,
},
"results": [],
}
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
side_effect=RuntimeError("db down")
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
# Should not raise
svc._sync_results_to_database("user1", 1)
error = svc.get_result_persistence_error(1, "user1")
assert error is not None
assert error["code"] == "database_write_failed"
def test_no_accuracy_for_non_completed_status(self):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkStatus,
)
svc = _make_service()
svc.active_runs[("user1", 1)] = {
"thread_complete": True,
"data": {"username": "user1", "user_password": None},
"completion_info": {
"status": BenchmarkStatus.FAILED,
"error_message": "crashed",
},
"results": [
{
"example_id": "ex1",
"query_hash": "h1",
"dataset_type": "simpleqa",
"question": "Q?",
"correct_answer": "A",
"is_correct": True,
"processing_time": 10.0,
"task_index": 0,
},
],
}
mock_session = MagicMock()
mock_benchmark_run = MagicMock()
mock_benchmark_run.status = BenchmarkStatus.FAILED
mock_session.query.return_value.filter.return_value.first.return_value = mock_benchmark_run
# _persist_unsaved_results reads existing query_hashes for the run.
mock_session.query.return_value.filter.return_value.all.return_value = []
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
svc._sync_results_to_database("user1", 1)
# Should NOT set overall_accuracy for failed runs
# (the code only calculates accuracy when status == COMPLETED)
# The mock starts with whatever default it has; we just verify
# the condition was checked by checking the status was set to FAILED
assert mock_benchmark_run.status == BenchmarkStatus.FAILED
# ============================================================
# _calculate_final_accuracy
# ============================================================
class TestCalculateFinalAccuracy:
def test_calculates_accuracy(self):
svc = _make_service()
mock_session = MagicMock()
mock_r1 = MagicMock()
mock_r1.is_correct = True
mock_r1.processing_time = 30.0
mock_r2 = MagicMock()
mock_r2.is_correct = False
mock_r2.processing_time = 60.0
mock_session.query.return_value.filter.return_value.filter.return_value.all.return_value = [
mock_r1,
mock_r2,
]
mock_run = MagicMock()
mock_session.query.return_value.filter.return_value.first.return_value = mock_run
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
svc._calculate_final_accuracy(1, "user1")
assert mock_run.overall_accuracy == 50.0
def test_no_results(self):
svc = _make_service()
mock_session = MagicMock()
mock_session.query.return_value.filter.return_value.filter.return_value.all.return_value = []
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
# Should not raise
svc._calculate_final_accuracy(1)
def test_handles_exception(self):
svc = _make_service()
mock_session = MagicMock()
mock_session.query.side_effect = RuntimeError("db error")
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
# Should not raise
svc._calculate_final_accuracy(1)
# ============================================================
# update_benchmark_status
# ============================================================
class TestUpdateBenchmarkStatus:
def test_update_status_in_progress(self):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkStatus,
)
svc = _make_service()
mock_session = MagicMock()
mock_run = MagicMock()
mock_run.start_time = None
mock_run.end_time = None
mock_session.query.return_value.filter.return_value.first.return_value = mock_run
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
svc.update_benchmark_status(1, BenchmarkStatus.IN_PROGRESS)
assert mock_run.status == BenchmarkStatus.IN_PROGRESS
assert mock_run.start_time is not None
def test_update_status_completed(self):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkStatus,
)
svc = _make_service()
mock_session = MagicMock()
mock_run = MagicMock()
mock_run.start_time = datetime.now(UTC)
mock_run.end_time = None
mock_session.query.return_value.filter.return_value.first.return_value = mock_run
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
svc.update_benchmark_status(1, BenchmarkStatus.COMPLETED)
assert mock_run.end_time is not None
def test_update_status_failed_sets_end_time(self):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkStatus,
)
svc = _make_service()
mock_session = MagicMock()
mock_run = MagicMock()
mock_run.start_time = datetime.now(UTC)
mock_run.end_time = None
mock_session.query.return_value.filter.return_value.first.return_value = mock_run
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
svc.update_benchmark_status(
1, BenchmarkStatus.FAILED, error_message="oops"
)
assert mock_run.end_time is not None
assert mock_run.error_message == "oops"
def test_update_status_not_found(self):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkStatus,
)
svc = _make_service()
mock_session = MagicMock()
mock_session.query.return_value.filter.return_value.first.return_value = None
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
# Should not raise
svc.update_benchmark_status(1, BenchmarkStatus.COMPLETED)
def test_update_status_db_error(self):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkStatus,
)
svc = _make_service()
mock_session = MagicMock()
mock_session.query.side_effect = RuntimeError("db error")
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
# Should not raise
svc.update_benchmark_status(1, BenchmarkStatus.COMPLETED)
mock_session.rollback.assert_called_once()
# ============================================================
# cancel_benchmark
# ============================================================
class TestCancelBenchmark:
def test_cancel_active_run(self):
svc = _make_service()
# Owned by the cancelling user: active_runs is keyed by (username, id),
# so cancel only ever finds — and stops — the in-memory run for the
# requesting owner.
svc.active_runs[("user1", 1)] = {
"status": "running",
"data": {"username": "user1"},
}
with patch.object(svc, "update_benchmark_status"):
result = svc.cancel_benchmark(1, "user1")
assert result is True
assert svc.active_runs[("user1", 1)]["status"] == "cancelled"
def test_cancel_nonexistent_run(self):
svc = _make_service()
with patch.object(svc, "update_benchmark_status"):
result = svc.cancel_benchmark(999, "user1")
assert result is True
def test_cancel_exception(self):
svc = _make_service()
with patch.object(
svc, "update_benchmark_status", side_effect=RuntimeError("err")
):
result = svc.cancel_benchmark(1)
assert result is False
# ============================================================
# get_benchmark_status
# ============================================================
class TestGetBenchmarkStatus:
def test_returns_none_when_not_found(self):
svc = _make_service()
mock_session = MagicMock()
mock_session.query.return_value.filter.return_value.first.return_value = None
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
result = svc.get_benchmark_status(999)
assert result is None
def test_returns_status_with_running_accuracy(self):
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkStatus,
)
svc = _make_service()
mock_session = MagicMock()
mock_run = MagicMock()
mock_run.id = 1
mock_run.run_name = "test run"
mock_run.status = BenchmarkStatus.IN_PROGRESS
mock_run.completed_examples = 2
mock_run.total_examples = 10
mock_run.failed_examples = 0
mock_run.overall_accuracy = None
mock_run.processing_rate = None
mock_run.config_hash = "abc123"
mock_run.created_at = datetime.now(UTC)
mock_run.start_time = datetime.now(UTC) - timedelta(seconds=60)
mock_run.end_time = None
mock_run.error_message = None
mock_r1 = MagicMock()
mock_r1.is_correct = True
mock_r1.dataset_type = MagicMock()
mock_r1.dataset_type.value = "simpleqa"
mock_r2 = MagicMock()
mock_r2.is_correct = False
mock_r2.dataset_type = MagicMock()
mock_r2.dataset_type.value = "simpleqa"
# Mock the chain of queries
# We need to handle multiple query() calls differently
call_count = {"n": 0}
def side_effect_query(*args):
call_count["n"] += 1
m = MagicMock()
if call_count["n"] == 1:
# benchmark_run query
m.filter.return_value.first.return_value = mock_run
elif call_count["n"] == 2:
# current_results query
m.filter.return_value.filter.return_value.all.return_value = [
mock_r1,
mock_r2,
]
elif call_count["n"] == 3:
# all_results_for_timing query
m.filter.return_value.all.return_value = [mock_r1, mock_r2]
return m
mock_session.query.side_effect = side_effect_query
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
result = svc.get_benchmark_status(1, "user1")
assert result is not None
assert result["running_accuracy"] == 50.0
assert "simpleqa_accuracy" in result
def test_handles_exception(self):
svc = _make_service()
mock_session = MagicMock()
mock_session.query.side_effect = RuntimeError("db error")
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
result = svc.get_benchmark_status(1)
assert result is None
# ============================================================
# _run_benchmark_thread
# ============================================================
class TestRunBenchmarkThread:
@patch(f"{MODULE}._global_research_semaphore")
@patch(f"{SETTINGS_CTX_MODULE}.set_settings_context")
def test_thread_runs_tasks(self, mock_set_ctx, mock_semaphore):
svc = _make_service()
svc.active_runs[("user1", 1)] = {
"data": {
"username": "user1",
"user_password": None,
"config_hash": "abc",
"datasets_config": {"simpleqa": {"count": 1}},
"search_config": {"iterations": 1},
"evaluation_config": {},
"settings_snapshot": {},
},
"results": [],
}
mock_task_result = {"example_id": "ex1", "is_correct": True}
with (
patch.object(
svc,
"_create_task_queue",
return_value=[
{
"benchmark_run_id": 1,
"example_id": "ex1",
"task_index": 0,
}
],
),
patch.object(
svc, "_process_benchmark_task", return_value=mock_task_result
),
patch.object(svc, "_send_progress_update"),
patch.object(svc, "_sync_results_to_database"),
):
svc._run_benchmark_thread("user1", 1)
assert len(svc.active_runs[("user1", 1)]["results"]) == 1
assert svc.active_runs[("user1", 1)]["thread_complete"] is True
@patch(f"{MODULE}._global_research_semaphore")
@patch(f"{SETTINGS_CTX_MODULE}.set_settings_context")
def test_thread_handles_cancelled_run(self, mock_set_ctx, mock_semaphore):
svc = _make_service()
svc.active_runs[("user1", 1)] = {
"data": {
"username": "user1",
"user_password": None,
"config_hash": "abc",
"datasets_config": {},
"search_config": {},
"evaluation_config": {},
"settings_snapshot": {},
},
"status": "cancelled",
"results": [],
}
with (
patch.object(
svc,
"_create_task_queue",
return_value=[
{
"benchmark_run_id": 1,
"example_id": "ex1",
"task_index": 0,
}
],
),
patch.object(svc, "_sync_results_to_database"),
):
svc._run_benchmark_thread("user1", 1)
info = svc.active_runs[("user1", 1)]["completion_info"]
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkStatus,
)
assert info["status"] == BenchmarkStatus.CANCELLED
@patch(f"{MODULE}._global_research_semaphore")
@patch(f"{SETTINGS_CTX_MODULE}.set_settings_context")
def test_thread_handles_task_error_rate_limit(
self, mock_set_ctx, mock_semaphore
):
svc = _make_service()
svc.active_runs[("user1", 1)] = {
"data": {
"username": "user1",
"user_password": None,
"config_hash": "abc",
"datasets_config": {},
"search_config": {},
"evaluation_config": {},
"settings_snapshot": {},
},
"results": [],
}
with (
patch.object(
svc,
"_create_task_queue",
return_value=[
{
"benchmark_run_id": 1,
"example_id": "ex1",
"task_index": 0,
}
],
),
patch.object(
svc,
"_process_benchmark_task",
side_effect=RuntimeError("403 Forbidden rate limit"),
),
patch.object(svc, "_sync_results_to_database"),
):
svc._run_benchmark_thread("user1", 1)
assert svc.rate_limit_detected == {("user1", 1): True}
@patch(f"{MODULE}._global_research_semaphore")
@patch(f"{SETTINGS_CTX_MODULE}.set_settings_context")
def test_thread_handles_missing_data(self, mock_set_ctx, mock_semaphore):
svc = _make_service()
svc.active_runs[("user1", 1)] = {} # No "data" key
with patch.object(svc, "_sync_results_to_database"):
svc._run_benchmark_thread("user1", 1)
info = svc.active_runs[("user1", 1)]["completion_info"]
from local_deep_research.benchmarks.web_api.benchmark_service import (
BenchmarkStatus,
)
assert info["status"] == BenchmarkStatus.FAILED
@patch(f"{MODULE}._global_research_semaphore")
@patch(f"{SETTINGS_CTX_MODULE}.set_settings_context")
def test_thread_settings_context_values(self, mock_set_ctx, mock_semaphore):
"""Test that SettingsContext correctly extracts values from setting objects."""
svc = _make_service()
svc.active_runs[("user1", 1)] = {
"data": {
"username": "user1",
"user_password": None,
"config_hash": "abc",
"datasets_config": {},
"search_config": {},
"evaluation_config": {},
"settings_snapshot": {
"key1": {"value": "val1"},
"key2": "direct_val",
},
},
"results": [],
}
captured_ctx = {}
def capture_ctx(ctx):
captured_ctx["ctx"] = ctx
mock_set_ctx.side_effect = capture_ctx
with (
patch.object(svc, "_create_task_queue", return_value=[]),
patch.object(svc, "_sync_results_to_database"),
):
svc._run_benchmark_thread("user1", 1)
ctx = captured_ctx["ctx"]
assert ctx.get_setting("key1") == "val1"
assert ctx.get_setting("key2") == "direct_val"
assert ctx.get_setting("nonexistent", "default") == "default"
@patch(f"{MODULE}._global_research_semaphore")
@patch(f"{SETTINGS_CTX_MODULE}.set_settings_context")
def test_thread_total_zero_examples_progress(
self, mock_set_ctx, mock_semaphore
):
"""Cover branch where total_examples is 0 for progress calculation."""
svc = _make_service()
svc.active_runs[("user1", 1)] = {
"data": {
"username": "user1",
"user_password": None,
"config_hash": "abc",
"datasets_config": {},
"search_config": {},
"evaluation_config": {},
"settings_snapshot": {},
},
"results": [],
}
with (
patch.object(svc, "_create_task_queue", return_value=[]),
patch.object(svc, "_sync_results_to_database"),
):
svc._run_benchmark_thread("user1", 1)
# Completion should still happen (progress = 0 since total = 0)
assert svc.active_runs[("user1", 1)]["thread_complete"] is True
# ============================================================
# start_benchmark
# ============================================================
class TestStartBenchmark:
def test_start_benchmark_run_not_found(self):
svc = _make_service()
mock_session = MagicMock()
mock_session.query.return_value.filter.return_value.first.return_value = None
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as mock_get_session:
mock_get_session.return_value.__enter__ = Mock(
return_value=mock_session
)
mock_get_session.return_value.__exit__ = Mock(return_value=False)
result = svc.start_benchmark(999, "user1")
assert result is False
# ============================================================
# Progress callback in _process_benchmark_task
# ============================================================
class TestProgressCallback:
@patch(f"{MODULE}.grade_single_result")
@patch(f"{MODULE}.extract_answer_from_response")
@patch(f"{MODULE}.quick_summary")
@patch(f"{MODULE}.format_query")
def test_callback_milestone_types(
self, mock_format, mock_summary, mock_extract, mock_grade
):
"""Test that the progress callback categorizes log types correctly."""
mock_socket = MagicMock()
svc = _make_service(socket=mock_socket)
def capture_quick_summary(**kwargs):
cb = kwargs.get("progress_callback")
if cb:
# Test various status strings
cb("Starting phase", 10, {"phase": "init"})
cb("Completed search", 50, {})
cb("Error occurred", 80, {})
cb("Processing data", 60, {})
return {"summary": "result", "sources": []}
mock_format.return_value = "q"
mock_summary.side_effect = capture_quick_summary
mock_extract.return_value = {"extracted_answer": "ans"}
mock_grade.return_value = {
"is_correct": True,
"graded_confidence": "90",
"grader_response": "ok",
}
mock_settings = MagicMock()
mock_settings.snapshot = {}
task = {
"benchmark_run_id": 1,
"example_id": "ex1",
"dataset_type": "simpleqa",
"question": "Q?",
"correct_answer": "A",
"query_hash": "h1",
"task_index": 0,
"username": "user1",
"user_password": None,
}
with patch(
f"{SETTINGS_CTX_MODULE}.get_settings_context",
return_value=mock_settings,
):
svc._process_benchmark_task(task, {}, {})
# Verify socket was called multiple times (from callback + regular calls)
assert mock_socket.emit_to_subscribers.call_count >= 4
@patch(f"{MODULE}.grade_single_result")
@patch(f"{MODULE}.extract_answer_from_response")
@patch(f"{MODULE}.quick_summary")
@patch(f"{MODULE}.format_query")
def test_callback_handles_exception(
self, mock_format, mock_summary, mock_extract, mock_grade
):
"""Test that exceptions in the progress callback are caught."""
mock_socket = MagicMock()
mock_socket.emit_to_subscribers.side_effect = RuntimeError(
"socket dead"
)
svc = _make_service(socket=mock_socket)
def trigger_callback(**kwargs):
cb = kwargs.get("progress_callback")
if cb:
cb("test status", 50, {}) # Should not raise
return {"summary": "result", "sources": []}
mock_format.return_value = "q"
mock_summary.side_effect = trigger_callback
mock_extract.return_value = {"extracted_answer": "ans"}
mock_grade.return_value = {
"is_correct": True,
"graded_confidence": "90",
"grader_response": "ok",
}
mock_settings = MagicMock()
mock_settings.snapshot = {}
task = {
"benchmark_run_id": 1,
"example_id": "ex1",
"dataset_type": "simpleqa",
"question": "Q?",
"correct_answer": "A",
"query_hash": "h1",
"task_index": 0,
"username": "user1",
"user_password": None,
}
with patch(
f"{SETTINGS_CTX_MODULE}.get_settings_context",
return_value=mock_settings,
):
# Should not raise despite socket error in callback
result = svc._process_benchmark_task(task, {}, {})
assert result is not None