~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Tests for the metadata block added to /api/results/<id>/export.
Pins the contract introduced by the migration-0014 PR:
- Response includes a top-level ``metadata`` dict with ``ldr_version``,
``started_at`` (ISO-8601), and ``settings_snapshot``.
- Pre-0014 rows return null for the new fields without crashing the JS.
- ``success: True`` envelope is preserved (5 JS callsites depend on it —
see benchmark_results.html lines 95, 348, 556, 661, 722).
- ``started_at`` falls back to ``created_at`` when ``start_time`` is null
(e.g. a run that was created but never reached IN_PROGRESS).
- ``settings_snapshot`` round-trips as a JSON object — secrets are
expected to already be redacted at insert time, but this endpoint
doesn't re-redact, so we assert the round-trip preserves the dict.
Post-FastAPI-migration the benchmark export route lives on the FastAPI
router (``web.routers.benchmark``) and authenticates via the
``require_auth`` dependency. The old Flask helpers (``_make_app`` /
``_patch_auth_and_db`` registered a Flask blueprint) were removed with the
Flask benchmark blueprint, so this drives the endpoint through the FastAPI
TestClient with ``require_auth`` overridden and ``get_user_db_session``
patched to a mock session whose ``.query`` is routed by model class.
"""
import enum
from contextlib import contextmanager
from datetime import datetime, UTC
from unittest.mock import MagicMock, patch
import pytest
from fastapi.testclient import TestClient
# ---------------------------------------------------------------------------
# Helpers (FastAPI ports of the ones that lived in the now-deleted
# test_benchmark_routes_coverage.py Flask suite).
# ---------------------------------------------------------------------------
class _FakeDatasetType(enum.Enum):
SIMPLEQA = "simpleqa"
BROWSECOMP = "browsecomp"
def _make_app():
"""Return the real FastAPI app with ``require_auth`` overridden so the
route body executes as ``testuser`` without a real login/DB."""
from local_deep_research.web.fastapi_app import app
from local_deep_research.web.dependencies.auth import require_auth
app.dependency_overrides[require_auth] = lambda: "testuser"
return app
@contextmanager
def _patch_auth_and_db():
"""Patch ``get_user_db_session`` (imported locally inside the export
handler from ``database.session_context``) so the route body runs against
a mock session. Yields ``(mock_service, mock_settings_mgr, mock_db)`` to
mirror the old Flask helper's signature; only ``mock_db`` is used here."""
mock_db = MagicMock()
@contextmanager
def _session_ctx(username, *args, **kwargs):
yield mock_db
with (
patch(
"local_deep_research.database.session_context.get_user_db_session",
side_effect=_session_ctx,
),
patch(
"local_deep_research.web.routers.benchmark.benchmark_service"
) as mock_svc,
):
yield mock_svc, MagicMock(), mock_db
def _make_export_query_router(*, run=None, results=None):
"""Side-effect for ``mock_db.query`` that routes by model class for the
/export endpoint.
The endpoint runs two queries:
1. session.query(BenchmarkRun).options(...).filter(...).one_or_none()
2. session.query(BenchmarkResult).options(...).filter(...).order_by(...).all()
"""
def _route(model, *args):
chain = MagicMock()
chain.options.return_value = chain
chain.filter.return_value = chain
chain.order_by.return_value = chain
name = getattr(model, "__name__", "")
if "BenchmarkRun" in name:
chain.one_or_none.return_value = run
else:
chain.all.return_value = results or []
return chain
return _route
def _make_run_mock(
*, ldr_version=None, start_time=None, settings_snapshot=None
):
run = MagicMock()
run.ldr_version = ldr_version
run.start_time = start_time
run.created_at = datetime(2025, 1, 1, tzinfo=UTC)
run.settings_snapshot = settings_snapshot
return run
@pytest.fixture(autouse=True)
def _clear_overrides():
"""Ensure the ``require_auth`` override added by ``_make_app`` doesn't
leak into other tests sharing the module-level FastAPI app."""
yield
from local_deep_research.web.fastapi_app import app
from local_deep_research.web.dependencies.auth import require_auth
app.dependency_overrides.pop(require_auth, None)
def _client(app):
return TestClient(app, raise_server_exceptions=False)
def _result_row():
r = MagicMock()
r.example_id = "ex0"
r.dataset_type = _FakeDatasetType.SIMPLEQA
r.question = "Q?"
r.correct_answer = "A"
r.extracted_answer = "A"
r.is_correct = True
r.confidence = 0.9
r.processing_time = 5.0
r.completed_at = datetime(2025, 1, 1, tzinfo=UTC)
return r
def test_metadata_includes_ldr_version():
app = _make_app()
with _patch_auth_and_db() as (_svc, _mgr, mock_db):
mock_db.query.side_effect = _make_export_query_router(
run=_make_run_mock(ldr_version="1.6.10"),
results=[_result_row()],
)
resp = _client(app).get("/benchmark/api/results/1/export")
assert resp.status_code == 200
body = resp.json()
assert body["metadata"]["ldr_version"] == "1.6.10"
def test_metadata_started_at_uses_start_time_when_set():
app = _make_app()
start_dt = datetime(2026, 5, 1, 12, 30, 0, tzinfo=UTC)
with _patch_auth_and_db() as (_svc, _mgr, mock_db):
mock_db.query.side_effect = _make_export_query_router(
run=_make_run_mock(start_time=start_dt),
results=[_result_row()],
)
resp = _client(app).get("/benchmark/api/results/1/export")
body = resp.json()
assert body["metadata"]["started_at"] == start_dt.isoformat()
def test_metadata_started_at_falls_back_to_created_at():
"""If a run was created but never started, started_at falls back to
created_at so the YAML still has a date_tested anchor."""
app = _make_app()
run = _make_run_mock() # start_time defaults to None
with _patch_auth_and_db() as (_svc, _mgr, mock_db):
mock_db.query.side_effect = _make_export_query_router(
run=run, results=[_result_row()]
)
resp = _client(app).get("/benchmark/api/results/1/export")
body = resp.json()
assert body["metadata"]["started_at"] == run.created_at.isoformat()
def test_metadata_settings_snapshot_round_trip():
"""The snapshot is returned only when opted in via ?include_settings=1."""
app = _make_app()
snapshot = {
"llm.model": {"value": "qwen3.6:27b", "ui_element": "select"},
"search.fetch.mode": {
"value": "summary_focus_query",
"ui_element": "select",
},
}
with _patch_auth_and_db() as (_svc, _mgr, mock_db):
mock_db.query.side_effect = _make_export_query_router(
run=_make_run_mock(settings_snapshot=snapshot),
results=[_result_row()],
)
resp = _client(app).get(
"/benchmark/api/results/1/export?include_settings=1"
)
body = resp.json()
assert body["metadata"]["settings_snapshot"] == snapshot
def test_metadata_settings_snapshot_omitted_by_default():
"""Without the opt-in flag, the snapshot is NOT transferred — the
default summary download stays lean and doesn't ship the full config."""
app = _make_app()
snapshot = {"llm.model": {"value": "qwen3.6:27b", "ui_element": "select"}}
with _patch_auth_and_db() as (_svc, _mgr, mock_db):
mock_db.query.side_effect = _make_export_query_router(
run=_make_run_mock(settings_snapshot=snapshot),
results=[_result_row()],
)
resp = _client(app).get("/benchmark/api/results/1/export")
body = resp.json()
assert body["metadata"]["settings_snapshot"] is None
def test_pre_0014_row_returns_null_metadata_fields():
"""A row with all new columns NULL must serialize as JSON null, not
crash, and not omit the metadata block."""
app = _make_app()
run = _make_run_mock() # all provenance fields default to None
run.created_at = None # full pre-0014 — even created_at unset
with _patch_auth_and_db() as (_svc, _mgr, mock_db):
mock_db.query.side_effect = _make_export_query_router(
run=run, results=[]
)
resp = _client(app).get("/benchmark/api/results/1/export")
body = resp.json()
assert body["metadata"] == {
"ldr_version": None,
"started_at": None,
"settings_snapshot": None,
}
def test_run_not_found_returns_null_metadata():
"""Missing run still returns a metadata block (all None) so the JS
code path doesn't need a separate not-found branch."""
app = _make_app()
with _patch_auth_and_db() as (_svc, _mgr, mock_db):
mock_db.query.side_effect = _make_export_query_router(
run=None, results=[]
)
resp = _client(app).get("/benchmark/api/results/1/export")
assert resp.status_code == 200
body = resp.json()
assert body["success"] is True
assert body["metadata"]["ldr_version"] is None
assert body["metadata"]["settings_snapshot"] is None
assert body["results"] == []
def test_success_field_preserved_for_back_compat():
"""5 JS callsites in benchmark_results.html check `data.success` —
removing it would silently break the UI."""
app = _make_app()
with _patch_auth_and_db() as (_svc, _mgr, mock_db):
mock_db.query.side_effect = _make_export_query_router(
run=_make_run_mock(ldr_version="1.6.10"),
results=[_result_row()],
)
resp = _client(app).get("/benchmark/api/results/1/export")
body = resp.json()
assert body["success"] is True