Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

9.7 KB
Raw Blame
"""Tests for the metadata block added to /api/results/<id>/export.

Pins the contract introduced by the migration-0014 PR:
- Response includes a top-level ``metadata`` dict with ``ldr_version``,
  ``started_at`` (ISO-8601), and ``settings_snapshot``.
- Pre-0014 rows return null for the new fields without crashing the JS.
- ``success: True`` envelope is preserved (5 JS callsites depend on it —
  see benchmark_results.html lines 95, 348, 556, 661, 722).
- ``started_at`` falls back to ``created_at`` when ``start_time`` is null
  (e.g. a run that was created but never reached IN_PROGRESS).
- ``settings_snapshot`` round-trips as a JSON object — secrets are
  expected to already be redacted at insert time, but this endpoint
  doesn't re-redact, so we assert the round-trip preserves the dict.

Post-FastAPI-migration the benchmark export route lives on the FastAPI
router (``web.routers.benchmark``) and authenticates via the
``require_auth`` dependency. The old Flask helpers (``_make_app`` /
``_patch_auth_and_db`` registered a Flask blueprint) were removed with the
Flask benchmark blueprint, so this drives the endpoint through the FastAPI
TestClient with ``require_auth`` overridden and ``get_user_db_session``
patched to a mock session whose ``.query`` is routed by model class.
"""

import enum
from contextlib import contextmanager
from datetime import datetime, UTC
from unittest.mock import MagicMock, patch

import pytest
from fastapi.testclient import TestClient


# ---------------------------------------------------------------------------
# Helpers (FastAPI ports of the ones that lived in the now-deleted
# test_benchmark_routes_coverage.py Flask suite).
# ---------------------------------------------------------------------------


class _FakeDatasetType(enum.Enum):
    SIMPLEQA = "simpleqa"
    BROWSECOMP = "browsecomp"


def _make_app():
    """Return the real FastAPI app with ``require_auth`` overridden so the
    route body executes as ``testuser`` without a real login/DB."""
    from local_deep_research.web.fastapi_app import app
    from local_deep_research.web.dependencies.auth import require_auth

    app.dependency_overrides[require_auth] = lambda: "testuser"
    return app


@contextmanager
def _patch_auth_and_db():
    """Patch ``get_user_db_session`` (imported locally inside the export
    handler from ``database.session_context``) so the route body runs against
    a mock session. Yields ``(mock_service, mock_settings_mgr, mock_db)`` to
    mirror the old Flask helper's signature; only ``mock_db`` is used here."""
    mock_db = MagicMock()

    @contextmanager
    def _session_ctx(username, *args, **kwargs):
        yield mock_db

    with (
        patch(
            "local_deep_research.database.session_context.get_user_db_session",
            side_effect=_session_ctx,
        ),
        patch(
            "local_deep_research.web.routers.benchmark.benchmark_service"
        ) as mock_svc,
    ):
        yield mock_svc, MagicMock(), mock_db


def _make_export_query_router(*, run=None, results=None):
    """Side-effect for ``mock_db.query`` that routes by model class for the
    /export endpoint.

    The endpoint runs two queries:
      1. session.query(BenchmarkRun).options(...).filter(...).one_or_none()
      2. session.query(BenchmarkResult).options(...).filter(...).order_by(...).all()
    """

    def _route(model, *args):
        chain = MagicMock()
        chain.options.return_value = chain
        chain.filter.return_value = chain
        chain.order_by.return_value = chain
        name = getattr(model, "__name__", "")
        if "BenchmarkRun" in name:
            chain.one_or_none.return_value = run
        else:
            chain.all.return_value = results or []
        return chain

    return _route


def _make_run_mock(
    *, ldr_version=None, start_time=None, settings_snapshot=None
):
    run = MagicMock()
    run.ldr_version = ldr_version
    run.start_time = start_time
    run.created_at = datetime(2025, 1, 1, tzinfo=UTC)
    run.settings_snapshot = settings_snapshot
    return run


@pytest.fixture(autouse=True)
def _clear_overrides():
    """Ensure the ``require_auth`` override added by ``_make_app`` doesn't
    leak into other tests sharing the module-level FastAPI app."""
    yield
    from local_deep_research.web.fastapi_app import app
    from local_deep_research.web.dependencies.auth import require_auth

    app.dependency_overrides.pop(require_auth, None)


def _client(app):
    return TestClient(app, raise_server_exceptions=False)


def _result_row():
    r = MagicMock()
    r.example_id = "ex0"
    r.dataset_type = _FakeDatasetType.SIMPLEQA
    r.question = "Q?"
    r.correct_answer = "A"
    r.extracted_answer = "A"
    r.is_correct = True
    r.confidence = 0.9
    r.processing_time = 5.0
    r.completed_at = datetime(2025, 1, 1, tzinfo=UTC)
    return r


def test_metadata_includes_ldr_version():
    app = _make_app()
    with _patch_auth_and_db() as (_svc, _mgr, mock_db):
        mock_db.query.side_effect = _make_export_query_router(
            run=_make_run_mock(ldr_version="1.6.10"),
            results=[_result_row()],
        )
        resp = _client(app).get("/benchmark/api/results/1/export")
        assert resp.status_code == 200
        body = resp.json()
        assert body["metadata"]["ldr_version"] == "1.6.10"


def test_metadata_started_at_uses_start_time_when_set():
    app = _make_app()
    start_dt = datetime(2026, 5, 1, 12, 30, 0, tzinfo=UTC)
    with _patch_auth_and_db() as (_svc, _mgr, mock_db):
        mock_db.query.side_effect = _make_export_query_router(
            run=_make_run_mock(start_time=start_dt),
            results=[_result_row()],
        )
        resp = _client(app).get("/benchmark/api/results/1/export")
        body = resp.json()
        assert body["metadata"]["started_at"] == start_dt.isoformat()


def test_metadata_started_at_falls_back_to_created_at():
    """If a run was created but never started, started_at falls back to
    created_at so the YAML still has a date_tested anchor."""
    app = _make_app()
    run = _make_run_mock()  # start_time defaults to None
    with _patch_auth_and_db() as (_svc, _mgr, mock_db):
        mock_db.query.side_effect = _make_export_query_router(
            run=run, results=[_result_row()]
        )
        resp = _client(app).get("/benchmark/api/results/1/export")
        body = resp.json()
        assert body["metadata"]["started_at"] == run.created_at.isoformat()


def test_metadata_settings_snapshot_round_trip():
    """The snapshot is returned only when opted in via ?include_settings=1."""
    app = _make_app()
    snapshot = {
        "llm.model": {"value": "qwen3.6:27b", "ui_element": "select"},
        "search.fetch.mode": {
            "value": "summary_focus_query",
            "ui_element": "select",
        },
    }
    with _patch_auth_and_db() as (_svc, _mgr, mock_db):
        mock_db.query.side_effect = _make_export_query_router(
            run=_make_run_mock(settings_snapshot=snapshot),
            results=[_result_row()],
        )
        resp = _client(app).get(
            "/benchmark/api/results/1/export?include_settings=1"
        )
        body = resp.json()
        assert body["metadata"]["settings_snapshot"] == snapshot


def test_metadata_settings_snapshot_omitted_by_default():
    """Without the opt-in flag, the snapshot is NOT transferred — the
    default summary download stays lean and doesn't ship the full config."""
    app = _make_app()
    snapshot = {"llm.model": {"value": "qwen3.6:27b", "ui_element": "select"}}
    with _patch_auth_and_db() as (_svc, _mgr, mock_db):
        mock_db.query.side_effect = _make_export_query_router(
            run=_make_run_mock(settings_snapshot=snapshot),
            results=[_result_row()],
        )
        resp = _client(app).get("/benchmark/api/results/1/export")
        body = resp.json()
        assert body["metadata"]["settings_snapshot"] is None


def test_pre_0014_row_returns_null_metadata_fields():
    """A row with all new columns NULL must serialize as JSON null, not
    crash, and not omit the metadata block."""
    app = _make_app()
    run = _make_run_mock()  # all provenance fields default to None
    run.created_at = None  # full pre-0014 — even created_at unset
    with _patch_auth_and_db() as (_svc, _mgr, mock_db):
        mock_db.query.side_effect = _make_export_query_router(
            run=run, results=[]
        )
        resp = _client(app).get("/benchmark/api/results/1/export")
        body = resp.json()
        assert body["metadata"] == {
            "ldr_version": None,
            "started_at": None,
            "settings_snapshot": None,
        }


def test_run_not_found_returns_null_metadata():
    """Missing run still returns a metadata block (all None) so the JS
    code path doesn't need a separate not-found branch."""
    app = _make_app()
    with _patch_auth_and_db() as (_svc, _mgr, mock_db):
        mock_db.query.side_effect = _make_export_query_router(
            run=None, results=[]
        )
        resp = _client(app).get("/benchmark/api/results/1/export")
        assert resp.status_code == 200
        body = resp.json()
        assert body["success"] is True
        assert body["metadata"]["ldr_version"] is None
        assert body["metadata"]["settings_snapshot"] is None
        assert body["results"] == []


def test_success_field_preserved_for_back_compat():
    """5 JS callsites in benchmark_results.html check `data.success` —
    removing it would silently break the UI."""
    app = _make_app()
    with _patch_auth_and_db() as (_svc, _mgr, mock_db):
        mock_db.query.side_effect = _make_export_query_router(
            run=_make_run_mock(ldr_version="1.6.10"),
            results=[_result_row()],
        )
        resp = _client(app).get("/benchmark/api/results/1/export")
        body = resp.json()
        assert body["success"] is True