Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

20.1 KB
Raw Blame
"""Contract tests for the context-overflow API (FastAPI migration).

The context-overflow endpoints moved from the Flask-era ``/metrics/api/*``
guesses to the un-prefixed FastAPI router ``routers/context_overflow_api.py``:

* ``GET /api/context-overflow`` — analytics dashboard payload
  (fetched by ``static/js/components/context-overflow.js``)
* ``GET /api/research/{research_id}/context-overflow`` — per-research
  overflow check (fetched by ``progress.js``, ``details.js``, ``results.js``)

plus the diagnostics page ``GET /metrics/context-overflow`` declared in
``static/js/config/urls.js`` (``PAGES.METRICS_CONTEXT_OVERFLOW``).

These tests fence:

1. Every context-overflow URL the frontend declares (urls.js) or hardcodes
   in a ``fetch(...)`` call resolves to a registered GET route — the JS
   files are parsed so a path drift on either side fails the test.
   (tests/infrastructure_tests/test_urls_js.py checks a generic list of
   critical URLs which does NOT include the context-overflow entries.)
2. Both API endpoints require authentication (401 when anonymous).
3. The response shape keys the frontend destructures/reads, driven through
   the routes' REAL SQL against a seeded in-memory SQLite DB.
"""

import re
from contextlib import contextmanager
from datetime import UTC, datetime, timedelta
from pathlib import Path
from unittest.mock import patch

import pytest
from fastapi.routing import APIRoute
from fastapi.testclient import TestClient
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import StaticPool

from local_deep_research.database.models import Base
from local_deep_research.database.models.metrics import TokenUsage

OVERVIEW_URL = "/api/context-overflow"
RESEARCH_URL = "/api/research/{research_id}/context-overflow"

_REPO_ROOT = Path(__file__).parents[3]
_STATIC_JS = (
    _REPO_ROOT / "src" / "local_deep_research" / "web" / "static" / "js"
)
_URLS_JS = _STATIC_JS / "config" / "urls.js"


# ---------------------------------------------------------------------------
# Helpers: route table + JS parsing
# ---------------------------------------------------------------------------


def _app():
    from local_deep_research.web.fastapi_app import app

    return app


def _api_routes() -> list[tuple[str, frozenset]]:
    """(path, methods) for every APIRoute on the live app."""
    return [
        (r.path, frozenset(r.methods))
        for r in _app().routes
        if isinstance(r, APIRoute)
    ]


def _paths_match(js_path: str, app_path: str) -> bool:
    """Segment-wise match; any ``{param}`` segment matches anything."""
    js_parts = js_path.strip("/").split("/")
    app_parts = app_path.strip("/").split("/")
    if len(js_parts) != len(app_parts):
        return False
    for js_part, app_part in zip(js_parts, app_parts):
        if "{" in js_part or "{" in app_part:
            continue
        if js_part != app_part:
            return False
    return True


def _assert_get_route_exists(js_path: str, origin: str) -> None:
    routes = _api_routes()
    matches = [
        (path, methods)
        for path, methods in routes
        if _paths_match(js_path, path)
    ]
    assert matches, (
        f"Frontend path {js_path!r} (from {origin}) has no matching route "
        f"in the FastAPI route table"
    )
    assert any("GET" in methods for _, methods in matches), (
        f"Frontend path {js_path!r} (from {origin}) matched "
        f"{[m[0] for m in matches]} but none accepts GET"
    )


def _urls_js_context_overflow_entries() -> list[tuple[str, str]]:
    """(KEY, path) pairs in urls.js whose path mentions context-overflow."""
    content = _URLS_JS.read_text()
    return [
        (key, url)
        for key, url in re.findall(r"(\w+):\s*'([^']+)'", content)
        if "context-overflow" in url
    ]


def _normalize_fetch_path(raw: str) -> str:
    """Template-literal fetch path -> route-table comparable path."""
    path = raw.split("?", 1)[0]
    return re.sub(r"\$\{[^}]*\}", "{param}", path)


def _frontend_fetch_paths() -> dict[str, str]:
    """Every context-overflow path fetched by the JS components.

    Returns {normalized_path: 'file:rawpath'} scanning both template
    literals and plain-quoted fetch arguments.
    """
    found: dict[str, str] = {}
    fetch_re = re.compile(r"""fetch\(\s*(?:`([^`]+)`|'([^']+)'|"([^"]+)")""")
    for js_file in sorted(_STATIC_JS.rglob("*.js")):
        content = js_file.read_text()
        for match in fetch_re.finditer(content):
            raw = next(g for g in match.groups() if g is not None)
            if "context-overflow" not in raw:
                continue
            found[_normalize_fetch_path(raw)] = f"{js_file.name}:{raw}"
    return found


# ---------------------------------------------------------------------------
# Fixtures: authenticated client + seeded per-user DB
# ---------------------------------------------------------------------------


@pytest.fixture
def auth_client():
    """TestClient authenticated as ``testuser`` via dependency override.

    Overriding ``require_auth`` (instead of register/login) lets the seeded
    in-memory DB below stand in for the per-user encrypted DB.
    """
    from local_deep_research.web.dependencies.auth import require_auth

    app = _app()
    app.dependency_overrides[require_auth] = lambda: "testuser"
    try:
        yield TestClient(app)
    finally:
        app.dependency_overrides.pop(require_auth, None)


@contextmanager
def _seeded_db(*rows):
    """Patch the router's ``get_user_db_session`` with a seeded SQLite DB.

    StaticPool shares the single in-memory connection between the seeding
    session and the route handler (which runs in the TestClient
    threadpool), so the routes execute their REAL aggregation SQL.
    """
    engine = create_engine(
        "sqlite://",
        connect_args={"check_same_thread": False},
        poolclass=StaticPool,
    )
    Base.metadata.create_all(engine)
    Session = sessionmaker(bind=engine)

    seed = Session()
    seed.add_all(rows)
    seed.commit()
    seed.close()

    @contextmanager
    def _ctx(username=None, password=None, **kwargs):
        with Session() as session:
            yield session

    with patch(
        "local_deep_research.web.routers.context_overflow_api"
        ".get_user_db_session",
        _ctx,
    ):
        yield


_NOW = datetime.now(UTC)


def _usage(**overrides) -> TokenUsage:
    defaults = dict(
        research_id="res-1",
        timestamp=_NOW - timedelta(hours=1),
        model_provider="ollama",
        model_name="llama3",
        prompt_tokens=2000,
        completion_tokens=300,
        total_tokens=2300,
    )
    defaults.update(overrides)
    return TokenUsage(**defaults)


def _seed_rows() -> tuple[TokenUsage, TokenUsage, TokenUsage]:
    """Three rows: one truncated, one clean-with-context, one context-less."""
    truncated = _usage(
        timestamp=_NOW - timedelta(minutes=5),
        prompt_tokens=9000,
        completion_tokens=500,
        total_tokens=9500,
        context_limit=8192,
        context_truncated=True,
        tokens_truncated=1200,
        truncation_ratio=0.125,
        ollama_prompt_eval_count=8000,
        research_phase="synthesis",
        response_time_ms=1500,
        research_query="what is overflow",
    )
    clean = _usage(
        timestamp=_NOW - timedelta(hours=2),
        context_limit=8192,
        context_truncated=False,
        research_phase="search",
        response_time_ms=800,
    )
    no_context = _usage(
        research_id="res-2",
        timestamp=_NOW - timedelta(hours=3),
        model_provider="openai",
        model_name="gpt-4o-mini",
        prompt_tokens=1000,
        completion_tokens=200,
        total_tokens=1200,
    )
    return truncated, clean, no_context


# ---------------------------------------------------------------------------
# 1. Route registration matches what the frontend declares
# ---------------------------------------------------------------------------


class TestRouteRegistration:
    def test_canonical_api_paths_registered_with_get(self):
        """The two migrated endpoints exist verbatim in the route table."""
        route_map = {path: methods for path, methods in _api_routes()}
        for path in (OVERVIEW_URL, RESEARCH_URL):
            assert path in route_map, (
                f"{path} missing from FastAPI route table — the "
                f"context-overflow router moved or lost its route"
            )
            assert "GET" in route_map[path], (
                f"{path} registered without GET: {route_map[path]}"
            )

    def test_urls_js_context_overflow_entries_resolve(self):
        """Every context-overflow URL declared in urls.js is a GET route."""
        entries = _urls_js_context_overflow_entries()
        assert entries, (
            f"No context-overflow entries found in {_URLS_JS} — if the key "
            f"was renamed/removed, update this test's parser"
        )
        # The diagnostics page must stay among them: results.js/progress.js
        # link users to overflow details, and this entry is not covered by
        # test_urls_js.py's critical-mapping list.
        assert any(url == "/metrics/context-overflow" for _, url in entries), (
            f"PAGES.METRICS_CONTEXT_OVERFLOW drifted: {entries}"
        )
        for key, url in entries:
            _assert_get_route_exists(url, origin=f"urls.js:{key}")

    def test_component_fetch_paths_resolve(self):
        """Every fetch('.../context-overflow...') in the JS components hits
        a registered GET route.

        This is the regression the migration actually fixed: the Flask-era
        frontend fetched /metrics/api/... which 404'd silently
        (``if (!response.ok) return``), so a path drift never surfaces in
        the UI — only a test can catch it.
        """
        fetched = _frontend_fetch_paths()
        assert fetched, (
            "No context-overflow fetch() calls found under static/js — "
            "if the frontend refactored its fetches, update this parser"
        )
        for js_path, origin in fetched.items():
            _assert_get_route_exists(js_path, origin=origin)

    def test_frontend_fetches_both_canonical_endpoints(self):
        """The components call both migrated endpoints (dashboard and
        per-research), i.e. the parsed fetch set covers each canonical
        route — proves the parser and the frontend agree on the contract."""
        fetched = set(_frontend_fetch_paths())
        for canonical in (OVERVIEW_URL, RESEARCH_URL):
            assert any(
                _paths_match(js_path, canonical) for js_path in fetched
            ), f"No JS component fetches {canonical}; found only {fetched}"


# ---------------------------------------------------------------------------
# 2. Auth is required
# ---------------------------------------------------------------------------


class TestAuthRequired:
    @pytest.mark.parametrize(
        "path",
        [OVERVIEW_URL, "/api/research/some-id/context-overflow"],
    )
    def test_anonymous_request_is_401(self, path):
        client = TestClient(_app())
        resp = client.get(path)
        assert resp.status_code == 401, (
            f"GET {path} without a session returned {resp.status_code}, "
            f"expected 401 (require_auth dependency)"
        )


# ---------------------------------------------------------------------------
# 3. Response shape — /api/context-overflow (dashboard payload)
# ---------------------------------------------------------------------------

# context-overflow.js destructures exactly these from the payload:
#   const { overview, token_summary, model_stats, model_token_stats,
#           recent_truncated, chart_data, context_limits, phase_breakdown,
#           current_context_window, all_requests, pagination } = data;
FRONTEND_TOP_LEVEL_KEYS = {
    "overview",
    "token_summary",
    "model_stats",
    "model_token_stats",
    "recent_truncated",
    "chart_data",
    "context_limits",
    "phase_breakdown",
    "current_context_window",
    "all_requests",
    "pagination",
}


class TestOverviewResponseShape:
    def test_top_level_keys_and_status(self, auth_client):
        with _seeded_db(*_seed_rows()):
            resp = auth_client.get(OVERVIEW_URL)
        assert resp.status_code == 200
        data = resp.json()
        assert data["status"] == "success"
        missing = FRONTEND_TOP_LEVEL_KEYS - set(data)
        assert not missing, (
            f"Response lost keys the frontend destructures: {missing}"
        )

    def test_overview_aggregates_from_real_sql(self, auth_client):
        with _seeded_db(*_seed_rows()):
            resp = auth_client.get(OVERVIEW_URL + "?period=30d")
        overview = resp.json()["overview"]
        assert overview["total_requests"] == 3
        assert overview["requests_with_context_data"] == 2
        assert overview["truncated_requests"] == 1
        # 1 truncated / 2 with context data = 50%
        assert overview["truncation_rate"] == 50.0
        assert overview["avg_tokens_truncated"] == 1200

    def test_token_summary_keys(self, auth_client):
        with _seeded_db(*_seed_rows()):
            resp = auth_client.get(OVERVIEW_URL)
        summary = resp.json()["token_summary"]
        # context-overflow.js keys the empty-state off total_requests and
        # renders the rest in the summary table.
        for key in (
            "total_requests",
            "total_tokens",
            "total_prompt_tokens",
            "total_completion_tokens",
            "avg_prompt_tokens",
            "avg_completion_tokens",
            "max_prompt_tokens",
        ):
            assert key in summary, f"token_summary lost {key}"
        assert summary["total_requests"] == 3
        assert summary["total_tokens"] == 9500 + 2300 + 1200
        assert summary["max_prompt_tokens"] == 9000

    def test_chart_data_reconstructs_original_tokens(self, auth_client):
        """chart_data reports pre-truncation tokens for truncated rows:
        ollama_prompt_eval_count + tokens_truncated."""
        with _seeded_db(*_seed_rows()):
            resp = auth_client.get(OVERVIEW_URL)
        chart = resp.json()["chart_data"]
        assert len(chart) == 3
        truncated_points = [p for p in chart if p["truncated"]]
        assert len(truncated_points) == 1
        point = truncated_points[0]
        assert point["ollama_prompt_tokens"] == 8000
        assert point["original_prompt_tokens"] == 8000 + 1200
        assert point["context_limit"] == 8192
        assert point["tokens_truncated"] == 1200
        assert point["model"] == "llama3"

    def test_recent_truncated_and_model_stats(self, auth_client):
        with _seeded_db(*_seed_rows()):
            resp = auth_client.get(OVERVIEW_URL)
        data = resp.json()

        recent = data["recent_truncated"]
        assert len(recent) == 1
        assert recent[0]["original_tokens"] == 8000 + 1200
        assert recent[0]["research_id"] == "res-1"
        assert recent[0]["tokens_truncated"] == 1200

        # model_stats only covers rows WITH context_limit
        stats = data["model_stats"]
        assert len(stats) == 1
        assert stats[0]["model"] == "llama3"
        assert stats[0]["total_requests"] == 2
        assert stats[0]["truncated_count"] == 1
        assert stats[0]["truncation_rate"] == 50.0

        limits = data["context_limits"]
        assert limits == [{"model": "llama3", "limit": 8192, "count": 2}]

    def test_all_requests_and_pagination(self, auth_client):
        with _seeded_db(*_seed_rows()):
            resp = auth_client.get(OVERVIEW_URL)
        data = resp.json()
        rows = data["all_requests"]
        assert len(rows) == 3
        # Newest first
        assert rows[0]["context_truncated"] is True
        assert rows[0]["truncation_ratio"] == 12.5  # 0.125 -> percent
        for key in (
            "timestamp",
            "research_id",
            "model",
            "provider",
            "prompt_tokens",
            "completion_tokens",
            "total_tokens",
            "context_limit",
            "context_truncated",
            "tokens_truncated",
            "truncation_ratio",
            "research_phase",
        ):
            assert key in rows[0], f"all_requests row lost {key}"
        assert data["pagination"] == {
            "page": 1,
            "per_page": 50,
            "total_count": 3,
            "total_pages": 1,
        }

    def test_pagination_params_are_clamped(self, auth_client):
        """per_page caps at 500, page floors at 1 — DoS/negative guard."""
        with _seeded_db(*_seed_rows()):
            resp = auth_client.get(OVERVIEW_URL + "?per_page=9999&page=0")
        pagination = resp.json()["pagination"]
        assert pagination["per_page"] == 500
        assert pagination["page"] == 1

    def test_invalid_period_falls_back_instead_of_erroring(self, auth_client):
        with _seeded_db(*_seed_rows()):
            resp = auth_client.get(OVERVIEW_URL + "?period=--drop--")
        assert resp.status_code == 200
        data = resp.json()
        assert data["status"] == "success"
        # Fallback is 30d: the seeded rows are recent so all are included
        assert data["overview"]["total_requests"] == 3

    def test_empty_db_yields_success_with_zero_requests(self, auth_client):
        """The frontend's empty-state check is
        ``token_summary.total_requests === 0`` on a success payload."""
        with _seeded_db():
            resp = auth_client.get(OVERVIEW_URL)
        assert resp.status_code == 200
        data = resp.json()
        assert data["status"] == "success"
        assert data["token_summary"]["total_requests"] == 0


# ---------------------------------------------------------------------------
# 4. Response shape — /api/research/{id}/context-overflow
# ---------------------------------------------------------------------------


class TestResearchResponseShape:
    def test_truncated_research_overview_keys(self, auth_client):
        """results.js/progress.js read data.overview.truncation_occurred,
        truncated_count, tokens_lost, context_limit for the warning toast."""
        with _seeded_db(*_seed_rows()):
            resp = auth_client.get("/api/research/res-1/context-overflow")
        assert resp.status_code == 200
        payload = resp.json()
        assert payload["status"] == "success"
        overview = payload["data"]["overview"]
        assert overview["truncation_occurred"] is True
        assert overview["truncated_count"] == 1
        assert overview["tokens_lost"] == 1200
        assert overview["context_limit"] == 8192
        assert overview["total_requests"] == 2
        assert overview["max_tokens_used"] == 9000
        assert overview["total_tokens"] == 9500 + 2300

    def test_phase_stats_and_requests(self, auth_client):
        with _seeded_db(*_seed_rows()):
            resp = auth_client.get("/api/research/res-1/context-overflow")
        data = resp.json()["data"]
        assert set(data["phase_stats"]) == {"synthesis", "search"}
        assert data["phase_stats"]["synthesis"]["truncated_count"] == 1
        assert data["phase_stats"]["search"]["truncated_count"] == 0
        assert data["model"] == "llama3"
        assert data["provider"] == "ollama"
        assert len(data["requests"]) == 2
        for key in (
            "timestamp",
            "phase",
            "prompt_tokens",
            "completion_tokens",
            "total_tokens",
            "context_limit",
            "context_truncated",
            "tokens_truncated",
        ):
            assert key in data["requests"][0], f"requests row lost {key}"

    def test_unknown_research_returns_zero_shape_not_404(self, auth_client):
        """The frontend treats non-ok as 'API unavailable' and stays silent;
        an unknown/quiet research must therefore return a success payload
        with truncation_occurred=False, not an error."""
        with _seeded_db(*_seed_rows()):
            resp = auth_client.get("/api/research/nope/context-overflow")
        assert resp.status_code == 200
        payload = resp.json()
        assert payload["status"] == "success"
        overview = payload["data"]["overview"]
        assert overview["total_requests"] == 0
        assert overview["truncation_occurred"] is False
        assert payload["data"]["requests"] == []