~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Contract tests for the context-overflow API (FastAPI migration).
The context-overflow endpoints moved from the Flask-era ``/metrics/api/*``
guesses to the un-prefixed FastAPI router ``routers/context_overflow_api.py``:
* ``GET /api/context-overflow`` — analytics dashboard payload
(fetched by ``static/js/components/context-overflow.js``)
* ``GET /api/research/{research_id}/context-overflow`` — per-research
overflow check (fetched by ``progress.js``, ``details.js``, ``results.js``)
plus the diagnostics page ``GET /metrics/context-overflow`` declared in
``static/js/config/urls.js`` (``PAGES.METRICS_CONTEXT_OVERFLOW``).
These tests fence:
1. Every context-overflow URL the frontend declares (urls.js) or hardcodes
in a ``fetch(...)`` call resolves to a registered GET route — the JS
files are parsed so a path drift on either side fails the test.
(tests/infrastructure_tests/test_urls_js.py checks a generic list of
critical URLs which does NOT include the context-overflow entries.)
2. Both API endpoints require authentication (401 when anonymous).
3. The response shape keys the frontend destructures/reads, driven through
the routes' REAL SQL against a seeded in-memory SQLite DB.
"""
import re
from contextlib import contextmanager
from datetime import UTC, datetime, timedelta
from pathlib import Path
from unittest.mock import patch
import pytest
from fastapi.routing import APIRoute
from fastapi.testclient import TestClient
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import StaticPool
from local_deep_research.database.models import Base
from local_deep_research.database.models.metrics import TokenUsage
OVERVIEW_URL = "/api/context-overflow"
RESEARCH_URL = "/api/research/{research_id}/context-overflow"
_REPO_ROOT = Path(__file__).parents[3]
_STATIC_JS = (
_REPO_ROOT / "src" / "local_deep_research" / "web" / "static" / "js"
)
_URLS_JS = _STATIC_JS / "config" / "urls.js"
# ---------------------------------------------------------------------------
# Helpers: route table + JS parsing
# ---------------------------------------------------------------------------
def _app():
from local_deep_research.web.fastapi_app import app
return app
def _api_routes() -> list[tuple[str, frozenset]]:
"""(path, methods) for every APIRoute on the live app."""
return [
(r.path, frozenset(r.methods))
for r in _app().routes
if isinstance(r, APIRoute)
]
def _paths_match(js_path: str, app_path: str) -> bool:
"""Segment-wise match; any ``{param}`` segment matches anything."""
js_parts = js_path.strip("/").split("/")
app_parts = app_path.strip("/").split("/")
if len(js_parts) != len(app_parts):
return False
for js_part, app_part in zip(js_parts, app_parts):
if "{" in js_part or "{" in app_part:
continue
if js_part != app_part:
return False
return True
def _assert_get_route_exists(js_path: str, origin: str) -> None:
routes = _api_routes()
matches = [
(path, methods)
for path, methods in routes
if _paths_match(js_path, path)
]
assert matches, (
f"Frontend path {js_path!r} (from {origin}) has no matching route "
f"in the FastAPI route table"
)
assert any("GET" in methods for _, methods in matches), (
f"Frontend path {js_path!r} (from {origin}) matched "
f"{[m[0] for m in matches]} but none accepts GET"
)
def _urls_js_context_overflow_entries() -> list[tuple[str, str]]:
"""(KEY, path) pairs in urls.js whose path mentions context-overflow."""
content = _URLS_JS.read_text()
return [
(key, url)
for key, url in re.findall(r"(\w+):\s*'([^']+)'", content)
if "context-overflow" in url
]
def _normalize_fetch_path(raw: str) -> str:
"""Template-literal fetch path -> route-table comparable path."""
path = raw.split("?", 1)[0]
return re.sub(r"\$\{[^}]*\}", "{param}", path)
def _frontend_fetch_paths() -> dict[str, str]:
"""Every context-overflow path fetched by the JS components.
Returns {normalized_path: 'file:rawpath'} scanning both template
literals and plain-quoted fetch arguments.
"""
found: dict[str, str] = {}
fetch_re = re.compile(r"""fetch\(\s*(?:`([^`]+)`|'([^']+)'|"([^"]+)")""")
for js_file in sorted(_STATIC_JS.rglob("*.js")):
content = js_file.read_text()
for match in fetch_re.finditer(content):
raw = next(g for g in match.groups() if g is not None)
if "context-overflow" not in raw:
continue
found[_normalize_fetch_path(raw)] = f"{js_file.name}:{raw}"
return found
# ---------------------------------------------------------------------------
# Fixtures: authenticated client + seeded per-user DB
# ---------------------------------------------------------------------------
@pytest.fixture
def auth_client():
"""TestClient authenticated as ``testuser`` via dependency override.
Overriding ``require_auth`` (instead of register/login) lets the seeded
in-memory DB below stand in for the per-user encrypted DB.
"""
from local_deep_research.web.dependencies.auth import require_auth
app = _app()
app.dependency_overrides[require_auth] = lambda: "testuser"
try:
yield TestClient(app)
finally:
app.dependency_overrides.pop(require_auth, None)
@contextmanager
def _seeded_db(*rows):
"""Patch the router's ``get_user_db_session`` with a seeded SQLite DB.
StaticPool shares the single in-memory connection between the seeding
session and the route handler (which runs in the TestClient
threadpool), so the routes execute their REAL aggregation SQL.
"""
engine = create_engine(
"sqlite://",
connect_args={"check_same_thread": False},
poolclass=StaticPool,
)
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
seed = Session()
seed.add_all(rows)
seed.commit()
seed.close()
@contextmanager
def _ctx(username=None, password=None, **kwargs):
with Session() as session:
yield session
with patch(
"local_deep_research.web.routers.context_overflow_api"
".get_user_db_session",
_ctx,
):
yield
_NOW = datetime.now(UTC)
def _usage(**overrides) -> TokenUsage:
defaults = dict(
research_id="res-1",
timestamp=_NOW - timedelta(hours=1),
model_provider="ollama",
model_name="llama3",
prompt_tokens=2000,
completion_tokens=300,
total_tokens=2300,
)
defaults.update(overrides)
return TokenUsage(**defaults)
def _seed_rows() -> tuple[TokenUsage, TokenUsage, TokenUsage]:
"""Three rows: one truncated, one clean-with-context, one context-less."""
truncated = _usage(
timestamp=_NOW - timedelta(minutes=5),
prompt_tokens=9000,
completion_tokens=500,
total_tokens=9500,
context_limit=8192,
context_truncated=True,
tokens_truncated=1200,
truncation_ratio=0.125,
ollama_prompt_eval_count=8000,
research_phase="synthesis",
response_time_ms=1500,
research_query="what is overflow",
)
clean = _usage(
timestamp=_NOW - timedelta(hours=2),
context_limit=8192,
context_truncated=False,
research_phase="search",
response_time_ms=800,
)
no_context = _usage(
research_id="res-2",
timestamp=_NOW - timedelta(hours=3),
model_provider="openai",
model_name="gpt-4o-mini",
prompt_tokens=1000,
completion_tokens=200,
total_tokens=1200,
)
return truncated, clean, no_context
# ---------------------------------------------------------------------------
# 1. Route registration matches what the frontend declares
# ---------------------------------------------------------------------------
class TestRouteRegistration:
def test_canonical_api_paths_registered_with_get(self):
"""The two migrated endpoints exist verbatim in the route table."""
route_map = {path: methods for path, methods in _api_routes()}
for path in (OVERVIEW_URL, RESEARCH_URL):
assert path in route_map, (
f"{path} missing from FastAPI route table — the "
f"context-overflow router moved or lost its route"
)
assert "GET" in route_map[path], (
f"{path} registered without GET: {route_map[path]}"
)
def test_urls_js_context_overflow_entries_resolve(self):
"""Every context-overflow URL declared in urls.js is a GET route."""
entries = _urls_js_context_overflow_entries()
assert entries, (
f"No context-overflow entries found in {_URLS_JS} — if the key "
f"was renamed/removed, update this test's parser"
)
# The diagnostics page must stay among them: results.js/progress.js
# link users to overflow details, and this entry is not covered by
# test_urls_js.py's critical-mapping list.
assert any(url == "/metrics/context-overflow" for _, url in entries), (
f"PAGES.METRICS_CONTEXT_OVERFLOW drifted: {entries}"
)
for key, url in entries:
_assert_get_route_exists(url, origin=f"urls.js:{key}")
def test_component_fetch_paths_resolve(self):
"""Every fetch('.../context-overflow...') in the JS components hits
a registered GET route.
This is the regression the migration actually fixed: the Flask-era
frontend fetched /metrics/api/... which 404'd silently
(``if (!response.ok) return``), so a path drift never surfaces in
the UI — only a test can catch it.
"""
fetched = _frontend_fetch_paths()
assert fetched, (
"No context-overflow fetch() calls found under static/js — "
"if the frontend refactored its fetches, update this parser"
)
for js_path, origin in fetched.items():
_assert_get_route_exists(js_path, origin=origin)
def test_frontend_fetches_both_canonical_endpoints(self):
"""The components call both migrated endpoints (dashboard and
per-research), i.e. the parsed fetch set covers each canonical
route — proves the parser and the frontend agree on the contract."""
fetched = set(_frontend_fetch_paths())
for canonical in (OVERVIEW_URL, RESEARCH_URL):
assert any(
_paths_match(js_path, canonical) for js_path in fetched
), f"No JS component fetches {canonical}; found only {fetched}"
# ---------------------------------------------------------------------------
# 2. Auth is required
# ---------------------------------------------------------------------------
class TestAuthRequired:
@pytest.mark.parametrize(
"path",
[OVERVIEW_URL, "/api/research/some-id/context-overflow"],
)
def test_anonymous_request_is_401(self, path):
client = TestClient(_app())
resp = client.get(path)
assert resp.status_code == 401, (
f"GET {path} without a session returned {resp.status_code}, "
f"expected 401 (require_auth dependency)"
)
# ---------------------------------------------------------------------------
# 3. Response shape — /api/context-overflow (dashboard payload)
# ---------------------------------------------------------------------------
# context-overflow.js destructures exactly these from the payload:
# const { overview, token_summary, model_stats, model_token_stats,
# recent_truncated, chart_data, context_limits, phase_breakdown,
# current_context_window, all_requests, pagination } = data;
FRONTEND_TOP_LEVEL_KEYS = {
"overview",
"token_summary",
"model_stats",
"model_token_stats",
"recent_truncated",
"chart_data",
"context_limits",
"phase_breakdown",
"current_context_window",
"all_requests",
"pagination",
}
class TestOverviewResponseShape:
def test_top_level_keys_and_status(self, auth_client):
with _seeded_db(*_seed_rows()):
resp = auth_client.get(OVERVIEW_URL)
assert resp.status_code == 200
data = resp.json()
assert data["status"] == "success"
missing = FRONTEND_TOP_LEVEL_KEYS - set(data)
assert not missing, (
f"Response lost keys the frontend destructures: {missing}"
)
def test_overview_aggregates_from_real_sql(self, auth_client):
with _seeded_db(*_seed_rows()):
resp = auth_client.get(OVERVIEW_URL + "?period=30d")
overview = resp.json()["overview"]
assert overview["total_requests"] == 3
assert overview["requests_with_context_data"] == 2
assert overview["truncated_requests"] == 1
# 1 truncated / 2 with context data = 50%
assert overview["truncation_rate"] == 50.0
assert overview["avg_tokens_truncated"] == 1200
def test_token_summary_keys(self, auth_client):
with _seeded_db(*_seed_rows()):
resp = auth_client.get(OVERVIEW_URL)
summary = resp.json()["token_summary"]
# context-overflow.js keys the empty-state off total_requests and
# renders the rest in the summary table.
for key in (
"total_requests",
"total_tokens",
"total_prompt_tokens",
"total_completion_tokens",
"avg_prompt_tokens",
"avg_completion_tokens",
"max_prompt_tokens",
):
assert key in summary, f"token_summary lost {key}"
assert summary["total_requests"] == 3
assert summary["total_tokens"] == 9500 + 2300 + 1200
assert summary["max_prompt_tokens"] == 9000
def test_chart_data_reconstructs_original_tokens(self, auth_client):
"""chart_data reports pre-truncation tokens for truncated rows:
ollama_prompt_eval_count + tokens_truncated."""
with _seeded_db(*_seed_rows()):
resp = auth_client.get(OVERVIEW_URL)
chart = resp.json()["chart_data"]
assert len(chart) == 3
truncated_points = [p for p in chart if p["truncated"]]
assert len(truncated_points) == 1
point = truncated_points[0]
assert point["ollama_prompt_tokens"] == 8000
assert point["original_prompt_tokens"] == 8000 + 1200
assert point["context_limit"] == 8192
assert point["tokens_truncated"] == 1200
assert point["model"] == "llama3"
def test_recent_truncated_and_model_stats(self, auth_client):
with _seeded_db(*_seed_rows()):
resp = auth_client.get(OVERVIEW_URL)
data = resp.json()
recent = data["recent_truncated"]
assert len(recent) == 1
assert recent[0]["original_tokens"] == 8000 + 1200
assert recent[0]["research_id"] == "res-1"
assert recent[0]["tokens_truncated"] == 1200
# model_stats only covers rows WITH context_limit
stats = data["model_stats"]
assert len(stats) == 1
assert stats[0]["model"] == "llama3"
assert stats[0]["total_requests"] == 2
assert stats[0]["truncated_count"] == 1
assert stats[0]["truncation_rate"] == 50.0
limits = data["context_limits"]
assert limits == [{"model": "llama3", "limit": 8192, "count": 2}]
def test_all_requests_and_pagination(self, auth_client):
with _seeded_db(*_seed_rows()):
resp = auth_client.get(OVERVIEW_URL)
data = resp.json()
rows = data["all_requests"]
assert len(rows) == 3
# Newest first
assert rows[0]["context_truncated"] is True
assert rows[0]["truncation_ratio"] == 12.5 # 0.125 -> percent
for key in (
"timestamp",
"research_id",
"model",
"provider",
"prompt_tokens",
"completion_tokens",
"total_tokens",
"context_limit",
"context_truncated",
"tokens_truncated",
"truncation_ratio",
"research_phase",
):
assert key in rows[0], f"all_requests row lost {key}"
assert data["pagination"] == {
"page": 1,
"per_page": 50,
"total_count": 3,
"total_pages": 1,
}
def test_pagination_params_are_clamped(self, auth_client):
"""per_page caps at 500, page floors at 1 — DoS/negative guard."""
with _seeded_db(*_seed_rows()):
resp = auth_client.get(OVERVIEW_URL + "?per_page=9999&page=0")
pagination = resp.json()["pagination"]
assert pagination["per_page"] == 500
assert pagination["page"] == 1
def test_invalid_period_falls_back_instead_of_erroring(self, auth_client):
with _seeded_db(*_seed_rows()):
resp = auth_client.get(OVERVIEW_URL + "?period=--drop--")
assert resp.status_code == 200
data = resp.json()
assert data["status"] == "success"
# Fallback is 30d: the seeded rows are recent so all are included
assert data["overview"]["total_requests"] == 3
def test_empty_db_yields_success_with_zero_requests(self, auth_client):
"""The frontend's empty-state check is
``token_summary.total_requests === 0`` on a success payload."""
with _seeded_db():
resp = auth_client.get(OVERVIEW_URL)
assert resp.status_code == 200
data = resp.json()
assert data["status"] == "success"
assert data["token_summary"]["total_requests"] == 0
# ---------------------------------------------------------------------------
# 4. Response shape — /api/research/{id}/context-overflow
# ---------------------------------------------------------------------------
class TestResearchResponseShape:
def test_truncated_research_overview_keys(self, auth_client):
"""results.js/progress.js read data.overview.truncation_occurred,
truncated_count, tokens_lost, context_limit for the warning toast."""
with _seeded_db(*_seed_rows()):
resp = auth_client.get("/api/research/res-1/context-overflow")
assert resp.status_code == 200
payload = resp.json()
assert payload["status"] == "success"
overview = payload["data"]["overview"]
assert overview["truncation_occurred"] is True
assert overview["truncated_count"] == 1
assert overview["tokens_lost"] == 1200
assert overview["context_limit"] == 8192
assert overview["total_requests"] == 2
assert overview["max_tokens_used"] == 9000
assert overview["total_tokens"] == 9500 + 2300
def test_phase_stats_and_requests(self, auth_client):
with _seeded_db(*_seed_rows()):
resp = auth_client.get("/api/research/res-1/context-overflow")
data = resp.json()["data"]
assert set(data["phase_stats"]) == {"synthesis", "search"}
assert data["phase_stats"]["synthesis"]["truncated_count"] == 1
assert data["phase_stats"]["search"]["truncated_count"] == 0
assert data["model"] == "llama3"
assert data["provider"] == "ollama"
assert len(data["requests"]) == 2
for key in (
"timestamp",
"phase",
"prompt_tokens",
"completion_tokens",
"total_tokens",
"context_limit",
"context_truncated",
"tokens_truncated",
):
assert key in data["requests"][0], f"requests row lost {key}"
def test_unknown_research_returns_zero_shape_not_404(self, auth_client):
"""The frontend treats non-ok as 'API unavailable' and stays silent;
an unknown/quiet research must therefore return a success payload
with truncation_occurred=False, not an error."""
with _seeded_db(*_seed_rows()):
resp = auth_client.get("/api/research/nope/context-overflow")
assert resp.status_code == 200
payload = resp.json()
assert payload["status"] == "success"
overview = payload["data"]["overview"]
assert overview["total_requests"] == 0
assert overview["truncation_occurred"] is False
assert payload["data"]["requests"] == []