~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Behavioral proof that the notes AI endpoints are genuinely async (#5854).
The notes AI tranche (PR #6232) exists to return AnyIO worker threads to
the pool for the length of each model response. These tests demonstrate
that property end-to-end through the REAL FastAPI app — not by inspecting
code, but by observing runtime behavior:
1. ``test_ai_call_runs_on_the_event_loop_thread`` — the awaited AI call
executes as a coroutine on the event-loop thread, never on a pooled
worker thread. Under the old sync-def model the whole handler (LLM
call included) ran ON a worker thread; the thread identity of the
stub call is the observable difference.
2. ``test_concurrent_ai_requests_overlap`` — every request reaches a
held model call before any call is released. Serial execution cannot
reach that barrier.
3. ``test_sync_route_stays_responsive_while_ai_requests_in_flight`` —
the issue's own measure of done, in route form: with MORE AI requests
in flight than the process's ~40 AnyIO worker tokens (45 here, held
until explicitly released), a plain sync-def route (``/api/v1/health``
— served from that same worker pool) still answers before release.
Holding worker tokens across model calls cannot reach that barrier.
Harness notes (mirrors test_notes_router_fastapi.py):
* The real app is driven through ``httpx.AsyncClient`` + ``ASGITransport``
so requests can genuinely overlap on one event loop — the synchronous
``TestClient`` processes one request at a time and cannot express
these scenarios. No lifespan runs (same as the sync harness).
* A real register + login mints the encrypted-DB session the notes
routes require; ``NoteAIService`` is monkeypatched with a recording
stub whose async methods sleep to simulate the model hold.
* The client fixture disables rate limiting for these throughput tests
so the concurrency batches do not trip the 10/min AI bucket.
"""
import asyncio
import threading
import time
import uuid
import httpx
import pytest
import pytest_asyncio
from httpx import ASGITransport
PASSWORD = "TestPass123!"
AI_SLEEP = 0.6
@pytest.fixture()
def fake_ai(monkeypatch):
"""Record model calls, optionally holding them at a shared barrier.
The thread-identity test uses a short sleep. Concurrency checks set
``expected`` and release their calls explicitly, so slow request
scheduling cannot invalidate the overlap premise.
"""
from local_deep_research.web.routers import notes as notes_router
state = {
"calls": [],
"sleep": AI_SLEEP,
"expected": 0,
"started": asyncio.Event(),
"release": asyncio.Event(),
}
class _SlowAI:
def __init__(self, username, dbpw=None):
pass
async def summarize_note(self, note_id):
call = {
"method": "summarize_note",
"note_id": note_id,
"thread": threading.get_ident(),
"started": time.monotonic(),
"finished": None,
}
# Bind the record locally so overlapping calls each stamp
# their OWN finish — calls[-1] would race between coroutines.
state["calls"].append(call)
if state["expected"]:
if len(state["calls"]) == state["expected"]:
state["started"].set()
await state["release"].wait()
else:
await asyncio.sleep(state["sleep"])
call["finished"] = time.monotonic()
return f"summary of {note_id}"
monkeypatch.setattr(notes_router, "NoteAIService", _SlowAI)
return state
@pytest_asyncio.fixture()
async def client(monkeypatch):
"""Real registered + logged-in async client against the real app."""
from local_deep_research.web.fastapi_app import app
from local_deep_research.web.routers import notes as notes_router
# These throughput batches intentionally exceed normal request quotas.
# Disable the instances captured by the routes and middleware,
# independent of import order, and restore both at fixture teardown.
monkeypatch.setattr(notes_router.limiter, "enabled", False)
monkeypatch.setattr(app.state.limiter, "enabled", False)
async with httpx.AsyncClient(
transport=ASGITransport(app=app),
base_url="http://testserver",
follow_redirects=True,
) as c:
username = "noteai_" + uuid.uuid4().hex[:8]
await c.get("/auth/login") # establish the session for CSRF minting
csrf = (await c.get("/auth/csrf-token")).json().get("csrf_token", "")
reg = await c.post(
"/auth/register",
data={
"username": username,
"password": PASSWORD,
"confirm_password": PASSWORD,
"acknowledge": "true",
"csrf_token": csrf,
},
)
assert reg.status_code in (200, 302), (
f"register failed: {reg.status_code}"
)
login = await c.post(
"/auth/login",
data={
"username": username,
"password": PASSWORD,
# The CSRF token rotates on every mutating request — mint
# a fresh one here (register consumed the previous).
"csrf_token": (await c.get("/auth/csrf-token"))
.json()
.get("csrf_token", ""),
},
)
assert login.status_code in (200, 302), (
f"login failed: {login.status_code}"
)
c.headers["X-CSRFToken"] = (
(await c.get("/auth/csrf-token")).json().get("csrf_token", "")
)
yield c
async def _summarize(client, note_id):
return await client.post(f"/notes/api/notes/{note_id}/summarize")
# ---------------------------------------------------------------------------
# 1. WHERE the model hold executes
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_ai_call_runs_on_the_event_loop_thread(client, fake_ai):
"""The awaited AI call is a coroutine on the loop thread, not a worker.
Contrast pair: ``loop_thread`` is the thread running this test's event
loop; ``pool_thread`` is a genuine asyncio.to_thread worker. The stub's
recorded identity must equal the former and differ from the latter —
the sync-def model this tranche replaced would fail both assertions.
"""
loop_thread = threading.get_ident()
pool_thread = await asyncio.to_thread(threading.get_ident)
r = await _summarize(client, "note-1")
assert r.status_code == 200, r.text
assert r.json() == {"success": True, "summary": "summary of note-1"}
assert len(fake_ai["calls"]) == 1
call_thread = fake_ai["calls"][0]["thread"]
assert call_thread == loop_thread, (
"AI call must execute on the event-loop thread (it is awaited), "
f"got {call_thread} (loop={loop_thread})"
)
assert call_thread != pool_thread
# ---------------------------------------------------------------------------
# 2. Concurrent AI requests overlap
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_concurrent_ai_requests_overlap(client, fake_ai):
"""All four model calls start before any one is allowed to finish."""
fake_ai["expected"] = 4
tasks = [
asyncio.create_task(_summarize(client, f"note-{i}")) for i in range(4)
]
try:
await asyncio.wait_for(fake_ai["started"].wait(), timeout=10)
assert len(fake_ai["calls"]) == 4
assert all(call["finished"] is None for call in fake_ai["calls"])
finally:
fake_ai["release"].set()
responses = await asyncio.gather(*tasks, return_exceptions=True)
assert all(
not isinstance(response, BaseException) and response.status_code == 200
for response in responses
), responses
# ---------------------------------------------------------------------------
# 3. The measure of done: a sync route stays responsive under AI load
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_sync_route_stays_responsive_while_ai_requests_in_flight(
client, fake_ai
):
"""The sync health route answers while all 45 model calls remain held.
Wait for the actual calls, rather than assuming their middleware and
database work finished within a fixed fraction of a second. The bounded
wait still fails if model calls retain the pool's ~40 worker tokens.
"""
fake_ai["expected"] = 45
tasks = [
asyncio.create_task(_summarize(client, f"note-{i}")) for i in range(45)
]
try:
await asyncio.wait_for(fake_ai["started"].wait(), timeout=10)
assert len(fake_ai["calls"]) == 45
assert all(call["finished"] is None for call in fake_ai["calls"])
ping = await asyncio.wait_for(client.get("/api/v1/health"), timeout=10)
assert ping.status_code == 200, ping.text
assert all(call["finished"] is None for call in fake_ai["calls"])
finally:
# Drain every request even if the barrier or health check fails;
# pending ASGI work must not race database/session fixture teardown.
fake_ai["release"].set()
responses = await asyncio.gather(*tasks, return_exceptions=True)
assert all(
not isinstance(response, BaseException) and response.status_code == 200
for response in responses
), responses