~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Two regressions the FastAPI port introduced by omission, pinned here.
Both are cases where main had a decision written down and the migration
dropped it silently, leaving code that still imports and still runs.
1. **The log-queue daemon must not be able to abort the boot** (#5982).
main's ``web/app.py::main()`` wraps ``start_log_queue_processor()`` in
``try/except`` (PR #3488); the lifespan port called it bare. That matters
because ``web/fastapi_app.lifespan`` is explicitly two-tier -- see
``tests/web/test_lifespan_startup_shutdown.py``: an UNGUARDED step that
raises produces ``lifespan.startup.failed`` and uvicorn never serves.
``start_log_queue_processor`` ends in ``threading.Thread(...).start()``,
which raises ``RuntimeError: can't start new thread`` under a container
pids/``RLIMIT_NPROC`` ceiling, so an unguarded call turns "the log drain
could not spawn a thread" into "the server does not come up".
Checked structurally (AST) rather than by driving a real lifespan: the
guard sits well past ``init_lock()``, and a process may complete at most
ONE real lifespan cycle (that budget belongs to
``tests/web/test_lifespan_boot.py``). The same file already uses an AST
check for the same reason in ``test_no_error_path_in_the_secret_key_loader_returns_a_key``.
2. **The #4431 request-timing forensics must stay wired up** (#5959).
Landed on main as #4536, installed from ``app_factory.create_app`` under
``CI``/``TESTING``; merge ``5ad5f5a1b`` deleted ``app_factory.py`` and the
installation went with it, twice unnoticed because the module and its own
unit tests survived. These tests pin the wiring, not just the class.
"""
from __future__ import annotations
import ast
import json
import os
import subprocess
import sys
from pathlib import Path
import pytest
from local_deep_research.web import fastapi_app
FASTAPI_APP_PATH = Path(fastapi_app.__file__).resolve()
SRC_ROOT = FASTAPI_APP_PATH.parents[2]
# ---------------------------------------------------------------------------
# 1. #5982 -- the log-queue daemon start is guarded
# ---------------------------------------------------------------------------
def _lifespan_ast() -> ast.AsyncFunctionDef:
tree = ast.parse(FASTAPI_APP_PATH.read_text(encoding="utf-8"))
for node in tree.body:
if isinstance(node, ast.AsyncFunctionDef) and node.name == "lifespan":
return node
pytest.fail(
"no module-level `async def lifespan` in web/fastapi_app.py -- the "
"boot sequence moved and this test no longer checks anything"
)
def _calls_named(node: ast.AST, name: str) -> list[ast.Call]:
return [
n
for n in ast.walk(node)
if isinstance(n, ast.Call)
and isinstance(n.func, ast.Name)
and n.func.id == name
]
def test_the_log_queue_daemon_start_is_wrapped_so_it_cannot_abort_the_boot():
"""#5982: ``start_log_queue_processor()`` must sit inside a ``try``.
Its neighbours in the same lifespan (the AnyIO threadpool knob, the
weak-KDF check, themes.css) are all wrapped, each with the rationale in
the source. This one was fatal purely by omission.
"""
lifespan = _lifespan_ast()
calls = _calls_named(lifespan, "start_log_queue_processor")
assert calls, (
"premise broken: `lifespan` no longer calls "
"start_log_queue_processor() at all, so there is nothing to guard "
"-- if the log drain moved, move this test with it"
)
guarded = {
id(call)
for try_node in ast.walk(lifespan)
if isinstance(try_node, ast.Try)
for stmt in try_node.body
for call in _calls_named(stmt, "start_log_queue_processor")
}
unguarded = [call.lineno for call in calls if id(call) not in guarded]
assert not unguarded, (
"start_log_queue_processor() is called OUTSIDE any try/except in "
f"`lifespan` (fastapi_app.py line(s) {unguarded}). This lifespan is "
"two-tier: an unguarded step that raises yields "
"lifespan.startup.failed and uvicorn never serves. A best-effort "
"logging daemon that cannot spawn its thread (RuntimeError: can't "
"start new thread, under a container pids ceiling) must not be able "
"to stop the server booting. See #5982."
)
def test_the_log_queue_guard_catches_broadly_and_says_so():
"""A bare `except OSError` would not cover the realistic trigger."""
lifespan = _lifespan_ast()
handlers = [
try_node
for try_node in ast.walk(lifespan)
if isinstance(try_node, ast.Try)
and any(
_calls_named(stmt, "start_log_queue_processor")
for stmt in try_node.body
)
]
assert handlers, "no try block wraps the daemon start (see the test above)"
for try_node in handlers:
caught = [
handler.type.id
for handler in try_node.handlers
if isinstance(handler.type, ast.Name)
]
assert "Exception" in caught, (
"the guard around start_log_queue_processor() catches only "
f"{caught or 'nothing named'}; RuntimeError('can't start new "
"thread') is the realistic trigger, so it must catch Exception"
)
logged = any(
isinstance(n, ast.Attribute) and n.attr in ("exception", "warning")
for handler in try_node.handlers
for n in ast.walk(handler)
)
assert logged, (
"the guard swallows the failure without logging it; a silently "
"absent log drain is exactly the state that is hard to diagnose"
)
# ---------------------------------------------------------------------------
# 2. #5959 -- the request-timing forensics are wired up
# ---------------------------------------------------------------------------
class _Recorder:
"""Stands in for the module's loguru ``logger``.
Substituted wholesale rather than adding a loguru sink: the package is
``logger.disable``d at import (``local_deep_research/__init__.py``) and
only ``config_logger`` re-enables it, so a sink added from a test would
see nothing emitted from inside ``fastapi_app``.
"""
def __init__(self):
self.lines: list[tuple[str, str]] = []
def info(self, message, *args, **kwargs):
self.lines.append(("INFO", message))
def warning(self, message, *args, **kwargs):
self.lines.append(("WARNING", message))
def debug(self, message, *args, **kwargs):
self.lines.append(("DEBUG", message))
def exception(self, message, *args, **kwargs):
self.lines.append(("ERROR", message))
async def _ok_app(scope, receive, send):
await send({"type": "http.response.start", "status": 200, "headers": []})
await send({"type": "http.response.body", "body": b"ok"})
def _http_scope(path="/chat/", query=b""):
return {
"type": "http",
"method": "GET",
"path": path,
"query_string": query,
"headers": [],
}
async def _drive(middleware, scope):
sent: list[dict] = []
async def send(message):
sent.append(message)
async def receive():
return {"type": "http.request", "body": b"", "more_body": False}
await middleware(scope, receive, send)
return sent
@pytest.fixture
def recorder(monkeypatch):
rec = _Recorder()
monkeypatch.setattr(fastapi_app, "logger", rec)
return rec
@pytest.mark.asyncio
async def test_arrival_and_completion_lines_keep_the_grepped_format(recorder):
"""``[req] >`` / ``[req] <`` are what the CI workflow log-grep keys on.
The arrival line is the actual signal #4431 wants: its ABSENCE during a
60-second navigation timeout is what distinguishes "the request never
reached the server" from "the request reached the app and hung".
"""
middleware = fastapi_app.RequestTimingASGIMiddleware(_ok_app)
sent = await _drive(middleware, _http_scope())
assert sent[0]["status"] == 200, sent
messages = [message for _level, message in recorder.lines]
assert messages[0] == "[req] > GET /chat/", messages
assert messages[1].startswith("[req] < GET /chat/ 0."), messages
assert messages[1].endswith("s"), messages
@pytest.mark.asyncio
async def test_socketio_polls_carry_their_query_string(recorder):
"""transport/sid make engine.io poll churn correlatable."""
middleware = fastapi_app.RequestTimingASGIMiddleware(_ok_app)
await _drive(
middleware,
_http_scope("/socket.io/", b"EIO=4&transport=polling&sid=abc"),
)
assert (
recorder.lines[0][1]
== "[req] > GET /socket.io/?EIO=4&transport=polling&sid=abc"
), recorder.lines
@pytest.mark.asyncio
async def test_crlf_in_the_path_cannot_forge_a_log_line(recorder):
"""The forensics output is grep'd downstream, so a crafted path must
not be able to inject a second, fake ``[req]`` record."""
middleware = fastapi_app.RequestTimingASGIMiddleware(_ok_app)
await _drive(middleware, _http_scope("/a\r\n[req] > GET /forged"))
for _level, message in recorder.lines:
assert "\n" not in message and "\r" not in message, message
assert "\\r\\n" in recorder.lines[0][1], recorder.lines
@pytest.mark.asyncio
async def test_a_slow_request_is_logged_as_a_warning(recorder, monkeypatch):
"""The WARNING is the half of the format CI greps for by level."""
from local_deep_research.web.utils import request_timing
monkeypatch.setattr(request_timing, "SLOW_REQUEST_SECONDS", 0.0)
middleware = fastapi_app.RequestTimingASGIMiddleware(_ok_app)
await _drive(middleware, _http_scope())
assert recorder.lines[1][0] == "WARNING", recorder.lines
assert recorder.lines[1][1].endswith("s SLOW"), recorder.lines
@pytest.mark.asyncio
async def test_non_http_scopes_pass_straight_through(recorder):
"""lifespan/websocket scopes are not HTTP arrivals; wrapping them would
also mean logging every Socket.IO connection twice."""
seen: list[str] = []
async def inner(scope, receive, send):
seen.append(scope["type"])
middleware = fastapi_app.RequestTimingASGIMiddleware(inner)
await middleware({"type": "lifespan"}, None, None)
await middleware({"type": "websocket", "path": "/ws"}, None, None)
assert seen == ["lifespan", "websocket"]
assert recorder.lines == [], recorder.lines
def test_the_timing_gate_matches_mains_ci_or_testing_condition(monkeypatch):
"""main gated on ``CI or TESTING``; that half is kept verbatim."""
monkeypatch.setattr(fastapi_app.sys, "modules", dict(sys.modules))
fastapi_app.sys.modules.pop("pytest", None)
for env, expected in (
({}, False),
({"CI": "true"}, True),
({"TESTING": "1"}, True),
({"CI": "", "TESTING": ""}, False),
):
monkeypatch.setattr(os, "environ", dict(env))
assert fastapi_app._request_timing_enabled() is expected, env
def test_the_gate_stays_shut_inside_a_pytest_process():
"""Deliberate deviation from main, and the reason the exact
``app.user_middleware`` sequence pinned by
``tests/web/test_middleware_order_and_headers.py`` is unaffected on a CI
runner (where ``CI=true`` is set for the unit-test job too).
The forensics target the long-running UI-shard server, which CI starts as
its own process (``python -m local_deep_research.web.app``) -- pytest is
not imported there. ``request_timing._should_arm_freeze_dump`` already
refuses to arm its faulthandler timer under pytest for the same reason.
"""
assert "pytest" in sys.modules, "premise broken: not under pytest"
assert fastapi_app._request_timing_enabled() is False
def test_the_middleware_is_installed_and_outermost_under_CI():
"""The whole point of #5959: the class existing is not the same as the
class being installed. Run in a child interpreter because the gate is
(correctly) shut inside this one.
Outermost matters: ``add_middleware`` is LIFO, so only the last-added
layer measures true arrival-to-response time rather than the time
downstream of rate limiting, CORS and the security stack.
"""
probe = (
"import json;"
"from local_deep_research.web.fastapi_app import app;"
"print(json.dumps([m.cls.__name__ for m in app.user_middleware]))"
)
env = {
**os.environ,
"CI": "true",
"PYTHONPATH": str(SRC_ROOT),
"LDR_BOOTSTRAP_ALLOW_UNENCRYPTED": "true",
}
env.pop("PYTEST_CURRENT_TEST", None)
result = subprocess.run(
[sys.executable, "-c", probe],
capture_output=True,
text=True,
env=env,
timeout=300,
)
assert result.returncode == 0, (
f"importing the app with CI=true failed:\n{result.stderr[-3000:]}"
)
stack = json.loads(result.stdout.strip().splitlines()[-1])
assert "RequestTimingASGIMiddleware" in stack, (
"the #4431 request-timing forensics are not installed under CI. "
f"Middleware, outer->inner: {stack}. This is exactly the state "
"#5959 describes: the module survives, its unit tests pass, and "
"nothing installs it."
)
assert stack[0] == "RequestTimingASGIMiddleware", (
"request timing must be the OUTERMOST middleware (added last -- "
f"add_middleware is LIFO), but the stack is {stack}; from any inner "
"position it stops measuring arrival-to-response time"
)