Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

13.2 KB
Raw Blame
"""Two regressions the FastAPI port introduced by omission, pinned here.

Both are cases where main had a decision written down and the migration
dropped it silently, leaving code that still imports and still runs.

1. **The log-queue daemon must not be able to abort the boot** (#5982).
   main's ``web/app.py::main()`` wraps ``start_log_queue_processor()`` in
   ``try/except`` (PR #3488); the lifespan port called it bare. That matters
   because ``web/fastapi_app.lifespan`` is explicitly two-tier -- see
   ``tests/web/test_lifespan_startup_shutdown.py``: an UNGUARDED step that
   raises produces ``lifespan.startup.failed`` and uvicorn never serves.
   ``start_log_queue_processor`` ends in ``threading.Thread(...).start()``,
   which raises ``RuntimeError: can't start new thread`` under a container
   pids/``RLIMIT_NPROC`` ceiling, so an unguarded call turns "the log drain
   could not spawn a thread" into "the server does not come up".

   Checked structurally (AST) rather than by driving a real lifespan: the
   guard sits well past ``init_lock()``, and a process may complete at most
   ONE real lifespan cycle (that budget belongs to
   ``tests/web/test_lifespan_boot.py``). The same file already uses an AST
   check for the same reason in ``test_no_error_path_in_the_secret_key_loader_returns_a_key``.

2. **The #4431 request-timing forensics must stay wired up** (#5959).
   Landed on main as #4536, installed from ``app_factory.create_app`` under
   ``CI``/``TESTING``; merge ``5ad5f5a1b`` deleted ``app_factory.py`` and the
   installation went with it, twice unnoticed because the module and its own
   unit tests survived. These tests pin the wiring, not just the class.
"""

from __future__ import annotations

import ast
import json
import os
import subprocess
import sys
from pathlib import Path

import pytest

from local_deep_research.web import fastapi_app

FASTAPI_APP_PATH = Path(fastapi_app.__file__).resolve()
SRC_ROOT = FASTAPI_APP_PATH.parents[2]


# ---------------------------------------------------------------------------
# 1. #5982 -- the log-queue daemon start is guarded
# ---------------------------------------------------------------------------


def _lifespan_ast() -> ast.AsyncFunctionDef:
    tree = ast.parse(FASTAPI_APP_PATH.read_text(encoding="utf-8"))
    for node in tree.body:
        if isinstance(node, ast.AsyncFunctionDef) and node.name == "lifespan":
            return node
    pytest.fail(
        "no module-level `async def lifespan` in web/fastapi_app.py -- the "
        "boot sequence moved and this test no longer checks anything"
    )


def _calls_named(node: ast.AST, name: str) -> list[ast.Call]:
    return [
        n
        for n in ast.walk(node)
        if isinstance(n, ast.Call)
        and isinstance(n.func, ast.Name)
        and n.func.id == name
    ]


def test_the_log_queue_daemon_start_is_wrapped_so_it_cannot_abort_the_boot():
    """#5982: ``start_log_queue_processor()`` must sit inside a ``try``.

    Its neighbours in the same lifespan (the AnyIO threadpool knob, the
    weak-KDF check, themes.css) are all wrapped, each with the rationale in
    the source. This one was fatal purely by omission.
    """
    lifespan = _lifespan_ast()

    calls = _calls_named(lifespan, "start_log_queue_processor")
    assert calls, (
        "premise broken: `lifespan` no longer calls "
        "start_log_queue_processor() at all, so there is nothing to guard "
        "-- if the log drain moved, move this test with it"
    )

    guarded = {
        id(call)
        for try_node in ast.walk(lifespan)
        if isinstance(try_node, ast.Try)
        for stmt in try_node.body
        for call in _calls_named(stmt, "start_log_queue_processor")
    }

    unguarded = [call.lineno for call in calls if id(call) not in guarded]
    assert not unguarded, (
        "start_log_queue_processor() is called OUTSIDE any try/except in "
        f"`lifespan` (fastapi_app.py line(s) {unguarded}). This lifespan is "
        "two-tier: an unguarded step that raises yields "
        "lifespan.startup.failed and uvicorn never serves. A best-effort "
        "logging daemon that cannot spawn its thread (RuntimeError: can't "
        "start new thread, under a container pids ceiling) must not be able "
        "to stop the server booting. See #5982."
    )


def test_the_log_queue_guard_catches_broadly_and_says_so():
    """A bare `except OSError` would not cover the realistic trigger."""
    lifespan = _lifespan_ast()

    handlers = [
        try_node
        for try_node in ast.walk(lifespan)
        if isinstance(try_node, ast.Try)
        and any(
            _calls_named(stmt, "start_log_queue_processor")
            for stmt in try_node.body
        )
    ]
    assert handlers, "no try block wraps the daemon start (see the test above)"

    for try_node in handlers:
        caught = [
            handler.type.id
            for handler in try_node.handlers
            if isinstance(handler.type, ast.Name)
        ]
        assert "Exception" in caught, (
            "the guard around start_log_queue_processor() catches only "
            f"{caught or 'nothing named'}; RuntimeError('can't start new "
            "thread') is the realistic trigger, so it must catch Exception"
        )
        logged = any(
            isinstance(n, ast.Attribute) and n.attr in ("exception", "warning")
            for handler in try_node.handlers
            for n in ast.walk(handler)
        )
        assert logged, (
            "the guard swallows the failure without logging it; a silently "
            "absent log drain is exactly the state that is hard to diagnose"
        )


# ---------------------------------------------------------------------------
# 2. #5959 -- the request-timing forensics are wired up
# ---------------------------------------------------------------------------


class _Recorder:
    """Stands in for the module's loguru ``logger``.

    Substituted wholesale rather than adding a loguru sink: the package is
    ``logger.disable``d at import (``local_deep_research/__init__.py``) and
    only ``config_logger`` re-enables it, so a sink added from a test would
    see nothing emitted from inside ``fastapi_app``.
    """

    def __init__(self):
        self.lines: list[tuple[str, str]] = []

    def info(self, message, *args, **kwargs):
        self.lines.append(("INFO", message))

    def warning(self, message, *args, **kwargs):
        self.lines.append(("WARNING", message))

    def debug(self, message, *args, **kwargs):
        self.lines.append(("DEBUG", message))

    def exception(self, message, *args, **kwargs):
        self.lines.append(("ERROR", message))


async def _ok_app(scope, receive, send):
    await send({"type": "http.response.start", "status": 200, "headers": []})
    await send({"type": "http.response.body", "body": b"ok"})


def _http_scope(path="/chat/", query=b""):
    return {
        "type": "http",
        "method": "GET",
        "path": path,
        "query_string": query,
        "headers": [],
    }


async def _drive(middleware, scope):
    sent: list[dict] = []

    async def send(message):
        sent.append(message)

    async def receive():
        return {"type": "http.request", "body": b"", "more_body": False}

    await middleware(scope, receive, send)
    return sent


@pytest.fixture
def recorder(monkeypatch):
    rec = _Recorder()
    monkeypatch.setattr(fastapi_app, "logger", rec)
    return rec


@pytest.mark.asyncio
async def test_arrival_and_completion_lines_keep_the_grepped_format(recorder):
    """``[req] >`` / ``[req] <`` are what the CI workflow log-grep keys on.

    The arrival line is the actual signal #4431 wants: its ABSENCE during a
    60-second navigation timeout is what distinguishes "the request never
    reached the server" from "the request reached the app and hung".
    """
    middleware = fastapi_app.RequestTimingASGIMiddleware(_ok_app)

    sent = await _drive(middleware, _http_scope())

    assert sent[0]["status"] == 200, sent
    messages = [message for _level, message in recorder.lines]
    assert messages[0] == "[req] > GET /chat/", messages
    assert messages[1].startswith("[req] < GET /chat/ 0."), messages
    assert messages[1].endswith("s"), messages


@pytest.mark.asyncio
async def test_socketio_polls_carry_their_query_string(recorder):
    """transport/sid make engine.io poll churn correlatable."""
    middleware = fastapi_app.RequestTimingASGIMiddleware(_ok_app)

    await _drive(
        middleware,
        _http_scope("/socket.io/", b"EIO=4&transport=polling&sid=abc"),
    )

    assert (
        recorder.lines[0][1]
        == "[req] > GET /socket.io/?EIO=4&transport=polling&sid=abc"
    ), recorder.lines


@pytest.mark.asyncio
async def test_crlf_in_the_path_cannot_forge_a_log_line(recorder):
    """The forensics output is grep'd downstream, so a crafted path must
    not be able to inject a second, fake ``[req]`` record."""
    middleware = fastapi_app.RequestTimingASGIMiddleware(_ok_app)

    await _drive(middleware, _http_scope("/a\r\n[req] > GET /forged"))

    for _level, message in recorder.lines:
        assert "\n" not in message and "\r" not in message, message
    assert "\\r\\n" in recorder.lines[0][1], recorder.lines


@pytest.mark.asyncio
async def test_a_slow_request_is_logged_as_a_warning(recorder, monkeypatch):
    """The WARNING is the half of the format CI greps for by level."""
    from local_deep_research.web.utils import request_timing

    monkeypatch.setattr(request_timing, "SLOW_REQUEST_SECONDS", 0.0)
    middleware = fastapi_app.RequestTimingASGIMiddleware(_ok_app)

    await _drive(middleware, _http_scope())

    assert recorder.lines[1][0] == "WARNING", recorder.lines
    assert recorder.lines[1][1].endswith("s SLOW"), recorder.lines


@pytest.mark.asyncio
async def test_non_http_scopes_pass_straight_through(recorder):
    """lifespan/websocket scopes are not HTTP arrivals; wrapping them would
    also mean logging every Socket.IO connection twice."""
    seen: list[str] = []

    async def inner(scope, receive, send):
        seen.append(scope["type"])

    middleware = fastapi_app.RequestTimingASGIMiddleware(inner)
    await middleware({"type": "lifespan"}, None, None)
    await middleware({"type": "websocket", "path": "/ws"}, None, None)

    assert seen == ["lifespan", "websocket"]
    assert recorder.lines == [], recorder.lines


def test_the_timing_gate_matches_mains_ci_or_testing_condition(monkeypatch):
    """main gated on ``CI or TESTING``; that half is kept verbatim."""
    monkeypatch.setattr(fastapi_app.sys, "modules", dict(sys.modules))
    fastapi_app.sys.modules.pop("pytest", None)

    for env, expected in (
        ({}, False),
        ({"CI": "true"}, True),
        ({"TESTING": "1"}, True),
        ({"CI": "", "TESTING": ""}, False),
    ):
        monkeypatch.setattr(os, "environ", dict(env))
        assert fastapi_app._request_timing_enabled() is expected, env


def test_the_gate_stays_shut_inside_a_pytest_process():
    """Deliberate deviation from main, and the reason the exact
    ``app.user_middleware`` sequence pinned by
    ``tests/web/test_middleware_order_and_headers.py`` is unaffected on a CI
    runner (where ``CI=true`` is set for the unit-test job too).

    The forensics target the long-running UI-shard server, which CI starts as
    its own process (``python -m local_deep_research.web.app``) -- pytest is
    not imported there. ``request_timing._should_arm_freeze_dump`` already
    refuses to arm its faulthandler timer under pytest for the same reason.
    """
    assert "pytest" in sys.modules, "premise broken: not under pytest"
    assert fastapi_app._request_timing_enabled() is False


def test_the_middleware_is_installed_and_outermost_under_CI():
    """The whole point of #5959: the class existing is not the same as the
    class being installed. Run in a child interpreter because the gate is
    (correctly) shut inside this one.

    Outermost matters: ``add_middleware`` is LIFO, so only the last-added
    layer measures true arrival-to-response time rather than the time
    downstream of rate limiting, CORS and the security stack.
    """
    probe = (
        "import json;"
        "from local_deep_research.web.fastapi_app import app;"
        "print(json.dumps([m.cls.__name__ for m in app.user_middleware]))"
    )
    env = {
        **os.environ,
        "CI": "true",
        "PYTHONPATH": str(SRC_ROOT),
        "LDR_BOOTSTRAP_ALLOW_UNENCRYPTED": "true",
    }
    env.pop("PYTEST_CURRENT_TEST", None)

    result = subprocess.run(
        [sys.executable, "-c", probe],
        capture_output=True,
        text=True,
        env=env,
        timeout=300,
    )
    assert result.returncode == 0, (
        f"importing the app with CI=true failed:\n{result.stderr[-3000:]}"
    )
    stack = json.loads(result.stdout.strip().splitlines()[-1])

    assert "RequestTimingASGIMiddleware" in stack, (
        "the #4431 request-timing forensics are not installed under CI. "
        f"Middleware, outer->inner: {stack}. This is exactly the state "
        "#5959 describes: the module survives, its unit tests pass, and "
        "nothing installs it."
    )
    assert stack[0] == "RequestTimingASGIMiddleware", (
        "request timing must be the OUTERMOST middleware (added last -- "
        f"add_middleware is LIFO), but the stack is {stack}; from any inner "
        "position it stops measuring arrival-to-response time"
    )