Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

43.4 KB
Raw Blame
# allow: no-sut-import — drives the real CSRFMiddleware and real routes
"""CSRF coverage audit for the Flask/WSGI -> FastAPI/ASGI port (PR #3299).

The port replaced Flask-WTF's ``CSRFProtect`` with a bespoke ASGI
``CSRFMiddleware`` (``web/dependencies/csrf.py``). Two failure modes matter
and they pull in opposite directions:

A. SECURITY — a state-changing route that the middleware does not actually
   challenge. Cross-origin form -> silent mutation of the victim's data.
B. USABILITY — a legitimate UI call that does not send the token under the
   exact name the middleware reads. 403 on a button that looks fine.

What this file adds on top of the CSRF suites that already exist
(``tests/web/test_csrf_middleware_edges.py`` — synthetic app, per-flag edge
cases; ``tests/web/test_csrf_lifecycle_contracts.py`` — Flask-WTF parity and
the ``/ws`` prefix constraint; ``tests/security/test_csrf_protection.py`` /
``test_csrf_hardening.py`` / ``test_csrf_e2e_flow.py``):

* a CENSUS over the app's REAL route table — every registered
  POST/PUT/PATCH/DELETE path is driven through the REAL middleware object
  in both directions, rather than four synthetic methods on one synthetic
  path;
* proof that the constant-time compare is on the enforcement path (a spy,
  not a source grep) — an ``==`` rewrite would pass every other test here;
* token ROTATION across the real login / logout / register transitions;
* real full-stack probes of the three body shapes the middleware treats
  differently (JSON, multipart, urlencoded form-field fallback);
* the method-shaped hole: a mutating route parked on a verb outside
  ``_UNSAFE_METHODS`` sits outside the middleware entirely, and a
  per-path allowlist cannot pull it back in — the middleware and the
  router do not agree on how a path is spelled — so the only fix is the
  verb, pinned here against the real route table;
* an independent static audit of every ``<form method=post>`` and every
  state-changing ``fetch`` in the shipped frontend.

ANTI-VACUITY. Every enforcement assertion here is paired with a control
that drives the IDENTICAL path to the opposite outcome, so a route that
403s for everyone (or is simply broken) cannot pass. Every static
extractor carries a floor plus a positive and a negative control, so a
regex that stops matching fails instead of reporting "all clean".
"""

from __future__ import annotations

import asyncio
import re
import secrets as _secrets_mod
import uuid
from contextlib import contextmanager
from pathlib import Path
from unittest.mock import patch

import pytest
from local_deep_research.web.dependencies import csrf as csrf_mod
from local_deep_research.web.dependencies.csrf import (
    CSRFMiddleware,
    _UNSAFE_METHODS,
)

REPO_ROOT = Path(__file__).resolve().parents[2]
WEB = REPO_ROOT / "src" / "local_deep_research" / "web"
JS_ROOT = WEB / "static" / "js"
TEMPLATE_ROOT = WEB / "templates"

# Floors. If an extractor or the route table silently stops producing
# results these turn a green "nothing to report" into a red failure.
MIN_UNSAFE_ROUTES = 120
MIN_POST_FORMS = 5
MIN_TOTAL_FORMS = 12
MIN_JS_CALL_SITES = 80
MIN_TEMPLATE_INLINE_CALL_SITES = 15


# ---------------------------------------------------------------------------
# Harness: drive the REAL CSRFMiddleware object over a raw ASGI scope.
#
# No HTTP server, no DB, no route handlers — the inner app is a sentinel
# that records whether the middleware let the request through. This is the
# middleware under test, not a re-implementation of its rules.
# ---------------------------------------------------------------------------


class _Outcome:
    def __init__(self, reached: bool, status: int, body: bytes) -> None:
        self.reached = reached
        self.status = status
        self.body = body

    @property
    def is_csrf_rejection(self) -> bool:
        """403 whose body names CSRF as the reason.

        Distinguishes a CSRF refusal from an auth refusal or a broken
        route, both of which can also be a 403.
        """
        return (
            not self.reached
            and self.status == 403
            and b"csrf" in self.body.lower()
        )

    def __repr__(self) -> str:  # pragma: no cover - failure messages only
        return (
            f"<reached={self.reached} status={self.status} "
            f"body={self.body[:120]!r}>"
        )


def drive(
    method: str,
    path: str,
    *,
    session: dict | None = None,
    headers: dict[str, str] | None = None,
    body: bytes = b"",
) -> _Outcome:
    """Run one request through the real middleware; report what happened."""
    state = {"reached": False}

    async def sentinel(scope, receive, send):
        state["reached"] = True
        # Drain, so a middleware that buffered+replayed the body is
        # exercised the way a real handler would exercise it.
        more = True
        while more:
            msg = await receive()
            more = msg.get("more_body", False)
        await send(
            {
                "type": "http.response.start",
                "status": 200,
                "headers": [(b"content-type", b"application/json")],
            }
        )
        await send({"type": "http.response.body", "body": b'{"ok":true}'})

    sent: list[dict] = []
    delivered = {"done": False}

    async def receive():
        if delivered["done"]:
            return {"type": "http.disconnect"}
        delivered["done"] = True
        return {"type": "http.request", "body": body, "more_body": False}

    async def send(message):
        sent.append(message)

    raw_headers = [
        (k.lower().encode("latin-1"), v.encode("latin-1"))
        for k, v in (headers or {}).items()
    ]
    scope = {
        "type": "http",
        "asgi": {"version": "3.0", "spec_version": "2.3"},
        "http_version": "1.1",
        "method": method,
        "scheme": "http",
        "path": path,
        "raw_path": path.encode(),
        "query_string": b"",
        "root_path": "",
        "headers": raw_headers,
        "client": ("127.0.0.1", 5000),
        "server": ("testserver", 80),
        "session": {} if session is None else session,
    }

    asyncio.run(CSRFMiddleware(sentinel)(scope, receive, send))

    status = next(
        (m["status"] for m in sent if m["type"] == "http.response.start"), 0
    )
    payload = b"".join(
        m.get("body", b"") for m in sent if m["type"] == "http.response.body"
    )
    return _Outcome(state["reached"], status, payload)


def _concrete(path: str) -> str:
    """Turn a route template into a concrete path the middleware will see."""
    path = re.sub(r"\{[^{}]*:path\}", "x/y", path)
    return re.sub(r"\{[^{}]+\}", "1", path)


def _unsafe_routes(app) -> list[tuple[str, str]]:
    """(method, concrete path) for every registered state-changing route."""
    out = []
    for route in app.routes:
        methods = getattr(route, "methods", None) or set()
        path = getattr(route, "path", None)
        if not path:
            continue
        for method in sorted(set(methods) & set(_UNSAFE_METHODS)):
            out.append((method, _concrete(path)))
    return sorted(set(out))


# ---------------------------------------------------------------------------
# A. SECURITY — census over the real route table
# ---------------------------------------------------------------------------


def test_route_census_reaches_the_whole_state_changing_surface(app):
    """Floor: the census must actually see the app's mutating surface.

    Guards every assertion below: an empty or truncated route table would
    make the census pass by examining nothing.
    """
    routes = _unsafe_routes(app)
    assert len(routes) >= MIN_UNSAFE_ROUTES, (
        f"only {len(routes)} state-changing routes enumerated from the live "
        f"app.routes table (floor {MIN_UNSAFE_ROUTES}) — the enumeration "
        "broke, and a silently-empty census would pass every CSRF gate here"
    )
    # All four guarded verbs must be represented, or a whole verb could
    # lose enforcement without the census noticing.
    verbs = {m for m, _ in routes}
    assert verbs == set(_UNSAFE_METHODS), (
        f"census covers {sorted(verbs)} but the middleware guards "
        f"{sorted(_UNSAFE_METHODS)}"
    )


def test_every_state_changing_route_is_challenged_and_a_token_clears_it(app):
    """The core CSRF gate, over every registered mutating route.

    Two directions per route, through the identical middleware object:

    * NEGATIVE — authenticated-shaped session with no ``_csrf_token`` and
      no header: the middleware must refuse with a CSRF-named 403 and the
      inner app must never be reached.
    * POSITIVE (control) — same route, same method, session carrying a
      token that is echoed in ``X-CSRFToken``: the inner app MUST be
      reached. Without this half, a route the middleware happened to
      refuse for any other reason would read as "protected".
    """
    routes = _unsafe_routes(app)
    token = _secrets_mod.token_hex(32)

    unprotected: list[tuple[str, str, str]] = []
    not_passable: list[tuple[str, str, str]] = []

    for method, path in routes:
        bare = drive(
            method,
            path,
            session={"session_id": "s", "username": "victim"},
        )
        if not bare.is_csrf_rejection:
            unprotected.append((method, path, repr(bare)))

        withtok = drive(
            method,
            path,
            session={
                "session_id": "s",
                "username": "victim",
                "_csrf_token": token,
            },
            headers={"X-CSRFToken": token},
        )
        if not withtok.reached:
            not_passable.append((method, path, repr(withtok)))

    assert not not_passable, (
        "CONTROL FAILED — these routes were refused even WITH a valid "
        "session-bound token, so a 'refused without a token' result on them "
        f"would prove nothing: {not_passable[:10]}"
    )
    assert not unprotected, (
        "these state-changing routes are NOT CSRF-challenged; a cross-origin "
        "form/fetch riding the victim's session cookie would reach the "
        f"handler: {unprotected[:10]}"
    )


def test_socketio_mount_is_exempt_and_that_exemption_is_path_scoped(app):
    """The one deliberate exemption, proven live, with its own control.

    ``/ws`` is exempt (Socket.IO does its own handshake auth). The control
    is a sibling path one character different: it must still be challenged,
    proving the exemption is a path rule and not a dead middleware.
    """
    exempt = drive("POST", "/ws/socket.io/", session={"username": "victim"})
    assert exempt.reached, (
        "the Socket.IO mount must stay CSRF-exempt (it authenticates its own "
        f"handshake); got {exempt!r}"
    )

    control = drive("POST", "/api/start_research", session={"username": "v"})
    assert control.is_csrf_rejection, (
        "CONTROL FAILED — a non-exempt path was not challenged either, so "
        f"the exemption assertion above is vacuous: {control!r}"
    )

    # And the exemption must not be reachable by a registered route that
    # merely starts with the same two characters (csrf.py documents the
    # bare "/ws" startswith entry as a standing constraint).
    shadowed = [
        (m, p)
        for m, p in _unsafe_routes(app)
        if p.startswith("/ws") and not (p == "/ws" or p.startswith("/ws/"))
    ]
    assert not shadowed, (
        "these registered mutating routes start with the bare '/ws' exempt "
        f"prefix and therefore ship CSRF-exempt: {shadowed}"
    )


# ---------------------------------------------------------------------------
# A2. Token strength: constant-time comparison, actually on the path
# ---------------------------------------------------------------------------


def test_constant_time_compare_is_on_the_enforcement_path(monkeypatch):
    """The accept decision must run through ``secrets.compare_digest``.

    A source grep would pass on dead code. This spies on the primitive and
    drives a real request through the real middleware:

    * accepted request -> the spy fired, with (session token, presented
      token) as its operands. Rewriting the check to ``==`` keeps every
      other test in this file green and fails only here.
    * no-token request -> the spy must NOT fire (the middleware
      short-circuits) and the request must still be refused, which proves
      the spy is observing the comparison rather than always reporting
      "called".
    """
    calls: list[tuple] = []
    real = _secrets_mod.compare_digest

    def spy(a, b):
        calls.append((a, b))
        return real(a, b)

    monkeypatch.setattr(csrf_mod.secrets, "compare_digest", spy)

    token = "a" * 64
    accepted = drive(
        "POST",
        "/api/start_research",
        session={"_csrf_token": token},
        headers={"X-CSRFToken": token},
    )
    assert accepted.reached, (
        f"CONTROL FAILED — a valid token was not accepted: {accepted!r}"
    )
    assert calls == [(token, token)], (
        "the accept decision did not go through secrets.compare_digest "
        f"(spy saw {calls!r}) — a non-constant-time '==' comparison leaks "
        "the session token one byte at a time to a timing oracle"
    )

    calls.clear()
    refused = drive(
        "POST", "/api/start_research", session={"_csrf_token": token}
    )
    assert refused.is_csrf_rejection, (
        f"a request with no presented token must be refused: {refused!r}"
    )
    assert calls == [], (
        "compare_digest fired with no token presented — the spy is not "
        "actually tracking the comparison, so the assertion above is vacuous"
    )


# ---------------------------------------------------------------------------
# A3. Token binding + rotation across real auth transitions
# ---------------------------------------------------------------------------


def _mint(client) -> str:
    return client.get("/auth/csrf-token").json()["csrf_token"]


def test_token_rotates_on_register_logout_and_login(client):
    """The token must not survive an auth transition.

    ``routers/auth.py`` calls ``request.session.clear()`` at register, login
    and logout (session-fixation defence); because the CSRF token lives in
    that same session dict, clearing it is what rotates the token. If a
    future refactor preserves ``_csrf_token`` across the clear (an easy
    "keep the user's token so the page keeps working" change), a token
    phished before login stays valid against the authenticated session.

    Control: two mints with NO transition in between must return the SAME
    token, so "rotated" cannot be satisfied by a token that is simply
    regenerated on every call.
    """
    client.headers.update(
        {"X-Forwarded-For": f"10.{uuid.uuid4().int % 250 + 1}.7.3"}
    )
    client.get("/auth/login")

    t0 = _mint(client)
    assert len(t0) == 64 and all(c in "0123456789abcdef" for c in t0), (
        f"token is not 32 bytes of hex (secrets.token_hex(32)): {t0!r}"
    )

    # CONTROL: stable within a session, absent any auth transition.
    assert _mint(client) == t0, (
        "the token changed with no auth transition — every 'rotated' "
        "assertion below would then pass vacuously"
    )

    username = f"csrfcov_{uuid.uuid4().hex[:10]}"
    password = "TestPass123!"
    reg = client.post(
        "/auth/register",
        data={
            "username": username,
            "password": password,
            "confirm_password": password,
            "acknowledge": "true",
            "csrf_token": t0,
        },
        follow_redirects=False,
    )
    assert reg.status_code == 302, (
        f"registration did not succeed ({reg.status_code}); the rotation "
        f"assertions need a real transition: {reg.text[:200]}"
    )
    t1 = _mint(client)
    assert t1 != t0, "register did not rotate the CSRF token"

    out = client.post(
        "/auth/logout",
        headers={"X-CSRFToken": t1},
        follow_redirects=False,
    )
    assert out.status_code == 302, f"logout failed: {out.status_code}"
    t2 = _mint(client)
    assert t2 != t1, "logout did not rotate the CSRF token"

    login = client.post(
        "/auth/login",
        data={
            "username": username,
            "password": password,
            "csrf_token": t2,
        },
        follow_redirects=False,
    )
    assert login.status_code == 302, (
        f"login did not succeed ({login.status_code}): {login.text[:200]}"
    )
    t3 = _mint(client)
    assert t3 != t2, "login did not rotate the CSRF token"


def test_token_is_bound_to_its_own_session(client, app):
    """A token minted for one session must not authorise another's request.

    A double-submit scheme that only checked "cookie value == header value"
    would pass every other test here and still let an attacker who can set
    a cookie (subdomain, MITM on a sibling http host) forge requests.

    Control: the SAME victim request carrying the victim's OWN token takes
    the identical path and is NOT refused for CSRF.
    """
    from starlette.testclient import TestClient

    attacker_token = _mint(client)

    victim = TestClient(app)
    victim.headers.update(
        {"X-Forwarded-For": f"10.{uuid.uuid4().int % 250 + 1}.9.4"}
    )
    victim.get("/auth/login")
    victim_token = _mint(victim)
    assert victim_token != attacker_token, (
        "two independent sessions minted the same token — the binding test "
        "below would be vacuous"
    )

    stolen = victim.post(
        "/api/start_research",
        json={"query": "x"},
        headers={"X-CSRFToken": attacker_token},
    )
    assert stolen.status_code == 403 and "csrf" in stolen.text.lower(), (
        "a token from a DIFFERENT session was accepted (or rejected for a "
        f"non-CSRF reason): {stolen.status_code} {stolen.text[:200]}"
    )

    own = victim.post(
        "/api/start_research",
        json={"query": "x"},
        headers={"X-CSRFToken": victim_token},
    )
    assert not (own.status_code == 403 and "csrf" in own.text.lower()), (
        "CONTROL FAILED — the victim's own token was also refused for CSRF, "
        f"so the rejection above proves nothing: {own.status_code} "
        f"{own.text[:200]}"
    )


# ---------------------------------------------------------------------------
# A4. Full-stack probes of the three body shapes the middleware treats
#     differently (JSON / multipart / urlencoded form field)
# ---------------------------------------------------------------------------


@contextmanager
def _no_token(client):
    """Temporarily drop the fixture's default X-CSRFToken header."""
    saved = client.headers.pop("X-CSRFToken", None)
    try:
        yield client
    finally:
        if saved is not None:
            client.headers["X-CSRFToken"] = saved


def _csrf_refused(response) -> bool:
    return response.status_code == 403 and "csrf" in response.text.lower()


def test_json_mutation_needs_the_header_end_to_end(authenticated_client):
    """A JSON API mutation, authenticated, over the whole real stack.

    The middleware deliberately does NOT parse JSON bodies for a token, so
    a JSON caller that forgets the header must be refused — and the same
    call with the header must reach the handler.
    """
    client = authenticated_client
    with _no_token(client):
        bare = client.put("/settings/api/llm.temperature", json={"value": 0.42})
    assert _csrf_refused(bare), (
        "an authenticated JSON PUT with no X-CSRFToken reached past the "
        f"middleware: {bare.status_code} {bare.text[:200]}"
    )

    withtok = client.put("/settings/api/llm.temperature", json={"value": 0.42})
    assert not _csrf_refused(withtok), (
        "CONTROL FAILED — the same PUT with a valid token was also refused "
        f"for CSRF: {withtok.status_code} {withtok.text[:200]}"
    )


def test_multipart_upload_needs_the_header_end_to_end(authenticated_client):
    """Multipart bodies are never parsed for a token — header or nothing.

    This is the shape most likely to regress: Flask's ``request.form``
    covered multipart, this middleware does not, so an upload that relied
    on a hidden field would 403 with no server-side clue.
    """
    client = authenticated_client
    pdf = ("probe.pdf", b"%PDF-1.4\n%%EOF\n", "application/pdf")

    with _no_token(client):
        bare = client.post("/api/upload/pdf", files={"file": pdf})
    assert _csrf_refused(bare), (
        "an authenticated multipart upload with no X-CSRFToken reached past "
        f"the middleware: {bare.status_code} {bare.text[:200]}"
    )

    withtok = client.post("/api/upload/pdf", files={"file": pdf})
    assert not _csrf_refused(withtok), (
        "CONTROL FAILED — the same upload with a valid token was refused "
        f"for CSRF: {withtok.status_code} {withtok.text[:200]}"
    )


def test_no_js_urlencoded_form_fallback_works_end_to_end(authenticated_client):
    """The no-JS path: a plain HTML form posting ``csrf_token`` urlencoded.

    Both directions on the identical endpoint and body shape — a correct
    field value must pass the middleware, a wrong one must be refused.
    That pairing is what distinguishes "the fallback works" from "the
    endpoint accepts everything" and from "the endpoint is broken".
    """
    client = authenticated_client
    token = _mint(client)

    with _no_token(client):
        good = client.post(
            "/settings/save_settings",
            data={"csrf_token": token, "llm.temperature": "0.4"},
            follow_redirects=False,
        )
        bad = client.post(
            "/settings/save_settings",
            data={"csrf_token": "0" * 64, "llm.temperature": "0.4"},
            follow_redirects=False,
        )

    assert not _csrf_refused(good), (
        "the no-JS urlencoded fallback was refused with a VALID csrf_token "
        f"field — every plain HTML form 403s: {good.status_code} "
        f"{good.text[:200]}"
    )
    assert _csrf_refused(bad), (
        "CONTROL FAILED — a wrong csrf_token field value was NOT refused, so "
        "the form field is not actually being validated: "
        f"{bad.status_code} {bad.text[:200]}"
    )


def test_logout_is_csrf_protected_end_to_end(authenticated_client):
    """Forced logout is a real (if low-severity) CSRF target.

    It is also the one mutating control rendered on EVERY page, so it is
    the one most likely to be exempted "to make the header work".
    """
    client = authenticated_client
    with _no_token(client):
        bare = client.post("/auth/logout", follow_redirects=False)
    assert _csrf_refused(bare), (
        f"POST /auth/logout accepted a tokenless request: {bare.status_code} "
        f"{bare.text[:200]}"
    )

    token = _mint(client)
    good = client.post(
        "/auth/logout",
        headers={"X-CSRFToken": token},
        follow_redirects=False,
    )
    assert good.status_code == 302, (
        "CONTROL FAILED — logout WITH a valid token did not succeed either, "
        f"so the rejection above proves nothing: {good.status_code} "
        f"{good.text[:200]}"
    )


# ---------------------------------------------------------------------------
# A5. The method-shaped hole: a mutating route on a "safe" verb
# ---------------------------------------------------------------------------


@pytest.mark.parametrize(
    "path",
    [
        "/library/api/rag/index-all",
        "/library/api/collections/123/index",
    ],
)
def test_mutating_routes_are_not_reachable_on_an_unguarded_verb(
    authenticated_client,
    path,
):
    """A state-changing route must not sit on a verb the middleware skips.

    ``CSRFMiddleware`` challenges ``_UNSAFE_METHODS`` and nothing else, so
    a route that mutates state from a GET is outside the check entirely.
    An allowlist of state-changing GET paths does NOT close that: the
    middleware compares ``scope["path"]`` as a string while the router
    matches it with a ``$``-anchored regex, and the two disagree — uvicorn
    percent-decodes the target into ``scope["path"]``, so
    ``/library/api/rag/index-all%0A`` arrives with a trailing newline that
    Python's ``$`` still matches but a set-membership test does not. The
    allowlist misses, the router dispatches, the handler runs: fail-open.
    The verb is the only spelling-proof key, so ``index-all`` is a POST.

    CONTROL: the same path on its real verb IS challenged. This pins
    that the route exists and enforces CSRF, so a route shipped
    CSRF-exempt cannot satisfy it.
    """
    client = authenticated_client

    with _no_token(client):
        challenged = client.post(path)
    assert _csrf_refused(challenged), (
        f"CONTROL FAILED — POST {path} was not "
        "challenged for CSRF, so the route is either gone or unguarded and "
        f"the assertion below proves nothing: {challenged.status_code} "
        f"{challenged.text[:200]}"
    )

    stale = client.get(path + "?force_reindex=true")
    assert stale.status_code == 405, (
        f"GET {path} still routes. A mutating route on "
        "a verb the middleware skips is reachable from same-site markup "
        "(a link, an <img>/<script> src, an iframe) riding the victim's "
        f"session cookie: {stale.status_code} {stale.text[:200]}"
    )


def test_index_all_post_without_a_token_is_refused():
    """The bulk re-index trigger is challenged like any other mutation."""
    outcome = drive(
        "POST",
        "/library/api/rag/index-all",
        session={"username": "victim"},
    )
    assert outcome.is_csrf_rejection, (
        "POST /library/api/rag/index-all was not challenged for CSRF: "
        f"{outcome!r}"
    )


def test_collection_index_post_without_a_token_is_refused():
    """The per-collection trigger is challenged like any other mutation."""
    outcome = drive(
        "POST",
        "/library/api/collections/123/index",
        session={"username": "victim"},
    )
    assert outcome.is_csrf_rejection, (
        "POST /library/api/collections/123/index was not challenged for "
        f"CSRF: {outcome!r}"
    )


def test_collection_index_post_with_a_valid_token_passes_the_middleware():
    """CONTROL for the refusal above — a valid token passes the middleware.

    ``drive()`` wires the middleware around a sentinel with no router, so
    this pins the middleware's pass-through, not the collection-index
    handler.
    """
    token = "a" * 64
    outcome = drive(
        "POST",
        "/library/api/collections/123/index",
        session={"username": "victim", "_csrf_token": token},
        headers={"X-CSRFToken": token},
    )
    assert outcome.reached, (
        "CONTROL FAILED — the tokened POST was refused too, so the "
        f"rejection above proves nothing: {outcome!r}"
    )


def test_index_all_post_with_a_valid_token_passes_the_middleware():
    """CONTROL for the refusal above — a valid token passes the middleware.

    ``drive()`` wires the middleware around a sentinel with no router, so
    this pins the middleware's pass-through, not the index-all handler. A
    fetch-based SSE client sends X-CSRFToken like it does for every other
    POST, and must still get through the middleware.
    """
    token = "a" * 64
    outcome = drive(
        "POST",
        "/library/api/rag/index-all",
        session={"username": "victim", "_csrf_token": token},
        headers={"X-CSRFToken": token},
    )
    assert outcome.reached, (
        "CONTROL FAILED — the same request WITH a valid token was refused "
        f"too, so the rejection above proves nothing: {outcome!r}"
    )


def test_model_get_reads_the_cache_and_only_a_tokened_post_discovers(
    authenticated_client,
):
    """A legacy GET cannot clear models; a tokened POST reaches discovery.

    The plain GET at the end still builds the provider options from the
    current policy, but never discovers providers: GET reads only the
    model cache.
    """
    client = authenticated_client
    path = "/settings/api/available-models"
    with (
        patch(
            "local_deep_research.llm.providers.get_discovered_provider_options",
            return_value=[],
        ) as provider_options,
        patch(
            "local_deep_research.llm.providers.discover_providers",
            return_value={},
        ) as model_providers,
    ):
        legacy = client.get(f"{path}?force_refresh=true")
        assert legacy.status_code == 405, legacy.text
        assert legacy.headers["allow"] == "POST"
        provider_options.assert_not_called()
        model_providers.assert_not_called()

        with _no_token(client):
            bare = client.post(path)
        assert _csrf_refused(bare), bare.text
        provider_options.assert_not_called()
        model_providers.assert_not_called()

        refreshed = client.post(f"{path}?force_refresh=true")
        assert refreshed.status_code == 200, refreshed.text
        assert "provider_options" in refreshed.json()
        assert provider_options.call_count == 1
        model_providers.assert_called_once_with()

        cached = client.get(path)
        assert cached.status_code == 200, cached.text
        assert provider_options.call_count == 2
        model_providers.assert_called_once_with()


@pytest.mark.parametrize(
    "query",
    ["?refresh_provider=openai", "?force_refresh=true", ""],
    ids=["targeted", "force-refresh-flag", "cache-first-without-query"],
)
def test_model_refresh_requires_post_and_csrf(authenticated_client, query):
    """Every POST to the model endpoint can call providers with the user's
    stored keys and rewrite their cached ``ProviderModel`` rows: a targeted
    refresh, ``force_refresh=true`` (which used to delete every cached row
    on a plain GET), and a cache miss on a POST without a query. So each is
    a mutation: a GET carrying ``refresh_provider`` or
    ``force_refresh=true`` answers 405 with ``Allow: POST`` without
    reaching discovery, a POST with no token is refused by the middleware,
    and a tokened POST reaches the handler. A plain GET read, and a GET
    with ``force_refresh=false``, still reach the handler (controls); they
    read only the model cache.
    """
    client = authenticated_client
    path = "/settings/api/available-models"
    with patch(
        "local_deep_research.web.routers.settings."
        "_resolve_model_discovery_policy",
        side_effect=RuntimeError("discovery reached"),
    ) as discover:
        if query:
            legacy = client.get(f"{path}{query}")
            assert legacy.status_code == 405, legacy.text
            assert legacy.headers["allow"] == "POST"
            discover.assert_not_called()

        with _no_token(client):
            bare = client.post(f"{path}{query}")
        assert _csrf_refused(bare), bare.text
        discover.assert_not_called()

        client.post(f"{path}{query}")
        assert discover.call_count == 1, (
            "CONTROL FAILED — the tokened POST never reached the handler"
        )

        client.get(path)
        assert discover.call_count == 2, (
            "CONTROL FAILED — a plain GET read no longer reaches the handler"
        )

        client.get(f"{path}?force_refresh=false")
        assert discover.call_count == 3, (
            "CONTROL FAILED — a GET with force_refresh=false is a read and "
            "must still reach the handler"
        )


def test_the_challenge_does_not_depend_on_how_the_path_is_spelled():
    """No path spelling can move a mutation outside the middleware.

    This is the property a per-path allowlist cannot provide and the verb
    can. Each spelling below either reaches the router or does not; what
    matters is that the middleware challenges it either way, so the
    middleware can never be the component that lets one through. The
    percent-decoded trailing newline is the one that defeated an exact
    path match; the sub-path prefixes are the same failure under a
    ``root_path`` deployment.

    CONTROL: the identical spellings on a method the middleware skips are
    NOT refused, so "everything is refused" cannot satisfy this.
    """
    base = "/library/api/rag/index-all"
    spellings = [
        base,
        base + "\n",  # uvicorn's unquote() of a trailing %0A
        base + "\r",
        base + "/",
        base + " ",
        base + "%0A",  # literal, if something upstream did not decode it
        "/ldr" + base,  # root_path sub-path deployment
    ]

    for path in spellings:
        outcome = drive("POST", path, session={"username": "victim"})
        assert outcome.is_csrf_rejection, (
            f"a tokenless POST spelled {path!r} was not challenged — the "
            f"middleware's enforcement key is path-sensitive again: "
            f"{outcome!r}"
        )

    for path in spellings:
        outcome = drive("GET", path, session={"username": "victim"})
        assert outcome.reached, (
            "CONTROL FAILED — a GET at {!r} was refused as well, so the "
            "refusals above are not evidence that the METHOD is what "
            "decides: {!r}".format(path, outcome)
        )


# ---------------------------------------------------------------------------
# B. USABILITY — does every legitimate UI call send the token, under the
#    exact name the middleware reads?
#
#    Middleware contract: header ``X-CSRFToken`` (or the ``X-CSRF-Token``
#    alias), or a ``csrf_token`` field in an urlencoded body. Anything else
#    is a 403 on a button that looks fine.
# ---------------------------------------------------------------------------

_FORM_RE = re.compile(r"<form\b[^>]*>(.*?)</form>", re.I | re.S)
_FIELD_RE = re.compile(r"""name\s*=\s*["']csrf_token["']""", re.I)
_METHOD_ATTR_RE = re.compile(r"""method\s*=\s*["']?(\w+)""", re.I)
_SCRIPT_RE = re.compile(r"<script\b[^>]*>(.*?)</script>", re.S | re.I)

# JS helpers that inject the header for their caller. Verified against the
# real source by test_the_shared_fetch_wrapper_really_injects_the_header.
_TOKEN_INJECTING_WRAPPERS = (
    "fetchWithErrorHandling(",
    "postJSON(",
    "putJSON(",
    "apiRequest(",
    "deleteRequest(",
    "postRequest(",
)
_RAW_CALLERS = ("fetch(", "$.ajax(", "XMLHttpRequest")

_JS_METHOD_RE = re.compile(
    r"""method\s*:\s*['"](POST|PUT|PATCH|DELETE)['"]""", re.I
)
# The exact spellings CSRFMiddleware accepts, and nothing else.
_TOKEN_SPELLINGS = ("X-CSRFToken", "X-CSRF-Token", "csrf_token")
# How far around a call site to look for the token. Wide enough for the
# real code (headers objects are commonly assembled ~20 lines above the
# fetch); the negative control below proves it is not so wide that an
# unrelated file-level mention rescues a genuinely bare call.
_BACK, _FWD = 1600, 1200


def extract_forms(html: str, label: str) -> list[dict]:
    """Every <form>...</form> in one document."""
    found = []
    for m in _FORM_RE.finditer(html):
        open_tag = m.group(0)[: m.group(0).index(">") + 1]
        method_m = _METHOD_ATTR_RE.search(open_tag)
        found.append(
            {
                "file": label,
                "line": html[: m.start()].count("\n") + 1,
                "method": (method_m.group(1) if method_m else "GET").upper(),
                "has_token_field": bool(_FIELD_RE.search(m.group(1))),
            }
        )
    return found


def extract_js_call_sites(src: str, label: str) -> list[dict]:
    """Every state-changing fetch/ajax call site in one JS source."""
    found = []
    for m in _JS_METHOD_RE.finditer(src):
        head = src[max(0, m.start() - _BACK) : m.start()]
        caller, at = None, -1
        for name in _TOKEN_INJECTING_WRAPPERS + _RAW_CALLERS:
            i = head.rfind(name)
            if i > at:
                at, caller = i, name
        window = src[max(0, m.start() - _BACK) : m.end() + _FWD]
        found.append(
            {
                "file": label,
                "line": src[: m.start()].count("\n") + 1,
                "method": m.group(1).upper(),
                "caller": caller,
                "via_wrapper": caller in _TOKEN_INJECTING_WRAPPERS,
                "sends_token": any(s in window for s in _TOKEN_SPELLINGS),
            }
        )
    return found


def _unprotected(sites: list[dict]) -> list[dict]:
    return [s for s in sites if not (s["via_wrapper"] or s["sends_token"])]


# --- extractor controls -----------------------------------------------------

_GOOD_FORM = """
<form method="POST" action="/settings/save_settings">
  <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
  <input name="q">
</form>
"""
_BAD_FORM = """
<form method="POST" action="/settings/save_settings">
  <input name="q">
</form>
"""
_WRONG_NAME_FORM = """
<form method="POST" action="/x">
  <input type="hidden" name="_csrf" value="{{ csrf_token() }}"/>
</form>
"""
_GOOD_JS = """
async function save(v) {
  const csrfToken = document.querySelector('meta[name="csrf-token"]').content;
  await fetch('/x', {method: 'POST', headers: {'X-CSRFToken': csrfToken},
                     body: JSON.stringify(v)});
}
"""
_GOOD_JS_VIA_WRAPPER = """
async function save(v) {
  return window.api.fetchWithErrorHandling('/x', {
      method: 'POST', body: JSON.stringify(v)});
}
"""
_BAD_JS = """
async function save(v) {
  await fetch('/x', {method: 'POST', body: JSON.stringify(v)});
}
"""
_WRONG_HEADER_JS = """
async function save(v) {
  const t = document.querySelector('meta[name="csrf-token"]').content;
  await fetch('/x', {method: 'POST', headers: {'X-XSRF-TOKEN': t},
                     body: JSON.stringify(v)});
}
"""


def test_form_extractor_controls():
    """POSITIVE: it finds a real form and sees its token field.
    NEGATIVE: it flags a form with no field, and one whose field is named
    something the middleware does not read.
    """
    good = extract_forms(_GOOD_FORM, "ctl")
    assert len(good) == 1 and good[0]["method"] == "POST"
    assert good[0]["has_token_field"], (
        "extractor missed a correctly-named csrf_token field — it would "
        "report every real form as broken"
    )

    bad = extract_forms(_BAD_FORM, "ctl")
    assert len(bad) == 1 and not bad[0]["has_token_field"], (
        "extractor did NOT flag a POST form with no csrf_token field — it "
        "cannot detect the defect it exists to detect"
    )

    wrong = extract_forms(_WRONG_NAME_FORM, "ctl")
    assert len(wrong) == 1 and not wrong[0]["has_token_field"], (
        "extractor accepted the field name '_csrf', which the middleware "
        "never reads (it looks for exactly 'csrf_token')"
    )


def test_js_extractor_controls():
    """POSITIVE: an explicit header and a wrapper call are both accepted.
    NEGATIVE: a bare fetch, and one using a header name the middleware does
    not read, are both flagged.
    """
    assert not _unprotected(extract_js_call_sites(_GOOD_JS, "ctl")), (
        "extractor flagged a call site that DOES send X-CSRFToken"
    )
    wrapped = extract_js_call_sites(_GOOD_JS_VIA_WRAPPER, "ctl")
    assert len(wrapped) == 1 and wrapped[0]["via_wrapper"], (
        "extractor did not recognise window.api.fetchWithErrorHandling as "
        "the token-injecting wrapper"
    )

    bare = extract_js_call_sites(_BAD_JS, "ctl")
    assert len(bare) == 1 and _unprotected(bare), (
        "extractor did NOT flag a bare state-changing fetch — its clean "
        "report on the real frontend would be meaningless"
    )
    wrong = extract_js_call_sites(_WRONG_HEADER_JS, "ctl")
    assert _unprotected(wrong), (
        "extractor accepted the header name 'X-XSRF-TOKEN'; CSRFMiddleware "
        "reads only X-CSRFToken / X-CSRF-Token, so that call would 403"
    )


def test_the_shared_fetch_wrapper_really_injects_the_header():
    """The load-bearing premise of ``via_wrapper``.

    Most of the frontend delegates to ``api.js::fetchWithErrorHandling``.
    If that helper ever stops adding X-CSRFToken, ~80 call sites break at
    once and the wrapper-aware sweep below would still report them clean.
    """
    src = (JS_ROOT / "services" / "api.js").read_text(encoding="utf-8")
    start = src.index("async function fetchWithErrorHandling")
    body = src[start : start + 2000]
    assert "getCsrfToken()" in body and "'X-CSRFToken'" in body, (
        "api.js::fetchWithErrorHandling no longer injects the X-CSRFToken "
        "header; every call site that relies on it now 403s"
    )
    getter = src[src.index("function getCsrfToken") :][:400]
    assert 'meta[name="csrf-token"]' in getter, (
        'getCsrfToken no longer reads the <meta name="csrf-token"> tag '
        "that base.html renders"
    )


def test_base_template_publishes_the_token_to_the_frontend():
    """The single source every JS call site reads from."""
    base = (TEMPLATE_ROOT / "base.html").read_text(encoding="utf-8")
    assert re.search(
        r'<meta\s+name="csrf-token"\s+content="\{\{\s*csrf_token\(\)\s*\}\}"',
        base,
    ), (
        "base.html no longer renders <meta name='csrf-token'>; "
        "api.js::getCsrfToken() returns '' and every state-changing fetch "
        "in the app 403s"
    )


def test_every_post_form_in_templates_carries_the_token_field():
    """Static sweep over every shipped Jinja template."""
    forms: list[dict] = []
    for path in sorted(TEMPLATE_ROOT.rglob("*.html")):
        forms.extend(
            extract_forms(
                path.read_text(encoding="utf-8", errors="replace"),
                str(path.relative_to(WEB)),
            )
        )

    assert len(forms) >= MIN_TOTAL_FORMS, (
        f"only {len(forms)} <form> elements extracted from "
        f"{TEMPLATE_ROOT} (floor {MIN_TOTAL_FORMS}) — the extractor broke"
    )
    posts = [f for f in forms if f["method"] in _UNSAFE_METHODS]
    assert len(posts) >= MIN_POST_FORMS, (
        f"only {len(posts)} state-changing forms found (floor "
        f"{MIN_POST_FORMS}) — the method attribute parse broke"
    )

    missing = [f for f in posts if not f["has_token_field"]]
    assert not missing, (
        "these shipped forms POST without a csrf_token hidden field; the "
        "middleware refuses them with a 403 the user sees as 'the button "
        f"does nothing': {missing}"
    )


def test_every_state_changing_js_call_site_supplies_the_token():
    """Static sweep over static/js/**.js."""
    sites: list[dict] = []
    files = sorted(JS_ROOT.rglob("*.js"))
    for path in files:
        sites.extend(
            extract_js_call_sites(
                path.read_text(encoding="utf-8", errors="replace"),
                str(path.relative_to(WEB)),
            )
        )

    assert len(files) >= 40 and len(sites) >= MIN_JS_CALL_SITES, (
        f"scanned {len(files)} JS files and found {len(sites)} "
        f"state-changing call sites (floor {MIN_JS_CALL_SITES}) — the "
        "extractor broke, and an empty sweep passes vacuously"
    )

    bad = _unprotected(sites)
    assert not bad, (
        "these state-changing frontend calls send neither an X-CSRFToken "
        "header nor go through the token-injecting api.js wrapper; each is "
        f"a 403 on a working-looking control: {bad}"
    )


def test_every_state_changing_inline_template_script_supplies_the_token():
    """Same sweep over <script> blocks inlined in the templates.

    These are the easiest to miss in a port: they are not linted with the
    JS bundle and they cannot use module imports, so they hand-roll their
    own fetch calls.
    """
    sites: list[dict] = []
    for path in sorted(TEMPLATE_ROOT.rglob("*.html")):
        html = path.read_text(encoding="utf-8", errors="replace")
        for block in _SCRIPT_RE.finditer(html):
            sites.extend(
                extract_js_call_sites(
                    block.group(1), str(path.relative_to(WEB))
                )
            )

    assert len(sites) >= MIN_TEMPLATE_INLINE_CALL_SITES, (
        f"only {len(sites)} inline state-changing call sites found (floor "
        f"{MIN_TEMPLATE_INLINE_CALL_SITES}) — the <script> extraction broke"
    )
    bad = _unprotected(sites)
    assert not bad, (
        f"these inline-template calls mutate state without the token: {bad}"
    )