~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(web): apply response security headers consistently (#7065)
LearningCircuitd84a8a9c3b8d
diff --git a/changelog.d/+security-response-headers.bugfix.md b/changelog.d/+security-response-headers.bugfix.mdnew file mode 100644index 0000000000..9d460a5c4a--- /dev/null+++ b/changelog.d/+security-response-headers.bugfix.md@@ -0,0 +1 @@+Restore HSTS on HTTPS error responses and emit security headers once, preserving SSE no-transform directives, static caching, and repeated cookie headers.diff --git a/src/local_deep_research/web/fastapi_app.py b/src/local_deep_research/web/fastapi_app.pyindex 6509308c1a..cdedca1928 100644--- a/src/local_deep_research/web/fastapi_app.py+++ b/src/local_deep_research/web/fastapi_app.py@@ -688,41 +688,69 @@ class SecurityHeadersMiddleware:def __init__(self, app):self.app = app+ @classmethod+ def response_headers(+ cls, *, path: str, scheme: str, status: int+ ) -> list[tuple[bytes, bytes]]:+ """Apply the same header policy inside and outside the middleware."""+ headers = cls.unconditional_headers()+ if scheme == "https":+ headers.append(+ (+ b"strict-transport-security",+ b"max-age=31536000; includeSubDomains",+ )+ )+ legacy_asset = path.startswith("/redirect-static/") and status in (+ 200,+ 206,+ 304,+ )+ if not path.startswith("/static/") and not legacy_asset:+ headers.extend(cls.cache_headers())+ return headers+async def __call__(self, scope, receive, send):if scope["type"] != "http":await self.app(scope, receive, send)returnpath = scope.get("path", "")- is_secure = scope.get("scheme") == "https"async def send_wrapper(message):if message["type"] == "http.response.start":- headers = list(message.get("headers", []))-- # Security headers- headers.extend(self.unconditional_headers())-- # HSTS for secure connections- if is_secure:- headers.append(+ additions = self.response_headers(+ path=path,+ scheme=scope.get("scheme", "http"),+ status=message["status"],+ )+ # Keep the route's streaming directive while replacing its+ # caching policy. SSE must remain untransformed by proxies,+ # and dynamic responses must still be no-store.+ if any(+ name.lower() == b"cache-control"+ and any(+ directive.strip().lower() == b"no-transform"+ for directive in value.split(b",")+ )+ for name, value in message.get("headers", [])+ ):+ additions = [(- b"strict-transport-security",- b"max-age=31536000; includeSubDomains",+ name,+ value + b", no-transform"+ if name == b"cache-control"+ else value,)- )-- # Successful legacy assets use the shared static cache- # policy. Errors retain the existing no-store headers.- legacy_asset = path.startswith("/redirect-static/") and (- message["status"] in (200, 206, 304)- )- if not path.startswith("/static/") and not legacy_asset:- headers.extend(self.cache_headers())-- # Remove server header- headers = [(k, v) for k, v in headers if k.lower() != b"server"]-+ for name, value in additions+ ]+ replaced = {name for name, _ in additions} | {b"server"}+ headers = [+ (name, value)+ for name, value in message.get("headers", [])+ if name.lower() not in replaced+ ]+ headers.extend(additions)message = {**message, "headers": headers}await send(message)@@ -1472,9 +1500,10 @@ def _register_exception_handlers(app: FastAPI) -> None:status_code=500,headers={name.decode(): value.decode()- for name, value in (- SecurityHeadersMiddleware.unconditional_headers()- + SecurityHeadersMiddleware.cache_headers()+ for name, value in SecurityHeadersMiddleware.response_headers(+ path=request.url.path,+ scheme=request.url.scheme,+ status=500,)},)@@ -2064,9 +2093,10 @@ def _setup_rate_limiting(app: FastAPI) -> None:# tests/web/test_middleware_order_and_headers.py.headers={name.decode(): value.decode()- for name, value in (- SecurityHeadersMiddleware.unconditional_headers()- + SecurityHeadersMiddleware.cache_headers()+ for name, value in SecurityHeadersMiddleware.response_headers(+ path=request.url.path,+ scheme=request.url.scheme,+ status=429,)},)diff --git a/tests/web/test_security_headers_matrix.py b/tests/web/test_security_headers_matrix.pyindex 9a9c506606..06effb640b 100644--- a/tests/web/test_security_headers_matrix.py+++ b/tests/web/test_security_headers_matrix.py@@ -17,33 +17,18 @@ PRESENT with the right VALUES on the ordinary response classes:``tests/security/test_cookie_security.py`` — Secure/HttpOnly/SameSiteand the remember-me Max-Age rules on the session cookie.-Three questions those files do not ask, all of which this branch gets-wrong somewhere, are asked here.+These tests cover header multiplicity, responses built outside the+security middleware, and cookie handling.1. HOW MANY TIMES does each header appear?- ``SecurityHeadersMiddleware.__call__`` does ``headers.extend(...)``.- Flask's ``after_request`` (main's ``security/security_headers.py``)- did ``response.headers[name] = value``, which REPLACES. So where a- handler sets one of these headers itself, main emitted one header and- this branch emits two. Every sibling assertion above reads- ``resp.headers[name]``, which JOINS duplicates with ", " and is- therefore blind to the difference on any route none of them happens- to drive. Four production routes set their own ``Cache-Control`` and- one also sets its own ``X-Content-Type-Options`` (see- ``_PRODUCTION_SELF_STAMPING_SITES``).--2. Do the responses built OUTSIDE the middleware get HSTS?- Two handlers cannot use the middleware and restamp by hand from- ``unconditional_headers() + cache_headers()``: the catch-all- ``Exception`` handler (wired into ``ServerErrorMiddleware``, outside- every ``add_middleware`` layer) and the ``RateLimitExceeded`` handler- (``SlowAPIMiddleware`` is outermost). ``unconditional_headers()``- deliberately excludes HSTS because it is scheme-conditional — and- neither handler adds it back, so over HTTPS those two responses are- the only ones in the app with no ``Strict-Transport-Security``.- ``test_middleware_stack_contracts.py`` compares the five exit paths- over plain HTTP only, where HSTS is absent from all of them, so the- drift is invisible there.+ Handler-stamped streaming responses must still carry each managed+ header once. Raw header assertions detect duplicate fields that a+ joined header lookup could overlook.++2. Do responses built OUTSIDE the middleware follow the same policy?+ The catch-all exception and rate-limit handlers bypass the security+ middleware. Their HTTPS responses need HSTS, and their cache behavior+ must follow the same static-route rules as ordinary responses.3. Which cookies exist at all, and can one appear on a response thatnever passes ``SecureCookieMiddleware``?@@ -308,15 +293,7 @@ class TestStampedHeadersAppearExactlyOnce:@pytest.mark.parametrize("label", sorted(_RESPONSE_CLASSES))def test_hsts_exactly_once_over_https(self, label, https_client):- """Two of these responses are built outside the middleware and- fail this; they are excluded here and get their own dedicated- xfail tests in ``TestHstsOnOutOfBandResponses`` so the mechanism- is stated once rather than hidden behind a skip."""- if label == "500 catch-all handler":- pytest.skip(- "known defect, pinned by TestHstsOnOutOfBandResponses::"- "test_catch_all_500_carries_hsts_over_https"- )+ """Include the catch-all response built outside the middleware."""resp = _produce(https_client, label)assert len(_raw(resp, _HSTS)) == 1, (f"{label}: expected exactly one {_HSTS} over https, got "@@ -387,7 +364,7 @@ class TestHandlerStampedHeadersAreNotDuplicated:)def test_the_probe_response_really_is_handler_stamped(self, http_client):- """Positive control: without this, the two xfails below could+ """Positive control: without this, the two regression tests below could'fail' for the boring reason that the probe stopped stamping."""resp = http_client.get("/__sec_matrix__/self-stamped")assert resp.status_code == 200@@ -401,17 +378,6 @@ class TestHandlerStampedHeadersAreNotDuplicated:assert "no-store" in ", ".join(_raw(resp, "cache-control"))assert "nosniff" in ", ".join(_raw(resp, "x-content-type-options"))- @pytest.mark.xfail(- strict=True,- reason=(- "DEFECT: SecurityHeadersMiddleware appends instead of "- "replacing, so a handler that stamps X-Content-Type-Options "- "itself (routers/research.py::export_research_logs) ships two "- "of them. Flask's after_request assigned into "- "response.headers, which replaced. Remove this marker when "- "the middleware drops any header it is about to stamp."- ),- )def test_x_content_type_options_not_duplicated(self, http_client):resp = http_client.get("/__sec_matrix__/self-stamped")assert resp.status_code == 200@@ -421,15 +387,6 @@ class TestHandlerStampedHeadersAreNotDuplicated:f"X-Content-Type-Options headers: {values!r}")- @pytest.mark.xfail(- strict=True,- reason=(- "DEFECT: same append-vs-replace bug on Cache-Control. Four "- "production routes set their own (research.py export, and "- "the rag.py/library.py SSE endpoints), so every one of them "- "emits two Cache-Control headers where main emitted one."- ),- )def test_cache_control_not_duplicated(self, http_client):resp = http_client.get("/__sec_matrix__/self-stamped")assert resp.status_code == 200@@ -445,26 +402,26 @@ class TestHandlerStampedHeadersAreNotDuplicated:# ----------------------------------------------------------------------------def _https_scope():+def _request_scope(*, scheme="https", path="/api/v1/health"):"""A realistic HTTPS ASGI scope for the live app."""return {"type": "http","http_version": "1.1","method": "GET",- "scheme": "https",- "path": "/api/v1/health",- "raw_path": b"/api/v1/health",+ "scheme": scheme,+ "path": path,+ "raw_path": path.encode(),"query_string": b"","root_path": "","headers": [(b"host", b"testserver")],"client": ("10.9.9.9", 41234),- "server": ("testserver", 443),+ "server": ("testserver", 443 if scheme == "https" else 80),"app": _live_app,"state": {},}-def _call_rate_limit_handler():+def _call_rate_limit_handler(*, scheme="https", path="/api/v1/health"):"""Invoke the app's REGISTERED RateLimitExceeded handler.Driven directly rather than by exhausting a real bucket: the@@ -474,7 +431,7 @@ def _call_rate_limit_handler():already calls this handler directly for the same reason."""handler = _live_app.exception_handlers[RateLimitExceeded]- request = Request(_https_scope())+ request = Request(_request_scope(scheme=scheme, path=path))exc = RateLimitExceeded.__new__(RateLimitExceeded)exc.limit = Noneresult = handler(request, exc)@@ -491,7 +448,7 @@ class TestHstsOnOutOfBandResponses:def test_hsts_is_really_stamped_on_the_paths_that_do_pass_the_middleware(self, https_client):- """Positive control for the two xfails below: if HSTS were absent+ """Positive control for the two regression tests below: if HSTS were absentapp-wide, 'the 500 has no HSTS' would prove nothing."""resp = _produce(https_client, "404 JSON branch")assert _raw(resp, _HSTS) == ["max-age=31536000; includeSubDomains"], (@@ -499,19 +456,6 @@ class TestHstsOnOutOfBandResponses:"class cannot distinguish an out-of-band gap from a global one")- @pytest.mark.xfail(- strict=True,- reason=(- "DEFECT: the catch-all Exception handler restamps from "- "unconditional_headers() + cache_headers(), and HSTS is in "- "neither (it is scheme-conditional and the middleware adds it "- "separately). That handler is wired into ServerErrorMiddleware, "- "outside every add_middleware layer, so over HTTPS an unhandled "- "500 is the only response in the app with no HSTS. Flask's "- "after_request covered 500s too. Remove this marker when the "- "handler adds HSTS for request.url.scheme == 'https'."- ),- )def test_catch_all_500_carries_hsts_over_https(self, https_client):resp = _produce(https_client, "500 catch-all handler")assert resp.json() == {"error": "Server error"}@@ -519,16 +463,6 @@ class TestHstsOnOutOfBandResponses:f"unhandled 500 over https carries HSTS {_raw(resp, _HSTS)!r}")- @pytest.mark.xfail(- strict=True,- reason=(- "DEFECT: same root cause on the 429. SlowAPIMiddleware is "- "outermost, so its 429 never passes SecurityHeadersMiddleware "- "and the handler restamps from the same two HSTS-free lists. "- "It also never looks at the request scheme, so it cannot add "- "HSTS conditionally as things stand."- ),- )def test_rate_limit_429_carries_hsts_over_https(self):response = _call_rate_limit_handler()assert response.status_code == 429, (@@ -587,18 +521,6 @@ class TestStaticCarveOutAppliesToErrorResponsesToo:assert _raw(resp, "cache-control") == []assert len(_raw(resp, "content-security-policy")) == 1- @pytest.mark.xfail(- strict=True,- reason=(- "DEFECT (low severity, parity): the catch-all Exception "- "handler appends cache_headers() unconditionally, ignoring the "- "/static/ carve-out that the middleware — and main's Flask "- "after_request, which applied `if not request.path.startswith("- "'/static/')` to EVERY response including 500s — honour. So a "- "500 raised while serving a static asset is stamped no-store "- "while the 200 for the same path is not."- ),- )def test_static_500_is_not_stamped_no_store(self, static_error_client):resp = static_error_client.get("/static/boom")assert resp.status_code == 500@@ -851,3 +773,156 @@ finally:assert observed["openapi_url"] == (None if expected_testing else "/openapi.json")++[email protected]("scheme", ["http", "https"])+def test_header_replacement_preserves_repeated_cookie_fields(scheme):+ """ASGI headers are case-insensitive; Set-Cookie remains multi-valued."""+ messages = []+ cookies = [+ (b"set-cookie", b"first=one; HttpOnly"),+ (b"set-cookie", b"second=two; HttpOnly"),+ ]++ async def downstream(scope, receive, send):+ await send(+ {+ "type": "http.response.start",+ "status": 200,+ "headers": [+ (b"X-Content-Type-Options", b"incorrect"),+ (b"x-content-type-options", b"incorrect-again"),+ (b"Cache-Control", b"public"),+ (b"SERVER", b"probe"),+ *cookies,+ ],+ }+ )+ await send({"type": "http.response.body", "body": b"body-control"})++ async def send(message):+ messages.append(message)++ async def receive():+ return {"type": "http.request", "body": b"", "more_body": False}++ asyncio.run(+ SecurityHeadersMiddleware(downstream)(+ _request_scope(scheme=scheme), receive, send+ )+ )+ headers = messages[0]["headers"]+ assert [(k, v) for k, v in headers if k.lower() == b"set-cookie"] == cookies+ assert [+ v for k, v in headers if k.lower() == b"x-content-type-options"+ ] == [b"nosniff"]+ assert [v for k, v in headers if k.lower() == b"cache-control"] == [+ b"no-store, no-cache, must-revalidate, max-age=0"+ ]+ assert not any(k.lower() == b"server" for k, _ in headers)+ assert [+ v for k, v in headers if k.lower() == b"strict-transport-security"+ ] == ([b"max-age=31536000; includeSubDomains"] if scheme == "https" else [])+ assert messages[1] == {+ "type": "http.response.body",+ "body": b"body-control",+ }++[email protected]("scheme", ["http", "https"])+ "path,cache_expected",+ [+ ("/api/v1/health", True),+ ("/static/probe.css", False),+ ("/redirect-static/probe.js", True),+ ],+)+def test_rate_limit_handler_uses_the_scheme_and_static_cache_policy(+ scheme, path, cache_expected+):+ response = _call_rate_limit_handler(scheme=scheme, path=path)+ assert response.status_code == 429+ assert _raw(response, _HSTS) == (+ ["max-age=31536000; includeSubDomains"] if scheme == "https" else []+ )+ assert len(_raw(response, "content-security-policy")) == 1+ assert len(_raw(response, "cache-control")) == int(cache_expected)+ if cache_expected:+ assert "no-store" in _raw(response, "cache-control")[0]+++ "route_headers,no_transform",+ [+ pytest.param([], False, id="no-route-policy"),+ pytest.param(+ [(b"Cache-Control", b"no-cache, no-transform")],+ True,+ id="sse-policy",+ ),+ pytest.param(+ [(b"CACHE-CONTROL", b" public, max-age=600, \tNO-TRANSFORM ")],+ True,+ id="mixed-case",+ ),+ pytest.param(+ [+ (b"Cache-Control", b"public, max-age=600"),+ (b"cache-control", b"s-maxage=300, no-transform"),+ ],+ True,+ id="repeated-fields",+ ),+ pytest.param(+ [(b"cache-control", b"no-transform, NO-TRANSFORM, public")],+ True,+ id="repeated-directive",+ ),+ pytest.param(+ [(b"cache-control", b'public, x-no-transform, ext="no-transform"')],+ False,+ id="unrelated-directives",+ ),+ ],+)+def test_cache_replacement_preserves_only_no_transform(+ route_headers, no_transform+):+ """The stream directive survives; cacheable policy and duplicates do not."""+ messages = []++ async def downstream(scope, receive, send):+ await send(+ {+ "type": "http.response.start",+ "status": 200,+ "headers": route_headers,+ }+ )+ await send({"type": "http.response.body", "body": b"stream-control"})++ async def send(message):+ messages.append(message)++ async def receive():+ return {"type": "http.request", "body": b"", "more_body": False}++ asyncio.run(+ SecurityHeadersMiddleware(downstream)(_request_scope(), receive, send)+ )+ assert len(messages) == 2+ assert messages[0]["status"] == 200+ cache_control = [+ value+ for name, value in messages[0]["headers"]+ if name.lower() == b"cache-control"+ ]+ expected = b"no-store, no-cache, must-revalidate, max-age=0"+ if no_transform:+ expected += b", no-transform"+ assert cache_control == [expected]+ assert messages[1] == {+ "type": "http.response.body",+ "body": b"stream-control",+ }diff --git a/tests/web/test_streaming_and_sse_contracts.py b/tests/web/test_streaming_and_sse_contracts.pyindex efb8926a5c..8c39e0f1d8 100644--- a/tests/web/test_streaming_and_sse_contracts.py+++ b/tests/web/test_streaming_and_sse_contracts.py@@ -1236,11 +1236,9 @@ def test_sse_headers_and_framing_survive_the_full_middleware_stack("""The header fences prove the ROUTE sets them; this proves the STACKkeeps them.- ``SecurityHeadersMiddleware`` appends its own- ``Cache-Control: no-store, ...`` to every non-``/static/`` response, so an- SSE response ends up carrying two Cache-Control values. Asserting on- ``no-transform`` (which only the route sets) checks the route's value- survived rather than being replaced.+ ``SecurityHeadersMiddleware`` combines its no-store policy with the+ route's ``no-transform`` directive in one Cache-Control field. The+ route's directive must survive so proxies cannot transform the stream."""response = stream_client.get(f"{PROBE_PREFIX}/sse")