Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(security): enforce response cap before buffering (#7016)

LearningCircuit0537f4c3190a
diff --git a/changelog.d/7016.bugfix.md b/changelog.d/7016.bugfix.md
new file mode 100644
index 0000000000..c65ede927e
--- /dev/null
+++ b/changelog.d/7016.bugfix.md
@@ -0,0 +1,4 @@
+Journal-quality dataset downloads (DOAJ, OpenAlex, OpenAlex Institutions,
+JabRef and the predatory-journal lists) now retry when the connection breaks
+partway through a response body. Previously such a failure was raised on the
+first attempt without a retry.
diff --git a/changelog.d/7016.security.md b/changelog.d/7016.security.md
new file mode 100644
index 0000000000..f3b53ce0ae
--- /dev/null
+++ b/changelog.d/7016.security.md
@@ -0,0 +1,5 @@
+Web fetches now enforce the response-size limit while the body is read.
+Previously a response without a `Content-Length` header, such as a chunked one,
+could be read into memory in full before the limit was checked, and a
+compressed response with a small `Content-Length` could decompress past the
+limit unchecked.
diff --git a/src/local_deep_research/research_library/downloaders/base.py b/src/local_deep_research/research_library/downloaders/base.py
index d9e01e107e..02b5eb89e0 100644
--- a/src/local_deep_research/research_library/downloaders/base.py
+++ b/src/local_deep_research/research_library/downloaders/base.py
@@ -314,18 +314,19 @@ class BaseDownloader(ABC):
return False
def _ensure_decoded_body_cap(self, response: requests.Response) -> None:
- """Install the running decoded-byte body guard when SafeSession
- would not have.
+ """Install a second decoded-byte body guard for a valid, under-cap
+ ``Content-Length``.
``SafeSession.send`` installs ``_install_body_guard`` (which
bounds ``raw.read()``/``read_chunked()`` at
- ``MAX_RESPONSE_SIZE``, counting already-decoded bytes) only when
- ``Content-Length`` is absent or unparseable; a *valid, under-cap*
- ``Content-Length`` gets no body guard at all, so a gzip response
- with a small header can transparently decode past the cap while
- ``response.content`` / ``response.text`` is read. Install the same
- guard for exactly that gap so the decoded body of *this* response
- is bounded regardless of framing.
+ ``MAX_RESPONSE_SIZE``, counting already-decoded bytes) on every
+ response it accepts, including one whose *valid, under-cap*
+ ``Content-Length`` says nothing about how far a gzip body can
+ decode while ``response.content`` / ``response.text`` is read.
+ Install the same guard again for that case, so the decoded body
+ of *this* response stays bounded without relying on the session's
+ guard. For a ``SafeSession`` response the two counters count the
+ same bytes against the same threshold, so the second is harmless.
This only bounds the body of the final response that callers of
this method go on to read (``response.content`` in
@@ -362,11 +363,12 @@ class BaseDownloader(ABC):
len(sizes) == 1
and next(iter(sizes)) <= safe_requests.MAX_RESPONSE_SIZE
):
- # The one case SafeSession leaves unguarded: a single valid
- # Content-Length within the cap, where the decoded body can
- # still exceed the header (e.g. gzip). Installing the guard on
- # top of an existing one (negative or duplicate CL corner
- # cases) is harmless — both counters share the same threshold.
+ # A single valid Content-Length within the cap, where the
+ # decoded body can still exceed the header (e.g. gzip).
+ # SafeSession.send has guarded this response already; stacking
+ # a second guard here (or on the negative or duplicate CL
+ # corner cases) is harmless — both counters share the same
+ # threshold.
safe_requests._install_body_guard(response)
def _download_pdf(
@@ -424,15 +426,15 @@ class BaseDownloader(ABC):
else:
request_headers = dict(self.session.headers)
- # Make the request. stream=True is load-bearing: with
- # stream=False, requests.Session.send consumes the body
- # before returning, before _ensure_decoded_body_cap could
- # wrap the reads, so a small-Content-Length gzip body would
- # decode whole and unbounded. Streamed, the guard installed
- # below is what bounds the decoded read of
- # response.content at MAX_RESPONSE_SIZE. That covers the
- # final response only; SafeSession discards intermediate
- # redirect bodies unread.
+ # Make the request. stream=True leaves the body unread
+ # until the status check below, so a non-200 body is closed
+ # unread and _ensure_decoded_body_cap can wrap the reads
+ # before response.content. SafeSession.send bounds the
+ # decoded read at MAX_RESPONSE_SIZE either way; with
+ # stream=False it would read the body before returning,
+ # before that second guard could wrap anything. That covers
+ # the final response only; SafeSession discards
+ # intermediate redirect bodies unread.
response = self.session.get(
url,
headers=request_headers,
diff --git a/src/local_deep_research/research_library/downloaders/html.py b/src/local_deep_research/research_library/downloaders/html.py
index 7aca65646c..18d9334085 100644
--- a/src/local_deep_research/research_library/downloaders/html.py
+++ b/src/local_deep_research/research_library/downloaders/html.py
@@ -420,10 +420,10 @@ class HTMLDownloader(BaseDownloader):
# Non-HTML 200 is an answer, not a transport failure: do
# not call on_transport_failure. Bound the body before any
# recovery read, including the magic-byte check below.
- # _ensure_decoded_body_cap installs the decoded-byte guard
- # for the small-Content-Length gzip gap SafeSession leaves
- # unguarded; the read that follows is then capped at
- # MAX_RESPONSE_SIZE.
+ # _ensure_decoded_body_cap adds the decoded-byte guard for
+ # a small-Content-Length gzip body on top of the one
+ # SafeSession.send installs; the read that follows is
+ # then capped at MAX_RESPONSE_SIZE.
self._ensure_decoded_body_cap(response)
# Read the bounded body HERE, inside this method's outer
# try, instead of letting _is_pdf_content() do it. That
@@ -546,11 +546,12 @@ class HTMLDownloader(BaseDownloader):
try:
# stream=True keeps requests.Session.send from consuming the
# body before _ensure_decoded_body_cap can wrap the reads, so
- # the guard bounds the decoded read of response.text at
- # MAX_RESPONSE_SIZE even for a valid under-cap Content-Length
- # (the gap SafeSession leaves unguarded). This bounds the final
- # response only: with allow_redirects=True, SafeSession discards
- # each intermediate redirect body unread.
+ # that guard also bounds the decoded read of response.text at
+ # MAX_RESPONSE_SIZE for a valid under-cap Content-Length (a
+ # second guard: SafeSession.send installs one on every response
+ # it accepts). This bounds the final response only: with
+ # allow_redirects=True, SafeSession discards each intermediate
+ # redirect body unread.
response = self.session.get(
url,
timeout=self.timeout,
diff --git a/src/local_deep_research/security/safe_requests.py b/src/local_deep_research/security/safe_requests.py
index abb8ef0da0..f1be58320d 100644
--- a/src/local_deep_research/security/safe_requests.py
+++ b/src/local_deep_research/security/safe_requests.py
@@ -6,8 +6,12 @@ Wraps requests library to add SSRF protection and security best practices.
import datetime
import email.utils
+import functools
+import inspect
import threading
import time
+import types
+import weakref
from typing import TYPE_CHECKING, Any, Optional
from urllib.parse import urljoin
@@ -51,62 +55,234 @@ _MAX_REDIRECTS = 10
# constant so a future requests rename only requires a one-line edit.
_DEFAULT_REQUESTS_UA_PREFIX = "python-requests"
+# Instance attribute of a size-guarded raw: the list of _BodyGuard objects
+# its class-level read()/read_chunked() count against. See
+# _install_body_guard.
+_BODY_GUARDS_ATTR = "_ldr_body_guards"
-def _install_body_guard(response: requests.Response) -> None:
- """Install a bounded reader that enforces MAX_RESPONSE_SIZE.
- Wraps response.raw.read() to track cumulative bytes and raise
- ValueError if MAX_RESPONSE_SIZE is exceeded during body consumption.
- This transparently protects both streamed (.iter_content) and
- non-streamed (.text, .json(), .content) access patterns.
-
- A Transfer-Encoding: chunked body never reaches raw.read():
- urllib3's stream() dispatches it to read_chunked(), which pulls the
- socket through raw._fp. That generator is wrapped against the same
- counter so the two paths share one budget.
+class _BodyGuard:
+ """One installed body guard: the byte budget of a single response.
- Always installs — callers (currently only _check_response_size)
- are responsible for deciding when to call this function.
+ It holds the response only through a weak reference and never refers
+ to ``raw``, so storing it on ``raw`` forms no reference cycle.
"""
- raw = response.raw
- bytes_read = 0
- def count(data: bytes) -> None:
- nonlocal bytes_read
- bytes_read += len(data)
- if bytes_read > MAX_RESPONSE_SIZE:
- response.close()
+ __slots__ = ("bytes_read", "response_ref")
+
+ def __init__(self, response: requests.Response) -> None:
+ self.bytes_read = 0
+ self.response_ref = weakref.ref(response)
+
+ def limit(self, amt):
+ """Return the read size to request, at most one byte past the cap.
+
+ raw.read(None) can consume an entire hostile response in one call.
+ Limit even direct raw readers to one byte past the remaining cap,
+ enough to detect overflow without buffering the whole body.
+ """
+ allowance = MAX_RESPONSE_SIZE - self.bytes_read + 1
+ if amt is None or amt < 0 or amt > allowance:
+ return allowance
+ return amt
+
+ def count(self, raw: Any, data: bytes) -> None:
+ """Add ``data`` to the budget; past the cap, close and raise.
+
+ Raises:
+ ValueError: Once more than MAX_RESPONSE_SIZE bytes were read.
+ """
+ self.bytes_read += len(data)
+ if self.bytes_read > MAX_RESPONSE_SIZE:
+ owner = self.response_ref()
+ if owner is not None:
+ owner.close()
+ else:
+ # Only raw is still referenced. Close it the way
+ # Response.close() closes an unread body.
+ raw.close()
+ release_conn = getattr(raw, "release_conn", None)
+ if release_conn is not None:
+ release_conn()
raise ValueError(
- f"Response body too large: >{bytes_read} bytes "
- f"(max {MAX_RESPONSE_SIZE}, Content-Length absent or invalid)"
+ f"Response body too large: >{self.bytes_read} bytes "
+ f"(max {MAX_RESPONSE_SIZE})"
)
- # A response with no transport (raw=None — e.g. a synthetic
- # Response fed into resolve_redirects by tests or tooling) has no
- # body to buffer, so there is nothing to guard; skip rather than
- # crash. Real transport-backed responses always have raw set.
- if raw is None:
- return
+def _is_plain_method(cls: type, name: str) -> bool:
+ """Whether ``cls`` defines ``name`` as an ordinary instance method."""
+ attr = inspect.getattr_static(cls, name, None)
+ return isinstance(attr, (types.FunctionType, types.MethodDescriptorType))
+
+
[email protected]_cache(maxsize=32)
+def _size_guarded_class(base: type[Any], with_read_chunked: bool) -> type:
+ """Return a subclass of ``base`` whose reads count against its guards.
+
+ Its read(), and read_chunked() when ``with_read_chunked``, call
+ ``base``'s method (looked up when called) and count the result
+ against every _BodyGuard in the instance's ``_BODY_GUARDS_ATTR``
+ list, in install order, as nested wrappers would. It adds no slots,
+ so an instance of ``base`` can be switched to it by assigning
+ ``__class__``. Cached, so the raws of one class share one subclass.
+ """
+
+ class SizeGuarded(base):
+ __slots__ = ()
+ _ldr_size_guarded = True
+
+ def read(self, amt=None, *args, **kwargs):
+ guards = tuple(getattr(self, _BODY_GUARDS_ATTR))
+ # The last guard installed clamps first, as the outermost
+ # wrapper would.
+ for guard in reversed(guards):
+ amt = guard.limit(amt)
+ data = super().read(amt, *args, **kwargs)
+ for guard in guards:
+ guard.count(self, data)
+ return data
+
+ if with_read_chunked:
+
+ def read_chunked(self, amt=None, *args, **kwargs):
+ guards = tuple(getattr(self, _BODY_GUARDS_ATTR))
+ for guard in reversed(guards):
+ amt = guard.limit(amt)
+ for chunk in super().read_chunked(amt, *args, **kwargs):
+ for guard in guards:
+ guard.count(self, chunk)
+ yield chunk
+
+ SizeGuarded.__name__ = "".join(("SizeGuarded", base.__name__))
+ SizeGuarded.__qualname__ = SizeGuarded.__name__
+ return SizeGuarded
+
+
+def _class_guard_list(raw: Any) -> Optional[list]:
+ """Return the guard list that raw's class-level reads count against.
+
+ On the first guard this switches raw's class to the
+ ``_size_guarded_class`` of its class and gives raw an empty list. It
+ returns None, leaving raw unchanged, when raw cannot take that
+ subclass: read() or read_chunked() is set on the instance, the class
+ does not define read() as an ordinary method or defines
+ read_chunked() as something else, raw has no instance ``__dict__``
+ or its class overrides ``__setattr__``, or the ``__class__``
+ assignment is refused. That is the case for test doubles such as
+ mocks, ``SimpleNamespace`` and ``io.BytesIO``. requests' own
+ adapters always return urllib3 responses, which take the subclass.
+ """
+ cls: type = type(raw)
+ if getattr(cls, "_ldr_size_guarded", False):
+ existing: list = vars(raw).setdefault(_BODY_GUARDS_ATTR, [])
+ return existing
+ # A class with its own __setattr__ (a mock, for one) may not treat the
+ # __class__ assignment below as a plain attribute write.
+ if any("__setattr__" in vars(klass) for klass in cls.__mro__[:-1]):
+ return None
+ try:
+ own = vars(raw)
+ except TypeError:
+ return None
+ if "read" in own or "read_chunked" in own:
+ return None
+ if not _is_plain_method(cls, "read"):
+ return None
+ with_read_chunked = (
+ inspect.getattr_static(cls, "read_chunked", None) is not None
+ )
+ if with_read_chunked and not _is_plain_method(cls, "read_chunked"):
+ return None
+ try:
+ raw.__class__ = _size_guarded_class(cls, with_read_chunked)
+ except TypeError:
+ return None
+ guards: list = []
+ setattr(raw, _BODY_GUARDS_ATTR, guards)
+ return guards
+
+
+def _wrap_instance_reads(raw: Any, guard: _BodyGuard) -> None:
+ """Wrap raw's read() and read_chunked() on the instance.
+
+ The fallback for a raw that cannot take ``_size_guarded_class`` (see
+ ``_class_guard_list``). A second guard wraps the first guard's
+ wrappers, so each counts every byte. The wrappers hold ``raw``, as
+ the bound read() they replace does.
+ """
original_read = raw.read
def bounded_read(amt=None, *args, **kwargs):
- data = original_read(amt, *args, **kwargs)
- count(data)
+ data = original_read(guard.limit(amt), *args, **kwargs)
+ guard.count(raw, data)
return data
- raw.read = bounded_read # type: ignore[method-assign]
+ raw.read = bounded_read
original_read_chunked = getattr(raw, "read_chunked", None)
if original_read_chunked is None:
return
def bounded_read_chunked(amt=None, *args, **kwargs):
- for chunk in original_read_chunked(amt, *args, **kwargs):
- count(chunk)
+ for chunk in original_read_chunked(guard.limit(amt), *args, **kwargs):
+ guard.count(raw, chunk)
yield chunk
- raw.read_chunked = bounded_read_chunked # type: ignore[method-assign]
+ raw.read_chunked = bounded_read_chunked
+
+
+def _install_body_guard(response: requests.Response) -> None:
+ """Install a bounded reader that enforces MAX_RESPONSE_SIZE.
+
+ Counts the bytes response.raw.read() returns and raises ValueError
+ once more than MAX_RESPONSE_SIZE have been read during body
+ consumption. This transparently protects both streamed
+ (.iter_content) and non-streamed (.text, .json(), .content) access
+ patterns, including decompressed bodies whose wire Content-Length
+ was small.
+
+ A Transfer-Encoding: chunked body never reaches raw.read():
+ urllib3's stream() dispatches it to read_chunked(), which pulls the
+ socket through raw._fp. That generator is counted against the same
+ budget, so the two paths share one.
+
+ The guard (``_BodyGuard``) refers to the response only weakly and
+ never to ``raw``. A urllib3 response's ``raw`` is switched to a
+ cached subclass of its own class whose read() and read_chunked()
+ apply the guards listed on ``raw`` (``_size_guarded_class``), so
+ nothing stored on ``raw`` refers back to ``raw`` or to the response.
+ A dropped response, its ``raw`` and its cached body are therefore
+ freed by reference counting, however many guards are stacked. A
+ cycle through the guard would keep them, and urllib3's decoder
+ state, in memory until the cyclic garbage collector ran. Because
+ read() is found on the class, a call such as
+ ``safe_get(url, stream=True).raw.read()`` keeps ``raw`` alive while
+ it runs. If the response is gone when the cap is crossed, ``raw`` is
+ closed directly instead.
+
+ A raw that cannot take the subclass, such as a test double (a mock,
+ ``SimpleNamespace`` or ``io.BytesIO``; see ``_class_guard_list``),
+ gets instance-level wrappers instead. They hold ``raw``, as the
+ bound read() they replace does, so such a raw is freed only by the
+ cyclic garbage collector.
+
+ Always installs. The callers (_check_response_size, and
+ BaseDownloader._ensure_decoded_body_cap for a second guard) decide
+ when to call it.
+ """
+ raw = response.raw
+ # Synthetic responses used by callers/tests may have a cached body
+ # without a transport. Real streamed responses always have raw set.
+ if raw is None:
+ return
+
+ guard = _BodyGuard(response)
+ guards = _class_guard_list(raw)
+ if guards is not None:
+ guards.append(guard)
+ else:
+ _wrap_instance_reads(raw, guard)
def _check_response_size(response: requests.Response) -> None:
@@ -118,9 +294,9 @@ def _check_response_size(response: requests.Response) -> None:
(trailing/doubled commas) are filtered before parsing. Non-integer
or negative values cause the header to be treated as absent.
- When Content-Length is absent, unparseable, negative, or consists
- only of commas/whitespace, installs a body guard that enforces
- the size limit during body consumption.
+ Every accepted response gets a body guard. Content-Length counts wire
+ bytes; decompression can make the delivered body much larger, and a
+ server may understate its length.
Must be called before returning a response to the caller. On
rejection the response is closed to avoid leaking the connection.
@@ -158,13 +334,29 @@ def _check_response_size(response: requests.Response) -> None:
raise ValueError(
f"Response too large: {size} bytes (max {MAX_RESPONSE_SIZE})"
)
- # Valid Content-Length within limit — no body guard needed
+ # Valid wire length is only a precheck; decoded bytes still need
+ # the body guard during consumption.
+ _install_body_guard(response)
return
# No Content-Length header at all — install body guard
_install_body_guard(response)
+def _finish_response(
+ response: requests.Response, *, preload: bool
+) -> requests.Response:
+ """Install the size guard before optionally buffering the body."""
+ _check_response_size(response)
+ if preload:
+ try:
+ _ = response.content
+ except Exception:
+ safe_close(response, "response")
+ raise
+ return response
+
+
def _discard_redirect_body(response: requests.Response) -> None:
"""Size-check a redirect response, then drop its body unread.
@@ -326,7 +518,10 @@ def safe_get(
ValueError: If URL fails SSRF validation, or (with
``require_https``) the URL or a redirect hop is not https, or
``verify=False`` is passed, or a response's Content-Length
- (a redirect's included) exceeds MAX_RESPONSE_SIZE or conflicts
+ (a redirect's included) exceeds MAX_RESPONSE_SIZE or conflicts,
+ or the decoded body read before returning exceeds
+ MAX_RESPONSE_SIZE (with ``stream=True`` the caller's own body
+ read raises that instead)
requests.RequestException: If request fails
"""
if require_https:
@@ -370,6 +565,11 @@ def safe_get(
# each redirect target against SSRF rules
caller_wants_redirects = kwargs.pop("allow_redirects", True)
kwargs["allow_redirects"] = False
+ caller_stream = kwargs.get("stream", False)
+ # Requests otherwise buffers the body inside requests.get(), before
+ # _check_response_size can install its guard. Preserve the caller's
+ # normal preloaded-response contract after that guard is in place.
+ kwargs["stream"] = True
# Each hop's plain requests call would still buffer a redirect body
# while preparing Response.next; drop it unread instead.
kwargs["hooks"] = _with_redirect_body_hook(kwargs.get("hooks"))
@@ -470,9 +670,7 @@ def safe_get(
f"Too many redirects ({_MAX_REDIRECTS}) from {_redact(url)}"
)
- _check_response_size(response)
-
- return response
+ return _finish_response(response, preload=not caller_stream)
except requests.Timeout:
logger.warning(
@@ -521,9 +719,12 @@ def safe_post(
Raises:
ValueError: If the URL fails SSRF validation, if a 307/308
redirect would re-send the request body outside the scope of
- the URL the caller addressed, or if a response's
+ the URL the caller addressed, if a response's
Content-Length (a redirect's included) exceeds
- MAX_RESPONSE_SIZE or conflicts
+ MAX_RESPONSE_SIZE or conflicts, or if the decoded body read
+ before returning exceeds MAX_RESPONSE_SIZE (with
+ ``stream=True`` the caller's own body read raises that
+ instead)
requests.RequestException: If request fails
"""
# Validate URL to prevent SSRF
@@ -552,6 +753,8 @@ def safe_post(
# each redirect target against SSRF rules
caller_wants_redirects = kwargs.pop("allow_redirects", True)
kwargs["allow_redirects"] = False
+ caller_stream = kwargs.get("stream", False)
+ kwargs["stream"] = True
# Each hop's plain requests call would still buffer a redirect body
# while preparing Response.next; drop it unread instead.
kwargs["hooks"] = _with_redirect_body_hook(kwargs.get("hooks"))
@@ -665,9 +868,7 @@ def safe_post(
f"Too many redirects ({_MAX_REDIRECTS}) from {_redact(url)}"
)
- _check_response_size(response)
-
- return response
+ return _finish_response(response, preload=not caller_stream)
except requests.Timeout:
logger.warning(
@@ -714,8 +915,10 @@ class SafeSession(requests.Session):
validation, an enabled HTTPS downgrade guard refuses a redirect
or disabled certificate verification, a 307/308 redirect would
re-send the request body outside the scope of the URL the caller
- addressed, or the response Content-Length exceeds
- MAX_RESPONSE_SIZE. Note: ``safe_get``/``safe_post`` also raise
+ addressed, the response Content-Length exceeds MAX_RESPONSE_SIZE,
+ or the decoded body read before a request returns exceeds it
+ (with ``stream=True`` the caller's own body read raises that
+ instead). Note: ``safe_get``/``safe_post`` also raise
``ValueError`` for too-many-redirects, but ``SafeSession`` raises
``requests.TooManyRedirects`` for that case since it delegates
redirect counting to the ``requests`` library.
@@ -960,6 +1163,10 @@ class SafeSession(requests.Session):
# drops it, so that finally waits on collection. A leaked True would
# disarm the body-scope refusal for the rest of the thread.
preparing = self._preparing_next_only
+ caller_stream = kwargs.get("stream", self.stream)
+ # Session.send() preloads non-stream responses before returning.
+ # Force header-first transport, then read through our size guard.
+ kwargs["stream"] = True
try:
with dns_pinning.pinned_request(
request.url or "",
@@ -1016,8 +1223,7 @@ class SafeSession(requests.Session):
response = super().send(request, **kwargs)
finally:
self._redirect_local.preparing_next_only = preparing
- _check_response_size(response)
- return response
+ return _finish_response(response, preload=not caller_stream)
# Exponential backoff schedule (seconds). Kept short: journal-quality
@@ -1041,7 +1247,8 @@ def _parse_retry_after(retry_after_raw: Optional[str]) -> Optional[int]:
Returns ``None`` if the header is missing or unparseable, so the
caller can fall back to the exponential-backoff schedule. Accepts
- both RFC 7231 forms: delay-seconds (integer) and HTTP-date.
+ both RFC 7231 forms: delay-seconds (integer) and HTTP-date. An
+ HTTP-date that names no zone is read as UTC.
"""
if retry_after_raw is None:
return None
@@ -1050,12 +1257,18 @@ def _parse_retry_after(retry_after_raw: Optional[str]) -> Optional[int]:
except ValueError:
try:
retry_dt = email.utils.parsedate_to_datetime(retry_after_raw)
- except (ValueError, TypeError):
+ except (ValueError, TypeError, OverflowError):
+ # OverflowError: a numeric field too large for datetime().
logger.debug(
f"Unparseable Retry-After {retry_after_raw!r}; "
f"using backoff schedule"
)
return None
+ if retry_dt.tzinfo is None:
+ # "-0000", a missing zone and the asctime form parse naive,
+ # and naive minus aware raises TypeError. An HTTP-date is
+ # always UTC (RFC 9110 section 5.6.7).
+ retry_dt = retry_dt.replace(tzinfo=datetime.timezone.utc)
now_utc = datetime.datetime.now(datetime.timezone.utc)
seconds = int((retry_dt - now_utc).total_seconds())
return max(0, min(seconds, _MAX_RETRY_AFTER_SECONDS))
@@ -1086,15 +1299,15 @@ def safe_get_with_retries(
backoff schedule otherwise). SSRF-validation errors (``ValueError``)
and non-retryable HTTP 4xx responses are not retried.
- Without ``consume_body``, only failures raised inside ``safe_get``
- itself (DNS, connect, header timeout, retryable status) trigger a
- retry. The body isn't read until the caller touches ``.content`` /
- ``.text`` / ``.json()``, by which point this wrapper has already
- returned — so a mid-stream S3 hiccup (``ChunkedEncodingError``)
- propagates uncaught. ``consume_body=True`` reads the body inside
- the retry loop so those transient body-read failures are also
- retried. The cached body is still available to the caller via
- ``response.content`` after the wrapper returns.
+ Without ``consume_body``, ``safe_get`` reads the guarded body before
+ it returns (unless the caller passes ``stream=True``), so a body-read
+ failure is raised by ``safe_get`` itself and retried only if it is a
+ ``ConnectionError`` or ``Timeout``; a ``ChunkedEncodingError`` is
+ neither and propagates. With ``consume_body=True``, this wrapper
+ passes ``stream=True`` and reads the body inside the retry loop, so
+ transient mid-body failures such as ``ChunkedEncodingError`` are also
+ retried. The cached body is still available via ``response.content``
+ after return.
Args:
url: Target URL.
@@ -1117,12 +1330,17 @@ def safe_get_with_retries(
is cached on the response.
Raises:
- ValueError: If SSRF validation fails or, with
- ``consume_body=True``, the body-guard rejects an oversized
- response. Retries do not help in either case.
+ ValueError: If SSRF validation fails or the body guard rejects an
+ oversized response. Retries do not help in either case.
requests.RequestException: If every attempt fails.
"""
attempt = 0
+ if consume_body:
+ # Keep body-read failures inside this retry loop. Without
+ # stream=True, safe_get reads the body before returning, and a
+ # ChunkedEncodingError raised there is neither a ConnectionError
+ # nor a Timeout, so it would escape the loop unretried.
+ kwargs["stream"] = True
while True:
try:
response = safe_get(
@@ -1146,9 +1364,12 @@ def safe_get_with_retries(
attempt += 1
continue
- if response.status_code in _RETRYABLE_STATUS_CODES:
- if attempt >= max_retries:
- return response
+ # On the last attempt a retryable status falls through, so with
+ # consume_body its body is read below like any other response's.
+ if (
+ response.status_code in _RETRYABLE_STATUS_CODES
+ and attempt < max_retries
+ ):
parsed = _parse_retry_after(response.headers.get("Retry-After"))
wait = (
parsed
diff --git a/tests/research_library/downloaders/test_base_downloader.py b/tests/research_library/downloaders/test_base_downloader.py
index 55df352972..a4c8c1741e 100644
--- a/tests/research_library/downloaders/test_base_downloader.py
+++ b/tests/research_library/downloaders/test_base_downloader.py
@@ -600,10 +600,9 @@ class TestDownloadPdfBodyCap:
response = requests.Response()
response.status_code = 200
response.headers["Content-Type"] = "application/pdf"
- # A valid, under-cap Content-Length is exactly the case
- # SafeSession leaves unguarded (see
- # security.safe_requests._check_response_size): the abort can
- # only come from BaseDownloader's own guard install.
+ # A valid, under-cap Content-Length. The tests stub session.get,
+ # so SafeSession.send (which guards this framing too) never runs:
+ # the abort can only come from BaseDownloader's own guard install.
response.headers["Content-Length"] = "100"
response.raw = _FiniteBombRaw(chunks)
return response
@@ -668,18 +667,17 @@ class TestFetchHtmlBodyCap:
Twin of TestDownloadPdfBodyCap above for the HTMLDownloader leg
(``_fetch_html_with_final_url``): the same finite bomb, the same valid
- under-cap Content-Length SafeSession leaves unguarded, and the same
- abort contract.
+ under-cap Content-Length with SafeSession.send stubbed out, and the
+ same abort contract.
"""
def _bomb_response(self, chunks: int) -> requests.Response:
response = requests.Response()
response.status_code = 200
response.headers["Content-Type"] = "text/html; charset=utf-8"
- # A valid, under-cap Content-Length is exactly the case
- # SafeSession leaves unguarded (see
- # security.safe_requests._check_response_size): the abort can
- # only come from BaseDownloader's own guard install.
+ # A valid, under-cap Content-Length. The tests stub session.get,
+ # so SafeSession.send (which guards this framing too) never runs:
+ # the abort can only come from BaseDownloader's own guard install.
response.headers["Content-Length"] = "100"
response.url = "https://example.com/page"
# Pin the codec so .text never runs charset detection on the bomb
@@ -736,10 +734,12 @@ class TestDecodedBodyCapOnTheWire:
urllib3's ``stream`` hands it to ``read_chunked``. These serve each
framing from a local socket (127.0.0.1 only) through a real
``SafeSession`` and check both downloader legs stop at the cap on the
- *decoded* bytes. Red when the ``read_chunked`` wrap in
- ``safe_requests._install_body_guard`` is removed (chunked cases), or
- when ``_ensure_decoded_body_cap`` is not called (the small
- ``Content-Length`` cases).
+ *decoded* bytes. Red when the ``read_chunked`` override in
+ ``safe_requests._size_guarded_class``, which the body guard gives a
+ urllib3 response, is removed (chunked cases). The
+ small ``Content-Length`` cases are guarded twice, by
+ ``SafeSession.send`` and by ``_ensure_decoded_body_cap``, so they go
+ red only when both guard installs are removed.
"""
CAP = 64 * 1024
diff --git a/tests/security/test_safe_requests_preload_cap.py b/tests/security/test_safe_requests_preload_cap.py
new file mode 100644
index 0000000000..5157493000
--- /dev/null
+++ b/tests/security/test_safe_requests_preload_cap.py
@@ -0,0 +1,258 @@
+"""The response limit must run before Requests buffers a response body."""
+
+import contextlib
+import gc
+import gzip
+import threading
+import weakref
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+import pytest
+import requests
+import urllib3
+
+from local_deep_research.security import safe_requests
+
+
+def capped_server(monkeypatch):
+ monkeypatch.setattr(safe_requests, "MAX_RESPONSE_SIZE", 1024)
+
+ class Handler(BaseHTTPRequestHandler):
+ protocol_version = "HTTP/1.1"
+
+ def do_GET(self):
+ self._respond()
+
+ def do_POST(self):
+ self._respond()
+
+ def _respond(self):
+ if self.path == "/gzip":
+ # Built per request, so only a test that asks for /gzip
+ # compresses anything.
+ compressed = gzip.compress(b"g" * 2048)
+ self.send_response(200)
+ self.send_header("Content-Encoding", "gzip")
+ self.send_header("Content-Length", str(len(compressed)))
+ self.end_headers()
+ self.wfile.write(compressed)
+ elif self.path == "/small":
+ self.send_response(200)
+ self.send_header("Content-Length", "512")
+ self.end_headers()
+ self.wfile.write(b"s" * 512)
+ elif self.path == "/small-chunked":
+ self.send_response(200)
+ self.send_header("Transfer-Encoding", "chunked")
+ self.end_headers()
+ self.wfile.write(b"200\r\n" + b"s" * 512 + b"\r\n0\r\n\r\n")
+ else:
+ self.send_response(200)
+ self.send_header("Transfer-Encoding", "chunked")
+ self.end_headers()
+ self.wfile.write(b"800\r\n" + b"x" * 2048 + b"\r\n0\r\n\r\n")
+ self.wfile.flush()
+
+ def log_message(self, *args):
+ pass
+
+ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ try:
+ yield f"http://127.0.0.1:{server.server_port}"
+ finally:
+ server.shutdown()
+ server.server_close()
+ thread.join(timeout=2)
+
+
+def _fetch(client, url, **kwargs):
+ if client == "safe_get":
+ return safe_requests.safe_get(url, allow_localhost=True, **kwargs)
+ if client == "safe_post":
+ return safe_requests.safe_post(url, allow_localhost=True, **kwargs)
+ with safe_requests.SafeSession(allow_localhost=True) as session:
+ return session.request(
+ "POST" if client == "session_post" else "GET", url, **kwargs
+ )
+
+
+ "client", ["safe_get", "safe_post", "session_get", "session_post"]
+)
+def test_chunked_body_over_cap_is_rejected_before_default_fetch_returns(
+ capped_server, client
+):
+ with pytest.raises(ValueError, match="too large"):
+ _fetch(client, f"{capped_server}/chunked")
+
+
[email protected]("client", ["safe_get", "session_get"])
+def test_compressed_body_is_capped_by_decoded_size(capped_server, client):
+ with pytest.raises(ValueError, match="too large"):
+ _fetch(client, f"{capped_server}/gzip")
+
+
[email protected]("client", ["safe_get", "safe_post", "session_get"])
+def test_small_body_remains_available(capped_server, client):
+ response = _fetch(client, f"{capped_server}/small")
+ try:
+ assert response.content == b"s" * 512
+ finally:
+ response.close()
+
+
[email protected]("requested", [None, 1_000_000])
+def test_direct_raw_read_is_limited_before_allocation(monkeypatch, requested):
+ monkeypatch.setattr(safe_requests, "MAX_RESPONSE_SIZE", 1024)
+
+ class Raw:
+ consumed = 0
+
+ def read(self, amount=None):
+ self.consumed += amount
+ return b"x" * amount
+
+ def close(self):
+ pass
+
+ raw = Raw()
+ response = requests.Response()
+ response.raw = raw
+ safe_requests._check_response_size(response)
+
+ with pytest.raises(ValueError, match="too large"):
+ response.raw.read(requested)
+ assert raw.consumed == 1025
+
+
+def _cyclic_gc_off():
+ """Leave reference counting as the only way objects are freed."""
+ was_enabled = gc.isenabled()
+ gc.disable()
+ try:
+ yield
+ finally:
+ if was_enabled:
+ gc.enable()
+
+
[email protected]("path", ["/small", "/small-chunked"])
+ "client", ["safe_get", "safe_post", "session_get", "session_post"]
+)
+def test_dropped_guarded_response_is_freed_by_refcount(
+ capped_server, client, path
+):
+ """A guarded response, its ``raw`` and its body are freed once dropped.
+
+ The guard's state is stored on ``response.raw``. If it referred back
+ to the response, the cycle Response -> raw -> guard -> Response would
+ keep every response, with a body of up to MAX_RESPONSE_SIZE, alive
+ until the cyclic garbage collector ran. If it referred back to
+ ``raw`` (a wrapper holding the bound ``raw.read`` it replaces), the
+ cycle raw -> wrapper -> raw would keep ``raw`` and urllib3's decoder
+ state alive the same way. With that collector off, only reference
+ counting can free them.
+ """
+ with _cyclic_gc_off():
+ response = _fetch(client, f"{capped_server}{path}")
+ body = response.content
+ alive = weakref.ref(response)
+ raw_alive = weakref.ref(response.raw)
+ del response
+ freed = alive() is None
+ raw_freed = raw_alive() is None
+ assert body == b"s" * 512
+ assert freed, "the body guard keeps a dropped response alive"
+ assert raw_freed, "the body guard keeps a dropped response's raw alive"
+
+
+def test_dropped_response_with_a_second_guard_is_freed_by_refcount(
+ capped_server,
+):
+ """A second guard, as ``_ensure_decoded_body_cap`` adds, keeps no cycle."""
+ with safe_requests.SafeSession(allow_localhost=True) as session:
+ with _cyclic_gc_off():
+ response = session.get(f"{capped_server}/small", stream=True)
+ safe_requests._install_body_guard(response)
+ body = response.content
+ alive = weakref.ref(response)
+ raw_alive = weakref.ref(response.raw)
+ del response
+ freed = alive() is None
+ raw_freed = raw_alive() is None
+ assert body == b"s" * 512
+ assert freed, "the stacked body guards keep a dropped response alive"
+ assert raw_freed, "the stacked body guards keep a dropped raw alive"
+
+
[email protected]("path", ["/small", "/small-chunked"])
+ "client", ["safe_get", "safe_post", "session_get", "session_post"]
+)
+def test_chained_raw_read_of_a_streamed_response_still_works(
+ capped_server, client, path
+):
+ """``safe_get(url, stream=True).raw.read()`` still returns the body.
+
+ Nothing but the call itself holds ``raw`` here. A guard that kept no
+ reference to ``raw`` but put its reader on the instance would find
+ ``raw`` already freed when that reader ran: for an attribute found on
+ the instance, CPython releases the object it was looked up on before
+ making the call.
+ """
+ with _cyclic_gc_off():
+ body = _fetch(client, f"{capped_server}{path}", stream=True).raw.read()
+ assert body == b"s" * 512
+
+
+ "client", ["safe_get", "safe_post", "session_get", "session_post"]
+)
+def test_chained_raw_read_over_cap_is_rejected(capped_server, client):
+ """A chained direct read past the cap is still rejected."""
+ with _cyclic_gc_off():
+ with pytest.raises(ValueError, match="too large"):
+ _fetch(client, f"{capped_server}/chunked", stream=True).raw.read()
+
+
+def test_guarded_raws_of_one_class_share_one_subclass(capped_server):
+ """Guarding a response does not create a class per response.
+
+ A class is freed only by the cyclic garbage collector, so a new one
+ per response would bring back the garbage this guard avoids.
+ """
+ first = _fetch("safe_get", f"{capped_server}/small")
+ second = _fetch("safe_get", f"{capped_server}/small-chunked")
+ try:
+ assert type(first.raw) is type(second.raw)
+ assert isinstance(first.raw, urllib3.response.HTTPResponse)
+ finally:
+ first.close()
+ second.close()
+
+
+def test_over_cap_read_closes_raw_after_the_response_is_dropped(
+ capped_server,
+):
+ """Code that keeps only ``raw`` still hits the cap, and raw is closed.
+
+ The guard holds the response weakly, so here the response is already
+ gone when the cap is crossed, and the guard closes ``raw`` itself.
+ """
+ with safe_requests.SafeSession(allow_localhost=True) as session:
+ with _cyclic_gc_off():
+ response = session.get(f"{capped_server}/chunked", stream=True)
+ raw = response.raw
+ alive = weakref.ref(response)
+ del response
+ freed = alive() is None
+ with pytest.raises(ValueError, match="too large"):
+ raw.read()
+ closed = raw.closed
+ assert freed, "the body guard keeps a dropped response alive"
+ assert closed, "the guard left the over-cap stream open"
diff --git a/tests/security/test_safe_requests_retries.py b/tests/security/test_safe_requests_retries.py
index afe22dae70..5ce0859829 100644
--- a/tests/security/test_safe_requests_retries.py
+++ b/tests/security/test_safe_requests_retries.py
@@ -1,7 +1,10 @@
"""Exponential-backoff retry wrapper for `safe_get`."""
import email.utils
+import threading
+import time
from datetime import datetime, timedelta, timezone
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from unittest.mock import MagicMock, patch
import pytest
@@ -187,6 +190,98 @@ def test_retry_after_http_date_form_is_parsed():
assert 45 <= slept <= 75, f"expected ~60s, got {slept}s"
[email protected]("form", ["-0000", "no zone", "asctime"])
+def test_retry_after_http_date_without_a_zone_is_read_as_utc(form):
+ """An HTTP-date that names no zone is UTC, and must be waited for.
+
+ ``email.utils`` parses these forms to a naive datetime. Subtracting
+ the aware current time from it raised ``TypeError`` out of the retry
+ loop, so the fetch failed instead of being retried.
+ """
+ future = datetime.now(timezone.utc) + timedelta(seconds=60)
+ if form == "asctime":
+ http_date = time.asctime(time.gmtime(future.timestamp()))
+ else:
+ http_date = email.utils.formatdate(future.timestamp())
+ assert http_date.endswith(" -0000")
+ if form == "no zone":
+ http_date = http_date.removesuffix(" -0000")
+ assert email.utils.parsedate_to_datetime(http_date).tzinfo is None
+ rate_limited = _mock_response(429, headers={"Retry-After": http_date})
+ ok = _mock_response(200)
+ with (
+ patch(
+ "local_deep_research.security.safe_requests.safe_get",
+ side_effect=[rate_limited, ok],
+ ),
+ patch(
+ "local_deep_research.security.safe_requests.time.sleep"
+ ) as mock_sleep,
+ ):
+ resp = safe_get_with_retries(
+ "https://example.com/x", backoff_times=(1, 1, 1)
+ )
+ assert resp is ok
+ slept = mock_sleep.call_args[0][0]
+ # 30s band absorbs clock jitter and test-harness latency.
+ assert 45 <= slept <= 75, f"expected ~60s, got {slept}s"
+
+
+def test_retry_after_http_date_with_an_overflowing_field_uses_schedule():
+ """A date field too large for ``datetime`` falls back to the schedule.
+
+ Depending on the Python version, ``parsedate_to_datetime`` raises
+ ``OverflowError`` directly or wraps it in ``ValueError``. Both must
+ preserve the retry behavior.
+ """
+ huge = "Wed, 21 Oct 2015 99999999999999999999:28:00 GMT"
+ rate_limited = _mock_response(429, headers={"Retry-After": huge})
+ ok = _mock_response(200)
+ with (
+ patch(
+ "local_deep_research.security.safe_requests.safe_get",
+ side_effect=[rate_limited, ok],
+ ),
+ patch(
+ "local_deep_research.security.safe_requests.time.sleep"
+ ) as mock_sleep,
+ ):
+ resp = safe_get_with_retries(
+ "https://example.com/x", backoff_times=(1, 2, 4)
+ )
+ assert resp is ok
+ assert mock_sleep.call_args[0][0] == 1
+
+
[email protected]("parse_error", [OverflowError, ValueError])
+def test_retry_after_date_parser_error_uses_schedule(parse_error):
+ """Keep both parser error forms covered on every supported Python."""
+ http_date = "Wed, 21 Oct 2015 07:28:00 GMT"
+ rate_limited = _mock_response(429, headers={"Retry-After": http_date})
+ ok = _mock_response(200)
+ with (
+ patch(
+ "local_deep_research.security.safe_requests.safe_get",
+ side_effect=[rate_limited, ok],
+ ) as mock_get,
+ patch(
+ "local_deep_research.security.safe_requests.email.utils.parsedate_to_datetime",
+ side_effect=parse_error("invalid date field"),
+ ) as mock_parse,
+ patch(
+ "local_deep_research.security.safe_requests.time.sleep"
+ ) as mock_sleep,
+ ):
+ resp = safe_get_with_retries(
+ "https://example.com/x", backoff_times=(1, 2, 4)
+ )
+ assert resp is ok
+ assert mock_get.call_count == 2
+ mock_parse.assert_called_once_with(http_date)
+ mock_sleep.assert_called_once_with(1)
+ rate_limited.close.assert_called_once()
+
+
def test_retry_after_unparseable_falls_back_to_schedule():
"""Garbage Retry-After falls back to the backoff schedule."""
rate_limited = _mock_response(429, headers={"Retry-After": "garbage"})
@@ -363,3 +458,127 @@ def test_consume_body_returns_cached_body_to_caller():
):
resp = safe_get_with_retries("https://example.com/x", consume_body=True)
assert resp.content == b"hello world"
+
+
+# ---------------------------------------------------------------------------
+# consume_body=True through the real safe_get (nothing patched)
+# ---------------------------------------------------------------------------
+#
+# The consume_body tests above patch safe_get, so they cannot see whether
+# the body is still unread when safe_get returns. safe_get reads the body
+# itself unless stream=True is passed, and a ChunkedEncodingError raised
+# there is neither a ConnectionError nor a Timeout, so it would escape the
+# wrapper unretried. These run the real safe_get over a real requests
+# transport against a server on 127.0.0.1.
+
+_BODY = b"0123456789abcdef" * 256 # 4 KiB
+
+
+def loopback():
+ """Start a 127.0.0.1 server that answers from a script.
+
+ ``serve(script)`` returns ``(url, seen)``. ``script`` names one
+ response per request, in order, and its last entry repeats:
+ ``"ok-length"`` / ``"ok-chunked"`` send ``_BODY`` with a
+ Content-Length or chunked framing, ``"broken-length"`` /
+ ``"broken-chunked"`` promise all of it but send a third and hang up
+ (the client raises ``ChunkedEncodingError`` mid-body), and
+ ``"busy"`` sends ``_BODY`` with status 503. ``seen`` lists the
+ request paths received.
+ """
+ servers = []
+
+ def serve(script):
+ seen = []
+ lock = threading.Lock()
+
+ class Handler(BaseHTTPRequestHandler):
+ protocol_version = "HTTP/1.1"
+
+ def do_GET(self):
+ with lock:
+ seen.append(self.path)
+ kind = script[min(len(seen), len(script)) - 1]
+ self.close_connection = True
+ body = _BODY[: len(_BODY) // 3] if "broken" in kind else _BODY
+ self.send_response(503 if kind == "busy" else 200)
+ self.send_header("Connection", "close")
+ if kind.endswith("chunked"):
+ self.send_header("Transfer-Encoding", "chunked")
+ self.end_headers()
+ self.wfile.write(b"%x\r\n" % len(_BODY) + body)
+ if "broken" not in kind:
+ self.wfile.write(b"\r\n0\r\n\r\n")
+ else:
+ self.send_header("Content-Length", str(len(_BODY)))
+ self.end_headers()
+ self.wfile.write(body)
+ self.wfile.flush()
+
+ def log_message(self, *args):
+ pass
+
+ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ servers.append((server, thread))
+ return f"http://127.0.0.1:{server.server_port}/data", seen
+
+ yield serve
+ for server, thread in servers:
+ server.shutdown()
+ server.server_close()
+ thread.join(timeout=2)
+
+
[email protected]("framing", ["length", "chunked"])
+def test_consume_body_retries_mid_body_break_through_real_safe_get(
+ loopback, framing
+):
+ """A body that breaks mid-read is fetched again and returned.
+
+ Red if the wrapper stops passing ``stream=True``: safe_get then reads
+ the body before returning and the ``ChunkedEncodingError`` escapes
+ after one request.
+ """
+ url, seen = loopback([f"broken-{framing}", f"ok-{framing}"])
+
+ response = safe_get_with_retries(
+ url,
+ allow_localhost=True,
+ timeout=5,
+ consume_body=True,
+ backoff_times=(0,),
+ )
+
+ assert len(seen) == 2
+ assert response.status_code == 200
+ # Read and cached inside the wrapper: closing the transport loses
+ # nothing.
+ response.close()
+ assert response.content == _BODY
+
+
+def test_consume_body_final_retryable_status_returns_read_body(loopback):
+ """The last attempt's 503 comes back with its body already read.
+
+ ``stream=True`` makes safe_get hand over an unread body, so a
+ retryable status returned before the consume_body read would leave
+ it unread on the response.
+ """
+ url, seen = loopback(["busy"])
+
+ response = safe_get_with_retries(
+ url,
+ allow_localhost=True,
+ timeout=5,
+ consume_body=True,
+ max_retries=1,
+ backoff_times=(0,),
+ )
+
+ assert len(seen) == 2
+ assert response.status_code == 503
+ response.close()
+ assert response.content == _BODY
diff --git a/tests/security/test_ssrf_response_size.py b/tests/security/test_ssrf_response_size.py
index 66cc6e6252..36503b5eb9 100644
--- a/tests/security/test_ssrf_response_size.py
+++ b/tests/security/test_ssrf_response_size.py
@@ -4,6 +4,8 @@ Verifies try/except/else pattern ensures ValueError from int() parsing
does not mask ValueError from legitimate size-exceeded checks.
"""
+import socket
+
import requests
import pytest
from unittest.mock import patch, MagicMock
@@ -11,7 +13,7 @@ from unittest.mock import patch, MagicMock
from io import BytesIO
from types import SimpleNamespace
-from local_deep_research.security import ssrf_validator
+from local_deep_research.security import dns_pinning, ssrf_validator
from local_deep_research.security.safe_requests import (
safe_get,
safe_post,
@@ -23,7 +25,26 @@ from local_deep_research.security.safe_requests import (
@pytest.fixture
-def mock_validate_url():
+def mock_validate_url(monkeypatch):
+ # These cases mock HTTP responses, but the real pinning guard still
+ # resolves the target before the mocked request. Keep that lookup local
+ # so a DNS outage cannot prevent the size-limit assertions from running.
+ def resolve(host, port, *args, **kwargs):
+ if host != "example.com":
+ pytest.fail(
+ f"Unexpected DNS lookup in response-size test: {host!r}"
+ )
+ return [
+ (
+ socket.AF_INET,
+ socket.SOCK_STREAM,
+ socket.IPPROTO_TCP,
+ "",
+ ("93.184.216.34", port),
+ )
+ ]
+
+ monkeypatch.setattr(dns_pinning, "_real_getaddrinfo", resolve)
with patch.object(
ssrf_validator,
"validate_url",
@@ -414,6 +435,15 @@ class TestSafeSessionResponseSize:
class TestBodySizeEnforcement:
"""Verify _install_body_guard enforces size limits on responses without CL."""
+ @pytest.fixture(autouse=True)
+ def small_body_limit(self, monkeypatch):
+ # Exercise the same overflow and streaming boundaries without
+ # allocating GiB-sized bodies in each test worker.
+ monkeypatch.setitem(globals(), "MAX_RESPONSE_SIZE", 4096)
+ monkeypatch.setattr(
+ "local_deep_research.security.safe_requests.MAX_RESPONSE_SIZE", 4096
+ )
+
def _make_response(self, body: bytes, headers: dict | None = None):
"""Create a mock response with a real readable raw body."""
resp = MagicMock(spec=requests.Response)
diff --git a/tests/web_search_engines/engines/test_arxiv_pdf_egress_gating.py b/tests/web_search_engines/engines/test_arxiv_pdf_egress_gating.py
index 18ced2022f..9c3cdf7420 100644
--- a/tests/web_search_engines/engines/test_arxiv_pdf_egress_gating.py
+++ b/tests/web_search_engines/engines/test_arxiv_pdf_egress_gating.py
@@ -154,8 +154,9 @@ class TestFullContentUsesGatedPath:
HTML leg answers 404, so ``download_full_text`` falls back
to ``BaseDownloader._download_pdf``, whose response is a crafted
body decoding past ``MAX_RESPONSE_SIZE`` while claiming a valid,
- under-cap ``Content-Length`` — the exact framing SafeSession
- leaves unguarded. The abort can only come from
+ under-cap ``Content-Length``. ``session.get`` is stubbed, so
+ ``SafeSession.send`` (which guards that framing too) never runs:
+ the abort can only come from
``BaseDownloader._ensure_decoded_body_cap``'s guard install;
reverting it lets the finite bomb through as a "successful"
download and the ResponseBodyTooLarge outcome is never recorded.