~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(security): enforce response cap before buffering (#7016)
LearningCircuit0537f4c3190a
diff --git a/changelog.d/7016.bugfix.md b/changelog.d/7016.bugfix.mdnew file mode 100644index 0000000000..c65ede927e--- /dev/null+++ b/changelog.d/7016.bugfix.md@@ -0,0 +1,4 @@+Journal-quality dataset downloads (DOAJ, OpenAlex, OpenAlex Institutions,+JabRef and the predatory-journal lists) now retry when the connection breaks+partway through a response body. Previously such a failure was raised on the+first attempt without a retry.diff --git a/changelog.d/7016.security.md b/changelog.d/7016.security.mdnew file mode 100644index 0000000000..f3b53ce0ae--- /dev/null+++ b/changelog.d/7016.security.md@@ -0,0 +1,5 @@+Web fetches now enforce the response-size limit while the body is read.+Previously a response without a `Content-Length` header, such as a chunked one,+could be read into memory in full before the limit was checked, and a+compressed response with a small `Content-Length` could decompress past the+limit unchecked.diff --git a/src/local_deep_research/research_library/downloaders/base.py b/src/local_deep_research/research_library/downloaders/base.pyindex d9e01e107e..02b5eb89e0 100644--- a/src/local_deep_research/research_library/downloaders/base.py+++ b/src/local_deep_research/research_library/downloaders/base.py@@ -314,18 +314,19 @@ class BaseDownloader(ABC):return Falsedef _ensure_decoded_body_cap(self, response: requests.Response) -> None:- """Install the running decoded-byte body guard when SafeSession- would not have.+ """Install a second decoded-byte body guard for a valid, under-cap+ ``Content-Length``.``SafeSession.send`` installs ``_install_body_guard`` (whichbounds ``raw.read()``/``read_chunked()`` at- ``MAX_RESPONSE_SIZE``, counting already-decoded bytes) only when- ``Content-Length`` is absent or unparseable; a *valid, under-cap*- ``Content-Length`` gets no body guard at all, so a gzip response- with a small header can transparently decode past the cap while- ``response.content`` / ``response.text`` is read. Install the same- guard for exactly that gap so the decoded body of *this* response- is bounded regardless of framing.+ ``MAX_RESPONSE_SIZE``, counting already-decoded bytes) on every+ response it accepts, including one whose *valid, under-cap*+ ``Content-Length`` says nothing about how far a gzip body can+ decode while ``response.content`` / ``response.text`` is read.+ Install the same guard again for that case, so the decoded body+ of *this* response stays bounded without relying on the session's+ guard. For a ``SafeSession`` response the two counters count the+ same bytes against the same threshold, so the second is harmless.This only bounds the body of the final response that callers ofthis method go on to read (``response.content`` in@@ -362,11 +363,12 @@ class BaseDownloader(ABC):len(sizes) == 1and next(iter(sizes)) <= safe_requests.MAX_RESPONSE_SIZE):- # The one case SafeSession leaves unguarded: a single valid- # Content-Length within the cap, where the decoded body can- # still exceed the header (e.g. gzip). Installing the guard on- # top of an existing one (negative or duplicate CL corner- # cases) is harmless — both counters share the same threshold.+ # A single valid Content-Length within the cap, where the+ # decoded body can still exceed the header (e.g. gzip).+ # SafeSession.send has guarded this response already; stacking+ # a second guard here (or on the negative or duplicate CL+ # corner cases) is harmless — both counters share the same+ # threshold.safe_requests._install_body_guard(response)def _download_pdf(@@ -424,15 +426,15 @@ class BaseDownloader(ABC):else:request_headers = dict(self.session.headers)- # Make the request. stream=True is load-bearing: with- # stream=False, requests.Session.send consumes the body- # before returning, before _ensure_decoded_body_cap could- # wrap the reads, so a small-Content-Length gzip body would- # decode whole and unbounded. Streamed, the guard installed- # below is what bounds the decoded read of- # response.content at MAX_RESPONSE_SIZE. That covers the- # final response only; SafeSession discards intermediate- # redirect bodies unread.+ # Make the request. stream=True leaves the body unread+ # until the status check below, so a non-200 body is closed+ # unread and _ensure_decoded_body_cap can wrap the reads+ # before response.content. SafeSession.send bounds the+ # decoded read at MAX_RESPONSE_SIZE either way; with+ # stream=False it would read the body before returning,+ # before that second guard could wrap anything. That covers+ # the final response only; SafeSession discards+ # intermediate redirect bodies unread.response = self.session.get(url,headers=request_headers,diff --git a/src/local_deep_research/research_library/downloaders/html.py b/src/local_deep_research/research_library/downloaders/html.pyindex 7aca65646c..18d9334085 100644--- a/src/local_deep_research/research_library/downloaders/html.py+++ b/src/local_deep_research/research_library/downloaders/html.py@@ -420,10 +420,10 @@ class HTMLDownloader(BaseDownloader):# Non-HTML 200 is an answer, not a transport failure: do# not call on_transport_failure. Bound the body before any# recovery read, including the magic-byte check below.- # _ensure_decoded_body_cap installs the decoded-byte guard- # for the small-Content-Length gzip gap SafeSession leaves- # unguarded; the read that follows is then capped at- # MAX_RESPONSE_SIZE.+ # _ensure_decoded_body_cap adds the decoded-byte guard for+ # a small-Content-Length gzip body on top of the one+ # SafeSession.send installs; the read that follows is+ # then capped at MAX_RESPONSE_SIZE.self._ensure_decoded_body_cap(response)# Read the bounded body HERE, inside this method's outer# try, instead of letting _is_pdf_content() do it. That@@ -546,11 +546,12 @@ class HTMLDownloader(BaseDownloader):try:# stream=True keeps requests.Session.send from consuming the# body before _ensure_decoded_body_cap can wrap the reads, so- # the guard bounds the decoded read of response.text at- # MAX_RESPONSE_SIZE even for a valid under-cap Content-Length- # (the gap SafeSession leaves unguarded). This bounds the final- # response only: with allow_redirects=True, SafeSession discards- # each intermediate redirect body unread.+ # that guard also bounds the decoded read of response.text at+ # MAX_RESPONSE_SIZE for a valid under-cap Content-Length (a+ # second guard: SafeSession.send installs one on every response+ # it accepts). This bounds the final response only: with+ # allow_redirects=True, SafeSession discards each intermediate+ # redirect body unread.response = self.session.get(url,timeout=self.timeout,diff --git a/src/local_deep_research/security/safe_requests.py b/src/local_deep_research/security/safe_requests.pyindex abb8ef0da0..f1be58320d 100644--- a/src/local_deep_research/security/safe_requests.py+++ b/src/local_deep_research/security/safe_requests.py@@ -6,8 +6,12 @@ Wraps requests library to add SSRF protection and security best practices.import datetimeimport email.utils+import functools+import inspectimport threadingimport time+import types+import weakreffrom typing import TYPE_CHECKING, Any, Optionalfrom urllib.parse import urljoin@@ -51,62 +55,234 @@ _MAX_REDIRECTS = 10# constant so a future requests rename only requires a one-line edit._DEFAULT_REQUESTS_UA_PREFIX = "python-requests"+# Instance attribute of a size-guarded raw: the list of _BodyGuard objects+# its class-level read()/read_chunked() count against. See+# _install_body_guard.+_BODY_GUARDS_ATTR = "_ldr_body_guards"-def _install_body_guard(response: requests.Response) -> None:- """Install a bounded reader that enforces MAX_RESPONSE_SIZE.- Wraps response.raw.read() to track cumulative bytes and raise- ValueError if MAX_RESPONSE_SIZE is exceeded during body consumption.- This transparently protects both streamed (.iter_content) and- non-streamed (.text, .json(), .content) access patterns.-- A Transfer-Encoding: chunked body never reaches raw.read():- urllib3's stream() dispatches it to read_chunked(), which pulls the- socket through raw._fp. That generator is wrapped against the same- counter so the two paths share one budget.+class _BodyGuard:+ """One installed body guard: the byte budget of a single response.- Always installs — callers (currently only _check_response_size)- are responsible for deciding when to call this function.+ It holds the response only through a weak reference and never refers+ to ``raw``, so storing it on ``raw`` forms no reference cycle."""- raw = response.raw- bytes_read = 0- def count(data: bytes) -> None:- nonlocal bytes_read- bytes_read += len(data)- if bytes_read > MAX_RESPONSE_SIZE:- response.close()+ __slots__ = ("bytes_read", "response_ref")++ def __init__(self, response: requests.Response) -> None:+ self.bytes_read = 0+ self.response_ref = weakref.ref(response)++ def limit(self, amt):+ """Return the read size to request, at most one byte past the cap.++ raw.read(None) can consume an entire hostile response in one call.+ Limit even direct raw readers to one byte past the remaining cap,+ enough to detect overflow without buffering the whole body.+ """+ allowance = MAX_RESPONSE_SIZE - self.bytes_read + 1+ if amt is None or amt < 0 or amt > allowance:+ return allowance+ return amt++ def count(self, raw: Any, data: bytes) -> None:+ """Add ``data`` to the budget; past the cap, close and raise.++ Raises:+ ValueError: Once more than MAX_RESPONSE_SIZE bytes were read.+ """+ self.bytes_read += len(data)+ if self.bytes_read > MAX_RESPONSE_SIZE:+ owner = self.response_ref()+ if owner is not None:+ owner.close()+ else:+ # Only raw is still referenced. Close it the way+ # Response.close() closes an unread body.+ raw.close()+ release_conn = getattr(raw, "release_conn", None)+ if release_conn is not None:+ release_conn()raise ValueError(- f"Response body too large: >{bytes_read} bytes "- f"(max {MAX_RESPONSE_SIZE}, Content-Length absent or invalid)"+ f"Response body too large: >{self.bytes_read} bytes "+ f"(max {MAX_RESPONSE_SIZE})")- # A response with no transport (raw=None — e.g. a synthetic- # Response fed into resolve_redirects by tests or tooling) has no- # body to buffer, so there is nothing to guard; skip rather than- # crash. Real transport-backed responses always have raw set.- if raw is None:- return+def _is_plain_method(cls: type, name: str) -> bool:+ """Whether ``cls`` defines ``name`` as an ordinary instance method."""+ attr = inspect.getattr_static(cls, name, None)+ return isinstance(attr, (types.FunctionType, types.MethodDescriptorType))++[email protected]_cache(maxsize=32)+def _size_guarded_class(base: type[Any], with_read_chunked: bool) -> type:+ """Return a subclass of ``base`` whose reads count against its guards.++ Its read(), and read_chunked() when ``with_read_chunked``, call+ ``base``'s method (looked up when called) and count the result+ against every _BodyGuard in the instance's ``_BODY_GUARDS_ATTR``+ list, in install order, as nested wrappers would. It adds no slots,+ so an instance of ``base`` can be switched to it by assigning+ ``__class__``. Cached, so the raws of one class share one subclass.+ """++ class SizeGuarded(base):+ __slots__ = ()+ _ldr_size_guarded = True++ def read(self, amt=None, *args, **kwargs):+ guards = tuple(getattr(self, _BODY_GUARDS_ATTR))+ # The last guard installed clamps first, as the outermost+ # wrapper would.+ for guard in reversed(guards):+ amt = guard.limit(amt)+ data = super().read(amt, *args, **kwargs)+ for guard in guards:+ guard.count(self, data)+ return data++ if with_read_chunked:++ def read_chunked(self, amt=None, *args, **kwargs):+ guards = tuple(getattr(self, _BODY_GUARDS_ATTR))+ for guard in reversed(guards):+ amt = guard.limit(amt)+ for chunk in super().read_chunked(amt, *args, **kwargs):+ for guard in guards:+ guard.count(self, chunk)+ yield chunk++ SizeGuarded.__name__ = "".join(("SizeGuarded", base.__name__))+ SizeGuarded.__qualname__ = SizeGuarded.__name__+ return SizeGuarded+++def _class_guard_list(raw: Any) -> Optional[list]:+ """Return the guard list that raw's class-level reads count against.++ On the first guard this switches raw's class to the+ ``_size_guarded_class`` of its class and gives raw an empty list. It+ returns None, leaving raw unchanged, when raw cannot take that+ subclass: read() or read_chunked() is set on the instance, the class+ does not define read() as an ordinary method or defines+ read_chunked() as something else, raw has no instance ``__dict__``+ or its class overrides ``__setattr__``, or the ``__class__``+ assignment is refused. That is the case for test doubles such as+ mocks, ``SimpleNamespace`` and ``io.BytesIO``. requests' own+ adapters always return urllib3 responses, which take the subclass.+ """+ cls: type = type(raw)+ if getattr(cls, "_ldr_size_guarded", False):+ existing: list = vars(raw).setdefault(_BODY_GUARDS_ATTR, [])+ return existing+ # A class with its own __setattr__ (a mock, for one) may not treat the+ # __class__ assignment below as a plain attribute write.+ if any("__setattr__" in vars(klass) for klass in cls.__mro__[:-1]):+ return None+ try:+ own = vars(raw)+ except TypeError:+ return None+ if "read" in own or "read_chunked" in own:+ return None+ if not _is_plain_method(cls, "read"):+ return None+ with_read_chunked = (+ inspect.getattr_static(cls, "read_chunked", None) is not None+ )+ if with_read_chunked and not _is_plain_method(cls, "read_chunked"):+ return None+ try:+ raw.__class__ = _size_guarded_class(cls, with_read_chunked)+ except TypeError:+ return None+ guards: list = []+ setattr(raw, _BODY_GUARDS_ATTR, guards)+ return guards+++def _wrap_instance_reads(raw: Any, guard: _BodyGuard) -> None:+ """Wrap raw's read() and read_chunked() on the instance.++ The fallback for a raw that cannot take ``_size_guarded_class`` (see+ ``_class_guard_list``). A second guard wraps the first guard's+ wrappers, so each counts every byte. The wrappers hold ``raw``, as+ the bound read() they replace does.+ """original_read = raw.readdef bounded_read(amt=None, *args, **kwargs):- data = original_read(amt, *args, **kwargs)- count(data)+ data = original_read(guard.limit(amt), *args, **kwargs)+ guard.count(raw, data)return data- raw.read = bounded_read # type: ignore[method-assign]+ raw.read = bounded_readoriginal_read_chunked = getattr(raw, "read_chunked", None)if original_read_chunked is None:returndef bounded_read_chunked(amt=None, *args, **kwargs):- for chunk in original_read_chunked(amt, *args, **kwargs):- count(chunk)+ for chunk in original_read_chunked(guard.limit(amt), *args, **kwargs):+ guard.count(raw, chunk)yield chunk- raw.read_chunked = bounded_read_chunked # type: ignore[method-assign]+ raw.read_chunked = bounded_read_chunked+++def _install_body_guard(response: requests.Response) -> None:+ """Install a bounded reader that enforces MAX_RESPONSE_SIZE.++ Counts the bytes response.raw.read() returns and raises ValueError+ once more than MAX_RESPONSE_SIZE have been read during body+ consumption. This transparently protects both streamed+ (.iter_content) and non-streamed (.text, .json(), .content) access+ patterns, including decompressed bodies whose wire Content-Length+ was small.++ A Transfer-Encoding: chunked body never reaches raw.read():+ urllib3's stream() dispatches it to read_chunked(), which pulls the+ socket through raw._fp. That generator is counted against the same+ budget, so the two paths share one.++ The guard (``_BodyGuard``) refers to the response only weakly and+ never to ``raw``. A urllib3 response's ``raw`` is switched to a+ cached subclass of its own class whose read() and read_chunked()+ apply the guards listed on ``raw`` (``_size_guarded_class``), so+ nothing stored on ``raw`` refers back to ``raw`` or to the response.+ A dropped response, its ``raw`` and its cached body are therefore+ freed by reference counting, however many guards are stacked. A+ cycle through the guard would keep them, and urllib3's decoder+ state, in memory until the cyclic garbage collector ran. Because+ read() is found on the class, a call such as+ ``safe_get(url, stream=True).raw.read()`` keeps ``raw`` alive while+ it runs. If the response is gone when the cap is crossed, ``raw`` is+ closed directly instead.++ A raw that cannot take the subclass, such as a test double (a mock,+ ``SimpleNamespace`` or ``io.BytesIO``; see ``_class_guard_list``),+ gets instance-level wrappers instead. They hold ``raw``, as the+ bound read() they replace does, so such a raw is freed only by the+ cyclic garbage collector.++ Always installs. The callers (_check_response_size, and+ BaseDownloader._ensure_decoded_body_cap for a second guard) decide+ when to call it.+ """+ raw = response.raw+ # Synthetic responses used by callers/tests may have a cached body+ # without a transport. Real streamed responses always have raw set.+ if raw is None:+ return++ guard = _BodyGuard(response)+ guards = _class_guard_list(raw)+ if guards is not None:+ guards.append(guard)+ else:+ _wrap_instance_reads(raw, guard)def _check_response_size(response: requests.Response) -> None:@@ -118,9 +294,9 @@ def _check_response_size(response: requests.Response) -> None:(trailing/doubled commas) are filtered before parsing. Non-integeror negative values cause the header to be treated as absent.- When Content-Length is absent, unparseable, negative, or consists- only of commas/whitespace, installs a body guard that enforces- the size limit during body consumption.+ Every accepted response gets a body guard. Content-Length counts wire+ bytes; decompression can make the delivered body much larger, and a+ server may understate its length.Must be called before returning a response to the caller. Onrejection the response is closed to avoid leaking the connection.@@ -158,13 +334,29 @@ def _check_response_size(response: requests.Response) -> None:raise ValueError(f"Response too large: {size} bytes (max {MAX_RESPONSE_SIZE})")- # Valid Content-Length within limit — no body guard needed+ # Valid wire length is only a precheck; decoded bytes still need+ # the body guard during consumption.+ _install_body_guard(response)return# No Content-Length header at all — install body guard_install_body_guard(response)+def _finish_response(+ response: requests.Response, *, preload: bool+) -> requests.Response:+ """Install the size guard before optionally buffering the body."""+ _check_response_size(response)+ if preload:+ try:+ _ = response.content+ except Exception:+ safe_close(response, "response")+ raise+ return response++def _discard_redirect_body(response: requests.Response) -> None:"""Size-check a redirect response, then drop its body unread.@@ -326,7 +518,10 @@ def safe_get(ValueError: If URL fails SSRF validation, or (with``require_https``) the URL or a redirect hop is not https, or``verify=False`` is passed, or a response's Content-Length- (a redirect's included) exceeds MAX_RESPONSE_SIZE or conflicts+ (a redirect's included) exceeds MAX_RESPONSE_SIZE or conflicts,+ or the decoded body read before returning exceeds+ MAX_RESPONSE_SIZE (with ``stream=True`` the caller's own body+ read raises that instead)requests.RequestException: If request fails"""if require_https:@@ -370,6 +565,11 @@ def safe_get(# each redirect target against SSRF rulescaller_wants_redirects = kwargs.pop("allow_redirects", True)kwargs["allow_redirects"] = False+ caller_stream = kwargs.get("stream", False)+ # Requests otherwise buffers the body inside requests.get(), before+ # _check_response_size can install its guard. Preserve the caller's+ # normal preloaded-response contract after that guard is in place.+ kwargs["stream"] = True# Each hop's plain requests call would still buffer a redirect body# while preparing Response.next; drop it unread instead.kwargs["hooks"] = _with_redirect_body_hook(kwargs.get("hooks"))@@ -470,9 +670,7 @@ def safe_get(f"Too many redirects ({_MAX_REDIRECTS}) from {_redact(url)}")- _check_response_size(response)-- return response+ return _finish_response(response, preload=not caller_stream)except requests.Timeout:logger.warning(@@ -521,9 +719,12 @@ def safe_post(Raises:ValueError: If the URL fails SSRF validation, if a 307/308redirect would re-send the request body outside the scope of- the URL the caller addressed, or if a response's+ the URL the caller addressed, if a response'sContent-Length (a redirect's included) exceeds- MAX_RESPONSE_SIZE or conflicts+ MAX_RESPONSE_SIZE or conflicts, or if the decoded body read+ before returning exceeds MAX_RESPONSE_SIZE (with+ ``stream=True`` the caller's own body read raises that+ instead)requests.RequestException: If request fails"""# Validate URL to prevent SSRF@@ -552,6 +753,8 @@ def safe_post(# each redirect target against SSRF rulescaller_wants_redirects = kwargs.pop("allow_redirects", True)kwargs["allow_redirects"] = False+ caller_stream = kwargs.get("stream", False)+ kwargs["stream"] = True# Each hop's plain requests call would still buffer a redirect body# while preparing Response.next; drop it unread instead.kwargs["hooks"] = _with_redirect_body_hook(kwargs.get("hooks"))@@ -665,9 +868,7 @@ def safe_post(f"Too many redirects ({_MAX_REDIRECTS}) from {_redact(url)}")- _check_response_size(response)-- return response+ return _finish_response(response, preload=not caller_stream)except requests.Timeout:logger.warning(@@ -714,8 +915,10 @@ class SafeSession(requests.Session):validation, an enabled HTTPS downgrade guard refuses a redirector disabled certificate verification, a 307/308 redirect wouldre-send the request body outside the scope of the URL the caller- addressed, or the response Content-Length exceeds- MAX_RESPONSE_SIZE. Note: ``safe_get``/``safe_post`` also raise+ addressed, the response Content-Length exceeds MAX_RESPONSE_SIZE,+ or the decoded body read before a request returns exceeds it+ (with ``stream=True`` the caller's own body read raises that+ instead). Note: ``safe_get``/``safe_post`` also raise``ValueError`` for too-many-redirects, but ``SafeSession`` raises``requests.TooManyRedirects`` for that case since it delegatesredirect counting to the ``requests`` library.@@ -960,6 +1163,10 @@ class SafeSession(requests.Session):# drops it, so that finally waits on collection. A leaked True would# disarm the body-scope refusal for the rest of the thread.preparing = self._preparing_next_only+ caller_stream = kwargs.get("stream", self.stream)+ # Session.send() preloads non-stream responses before returning.+ # Force header-first transport, then read through our size guard.+ kwargs["stream"] = Truetry:with dns_pinning.pinned_request(request.url or "",@@ -1016,8 +1223,7 @@ class SafeSession(requests.Session):response = super().send(request, **kwargs)finally:self._redirect_local.preparing_next_only = preparing- _check_response_size(response)- return response+ return _finish_response(response, preload=not caller_stream)# Exponential backoff schedule (seconds). Kept short: journal-quality@@ -1041,7 +1247,8 @@ def _parse_retry_after(retry_after_raw: Optional[str]) -> Optional[int]:Returns ``None`` if the header is missing or unparseable, so thecaller can fall back to the exponential-backoff schedule. Accepts- both RFC 7231 forms: delay-seconds (integer) and HTTP-date.+ both RFC 7231 forms: delay-seconds (integer) and HTTP-date. An+ HTTP-date that names no zone is read as UTC."""if retry_after_raw is None:return None@@ -1050,12 +1257,18 @@ def _parse_retry_after(retry_after_raw: Optional[str]) -> Optional[int]:except ValueError:try:retry_dt = email.utils.parsedate_to_datetime(retry_after_raw)- except (ValueError, TypeError):+ except (ValueError, TypeError, OverflowError):+ # OverflowError: a numeric field too large for datetime().logger.debug(f"Unparseable Retry-After {retry_after_raw!r}; "f"using backoff schedule")return None+ if retry_dt.tzinfo is None:+ # "-0000", a missing zone and the asctime form parse naive,+ # and naive minus aware raises TypeError. An HTTP-date is+ # always UTC (RFC 9110 section 5.6.7).+ retry_dt = retry_dt.replace(tzinfo=datetime.timezone.utc)now_utc = datetime.datetime.now(datetime.timezone.utc)seconds = int((retry_dt - now_utc).total_seconds())return max(0, min(seconds, _MAX_RETRY_AFTER_SECONDS))@@ -1086,15 +1299,15 @@ def safe_get_with_retries(backoff schedule otherwise). SSRF-validation errors (``ValueError``)and non-retryable HTTP 4xx responses are not retried.- Without ``consume_body``, only failures raised inside ``safe_get``- itself (DNS, connect, header timeout, retryable status) trigger a- retry. The body isn't read until the caller touches ``.content`` /- ``.text`` / ``.json()``, by which point this wrapper has already- returned — so a mid-stream S3 hiccup (``ChunkedEncodingError``)- propagates uncaught. ``consume_body=True`` reads the body inside- the retry loop so those transient body-read failures are also- retried. The cached body is still available to the caller via- ``response.content`` after the wrapper returns.+ Without ``consume_body``, ``safe_get`` reads the guarded body before+ it returns (unless the caller passes ``stream=True``), so a body-read+ failure is raised by ``safe_get`` itself and retried only if it is a+ ``ConnectionError`` or ``Timeout``; a ``ChunkedEncodingError`` is+ neither and propagates. With ``consume_body=True``, this wrapper+ passes ``stream=True`` and reads the body inside the retry loop, so+ transient mid-body failures such as ``ChunkedEncodingError`` are also+ retried. The cached body is still available via ``response.content``+ after return.Args:url: Target URL.@@ -1117,12 +1330,17 @@ def safe_get_with_retries(is cached on the response.Raises:- ValueError: If SSRF validation fails or, with- ``consume_body=True``, the body-guard rejects an oversized- response. Retries do not help in either case.+ ValueError: If SSRF validation fails or the body guard rejects an+ oversized response. Retries do not help in either case.requests.RequestException: If every attempt fails."""attempt = 0+ if consume_body:+ # Keep body-read failures inside this retry loop. Without+ # stream=True, safe_get reads the body before returning, and a+ # ChunkedEncodingError raised there is neither a ConnectionError+ # nor a Timeout, so it would escape the loop unretried.+ kwargs["stream"] = Truewhile True:try:response = safe_get(@@ -1146,9 +1364,12 @@ def safe_get_with_retries(attempt += 1continue- if response.status_code in _RETRYABLE_STATUS_CODES:- if attempt >= max_retries:- return response+ # On the last attempt a retryable status falls through, so with+ # consume_body its body is read below like any other response's.+ if (+ response.status_code in _RETRYABLE_STATUS_CODES+ and attempt < max_retries+ ):parsed = _parse_retry_after(response.headers.get("Retry-After"))wait = (parseddiff --git a/tests/research_library/downloaders/test_base_downloader.py b/tests/research_library/downloaders/test_base_downloader.pyindex 55df352972..a4c8c1741e 100644--- a/tests/research_library/downloaders/test_base_downloader.py+++ b/tests/research_library/downloaders/test_base_downloader.py@@ -600,10 +600,9 @@ class TestDownloadPdfBodyCap:response = requests.Response()response.status_code = 200response.headers["Content-Type"] = "application/pdf"- # A valid, under-cap Content-Length is exactly the case- # SafeSession leaves unguarded (see- # security.safe_requests._check_response_size): the abort can- # only come from BaseDownloader's own guard install.+ # A valid, under-cap Content-Length. The tests stub session.get,+ # so SafeSession.send (which guards this framing too) never runs:+ # the abort can only come from BaseDownloader's own guard install.response.headers["Content-Length"] = "100"response.raw = _FiniteBombRaw(chunks)return response@@ -668,18 +667,17 @@ class TestFetchHtmlBodyCap:Twin of TestDownloadPdfBodyCap above for the HTMLDownloader leg(``_fetch_html_with_final_url``): the same finite bomb, the same valid- under-cap Content-Length SafeSession leaves unguarded, and the same- abort contract.+ under-cap Content-Length with SafeSession.send stubbed out, and the+ same abort contract."""def _bomb_response(self, chunks: int) -> requests.Response:response = requests.Response()response.status_code = 200response.headers["Content-Type"] = "text/html; charset=utf-8"- # A valid, under-cap Content-Length is exactly the case- # SafeSession leaves unguarded (see- # security.safe_requests._check_response_size): the abort can- # only come from BaseDownloader's own guard install.+ # A valid, under-cap Content-Length. The tests stub session.get,+ # so SafeSession.send (which guards this framing too) never runs:+ # the abort can only come from BaseDownloader's own guard install.response.headers["Content-Length"] = "100"response.url = "https://example.com/page"# Pin the codec so .text never runs charset detection on the bomb@@ -736,10 +734,12 @@ class TestDecodedBodyCapOnTheWire:urllib3's ``stream`` hands it to ``read_chunked``. These serve eachframing from a local socket (127.0.0.1 only) through a real``SafeSession`` and check both downloader legs stop at the cap on the- *decoded* bytes. Red when the ``read_chunked`` wrap in- ``safe_requests._install_body_guard`` is removed (chunked cases), or- when ``_ensure_decoded_body_cap`` is not called (the small- ``Content-Length`` cases).+ *decoded* bytes. Red when the ``read_chunked`` override in+ ``safe_requests._size_guarded_class``, which the body guard gives a+ urllib3 response, is removed (chunked cases). The+ small ``Content-Length`` cases are guarded twice, by+ ``SafeSession.send`` and by ``_ensure_decoded_body_cap``, so they go+ red only when both guard installs are removed."""CAP = 64 * 1024diff --git a/tests/security/test_safe_requests_preload_cap.py b/tests/security/test_safe_requests_preload_cap.pynew file mode 100644index 0000000000..5157493000--- /dev/null+++ b/tests/security/test_safe_requests_preload_cap.py@@ -0,0 +1,258 @@+"""The response limit must run before Requests buffers a response body."""++import contextlib+import gc+import gzip+import threading+import weakref+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer++import pytest+import requests+import urllib3++from local_deep_research.security import safe_requests+++def capped_server(monkeypatch):+ monkeypatch.setattr(safe_requests, "MAX_RESPONSE_SIZE", 1024)++ class Handler(BaseHTTPRequestHandler):+ protocol_version = "HTTP/1.1"++ def do_GET(self):+ self._respond()++ def do_POST(self):+ self._respond()++ def _respond(self):+ if self.path == "/gzip":+ # Built per request, so only a test that asks for /gzip+ # compresses anything.+ compressed = gzip.compress(b"g" * 2048)+ self.send_response(200)+ self.send_header("Content-Encoding", "gzip")+ self.send_header("Content-Length", str(len(compressed)))+ self.end_headers()+ self.wfile.write(compressed)+ elif self.path == "/small":+ self.send_response(200)+ self.send_header("Content-Length", "512")+ self.end_headers()+ self.wfile.write(b"s" * 512)+ elif self.path == "/small-chunked":+ self.send_response(200)+ self.send_header("Transfer-Encoding", "chunked")+ self.end_headers()+ self.wfile.write(b"200\r\n" + b"s" * 512 + b"\r\n0\r\n\r\n")+ else:+ self.send_response(200)+ self.send_header("Transfer-Encoding", "chunked")+ self.end_headers()+ self.wfile.write(b"800\r\n" + b"x" * 2048 + b"\r\n0\r\n\r\n")+ self.wfile.flush()++ def log_message(self, *args):+ pass++ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)+ thread = threading.Thread(target=server.serve_forever, daemon=True)+ thread.start()+ try:+ yield f"http://127.0.0.1:{server.server_port}"+ finally:+ server.shutdown()+ server.server_close()+ thread.join(timeout=2)+++def _fetch(client, url, **kwargs):+ if client == "safe_get":+ return safe_requests.safe_get(url, allow_localhost=True, **kwargs)+ if client == "safe_post":+ return safe_requests.safe_post(url, allow_localhost=True, **kwargs)+ with safe_requests.SafeSession(allow_localhost=True) as session:+ return session.request(+ "POST" if client == "session_post" else "GET", url, **kwargs+ )+++ "client", ["safe_get", "safe_post", "session_get", "session_post"]+)+def test_chunked_body_over_cap_is_rejected_before_default_fetch_returns(+ capped_server, client+):+ with pytest.raises(ValueError, match="too large"):+ _fetch(client, f"{capped_server}/chunked")++[email protected]("client", ["safe_get", "session_get"])+def test_compressed_body_is_capped_by_decoded_size(capped_server, client):+ with pytest.raises(ValueError, match="too large"):+ _fetch(client, f"{capped_server}/gzip")++[email protected]("client", ["safe_get", "safe_post", "session_get"])+def test_small_body_remains_available(capped_server, client):+ response = _fetch(client, f"{capped_server}/small")+ try:+ assert response.content == b"s" * 512+ finally:+ response.close()++[email protected]("requested", [None, 1_000_000])+def test_direct_raw_read_is_limited_before_allocation(monkeypatch, requested):+ monkeypatch.setattr(safe_requests, "MAX_RESPONSE_SIZE", 1024)++ class Raw:+ consumed = 0++ def read(self, amount=None):+ self.consumed += amount+ return b"x" * amount++ def close(self):+ pass++ raw = Raw()+ response = requests.Response()+ response.raw = raw+ safe_requests._check_response_size(response)++ with pytest.raises(ValueError, match="too large"):+ response.raw.read(requested)+ assert raw.consumed == 1025+++def _cyclic_gc_off():+ """Leave reference counting as the only way objects are freed."""+ was_enabled = gc.isenabled()+ gc.disable()+ try:+ yield+ finally:+ if was_enabled:+ gc.enable()++[email protected]("path", ["/small", "/small-chunked"])+ "client", ["safe_get", "safe_post", "session_get", "session_post"]+)+def test_dropped_guarded_response_is_freed_by_refcount(+ capped_server, client, path+):+ """A guarded response, its ``raw`` and its body are freed once dropped.++ The guard's state is stored on ``response.raw``. If it referred back+ to the response, the cycle Response -> raw -> guard -> Response would+ keep every response, with a body of up to MAX_RESPONSE_SIZE, alive+ until the cyclic garbage collector ran. If it referred back to+ ``raw`` (a wrapper holding the bound ``raw.read`` it replaces), the+ cycle raw -> wrapper -> raw would keep ``raw`` and urllib3's decoder+ state alive the same way. With that collector off, only reference+ counting can free them.+ """+ with _cyclic_gc_off():+ response = _fetch(client, f"{capped_server}{path}")+ body = response.content+ alive = weakref.ref(response)+ raw_alive = weakref.ref(response.raw)+ del response+ freed = alive() is None+ raw_freed = raw_alive() is None+ assert body == b"s" * 512+ assert freed, "the body guard keeps a dropped response alive"+ assert raw_freed, "the body guard keeps a dropped response's raw alive"+++def test_dropped_response_with_a_second_guard_is_freed_by_refcount(+ capped_server,+):+ """A second guard, as ``_ensure_decoded_body_cap`` adds, keeps no cycle."""+ with safe_requests.SafeSession(allow_localhost=True) as session:+ with _cyclic_gc_off():+ response = session.get(f"{capped_server}/small", stream=True)+ safe_requests._install_body_guard(response)+ body = response.content+ alive = weakref.ref(response)+ raw_alive = weakref.ref(response.raw)+ del response+ freed = alive() is None+ raw_freed = raw_alive() is None+ assert body == b"s" * 512+ assert freed, "the stacked body guards keep a dropped response alive"+ assert raw_freed, "the stacked body guards keep a dropped raw alive"++[email protected]("path", ["/small", "/small-chunked"])+ "client", ["safe_get", "safe_post", "session_get", "session_post"]+)+def test_chained_raw_read_of_a_streamed_response_still_works(+ capped_server, client, path+):+ """``safe_get(url, stream=True).raw.read()`` still returns the body.++ Nothing but the call itself holds ``raw`` here. A guard that kept no+ reference to ``raw`` but put its reader on the instance would find+ ``raw`` already freed when that reader ran: for an attribute found on+ the instance, CPython releases the object it was looked up on before+ making the call.+ """+ with _cyclic_gc_off():+ body = _fetch(client, f"{capped_server}{path}", stream=True).raw.read()+ assert body == b"s" * 512+++ "client", ["safe_get", "safe_post", "session_get", "session_post"]+)+def test_chained_raw_read_over_cap_is_rejected(capped_server, client):+ """A chained direct read past the cap is still rejected."""+ with _cyclic_gc_off():+ with pytest.raises(ValueError, match="too large"):+ _fetch(client, f"{capped_server}/chunked", stream=True).raw.read()+++def test_guarded_raws_of_one_class_share_one_subclass(capped_server):+ """Guarding a response does not create a class per response.++ A class is freed only by the cyclic garbage collector, so a new one+ per response would bring back the garbage this guard avoids.+ """+ first = _fetch("safe_get", f"{capped_server}/small")+ second = _fetch("safe_get", f"{capped_server}/small-chunked")+ try:+ assert type(first.raw) is type(second.raw)+ assert isinstance(first.raw, urllib3.response.HTTPResponse)+ finally:+ first.close()+ second.close()+++def test_over_cap_read_closes_raw_after_the_response_is_dropped(+ capped_server,+):+ """Code that keeps only ``raw`` still hits the cap, and raw is closed.++ The guard holds the response weakly, so here the response is already+ gone when the cap is crossed, and the guard closes ``raw`` itself.+ """+ with safe_requests.SafeSession(allow_localhost=True) as session:+ with _cyclic_gc_off():+ response = session.get(f"{capped_server}/chunked", stream=True)+ raw = response.raw+ alive = weakref.ref(response)+ del response+ freed = alive() is None+ with pytest.raises(ValueError, match="too large"):+ raw.read()+ closed = raw.closed+ assert freed, "the body guard keeps a dropped response alive"+ assert closed, "the guard left the over-cap stream open"diff --git a/tests/security/test_safe_requests_retries.py b/tests/security/test_safe_requests_retries.pyindex afe22dae70..5ce0859829 100644--- a/tests/security/test_safe_requests_retries.py+++ b/tests/security/test_safe_requests_retries.py@@ -1,7 +1,10 @@"""Exponential-backoff retry wrapper for `safe_get`."""import email.utils+import threading+import timefrom datetime import datetime, timedelta, timezone+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServerfrom unittest.mock import MagicMock, patchimport pytest@@ -187,6 +190,98 @@ def test_retry_after_http_date_form_is_parsed():assert 45 <= slept <= 75, f"expected ~60s, got {slept}s"[email protected]("form", ["-0000", "no zone", "asctime"])+def test_retry_after_http_date_without_a_zone_is_read_as_utc(form):+ """An HTTP-date that names no zone is UTC, and must be waited for.++ ``email.utils`` parses these forms to a naive datetime. Subtracting+ the aware current time from it raised ``TypeError`` out of the retry+ loop, so the fetch failed instead of being retried.+ """+ future = datetime.now(timezone.utc) + timedelta(seconds=60)+ if form == "asctime":+ http_date = time.asctime(time.gmtime(future.timestamp()))+ else:+ http_date = email.utils.formatdate(future.timestamp())+ assert http_date.endswith(" -0000")+ if form == "no zone":+ http_date = http_date.removesuffix(" -0000")+ assert email.utils.parsedate_to_datetime(http_date).tzinfo is None+ rate_limited = _mock_response(429, headers={"Retry-After": http_date})+ ok = _mock_response(200)+ with (+ patch(+ "local_deep_research.security.safe_requests.safe_get",+ side_effect=[rate_limited, ok],+ ),+ patch(+ "local_deep_research.security.safe_requests.time.sleep"+ ) as mock_sleep,+ ):+ resp = safe_get_with_retries(+ "https://example.com/x", backoff_times=(1, 1, 1)+ )+ assert resp is ok+ slept = mock_sleep.call_args[0][0]+ # 30s band absorbs clock jitter and test-harness latency.+ assert 45 <= slept <= 75, f"expected ~60s, got {slept}s"+++def test_retry_after_http_date_with_an_overflowing_field_uses_schedule():+ """A date field too large for ``datetime`` falls back to the schedule.++ Depending on the Python version, ``parsedate_to_datetime`` raises+ ``OverflowError`` directly or wraps it in ``ValueError``. Both must+ preserve the retry behavior.+ """+ huge = "Wed, 21 Oct 2015 99999999999999999999:28:00 GMT"+ rate_limited = _mock_response(429, headers={"Retry-After": huge})+ ok = _mock_response(200)+ with (+ patch(+ "local_deep_research.security.safe_requests.safe_get",+ side_effect=[rate_limited, ok],+ ),+ patch(+ "local_deep_research.security.safe_requests.time.sleep"+ ) as mock_sleep,+ ):+ resp = safe_get_with_retries(+ "https://example.com/x", backoff_times=(1, 2, 4)+ )+ assert resp is ok+ assert mock_sleep.call_args[0][0] == 1++[email protected]("parse_error", [OverflowError, ValueError])+def test_retry_after_date_parser_error_uses_schedule(parse_error):+ """Keep both parser error forms covered on every supported Python."""+ http_date = "Wed, 21 Oct 2015 07:28:00 GMT"+ rate_limited = _mock_response(429, headers={"Retry-After": http_date})+ ok = _mock_response(200)+ with (+ patch(+ "local_deep_research.security.safe_requests.safe_get",+ side_effect=[rate_limited, ok],+ ) as mock_get,+ patch(+ "local_deep_research.security.safe_requests.email.utils.parsedate_to_datetime",+ side_effect=parse_error("invalid date field"),+ ) as mock_parse,+ patch(+ "local_deep_research.security.safe_requests.time.sleep"+ ) as mock_sleep,+ ):+ resp = safe_get_with_retries(+ "https://example.com/x", backoff_times=(1, 2, 4)+ )+ assert resp is ok+ assert mock_get.call_count == 2+ mock_parse.assert_called_once_with(http_date)+ mock_sleep.assert_called_once_with(1)+ rate_limited.close.assert_called_once()++def test_retry_after_unparseable_falls_back_to_schedule():"""Garbage Retry-After falls back to the backoff schedule."""rate_limited = _mock_response(429, headers={"Retry-After": "garbage"})@@ -363,3 +458,127 @@ def test_consume_body_returns_cached_body_to_caller():):resp = safe_get_with_retries("https://example.com/x", consume_body=True)assert resp.content == b"hello world"+++# ---------------------------------------------------------------------------+# consume_body=True through the real safe_get (nothing patched)+# ---------------------------------------------------------------------------+#+# The consume_body tests above patch safe_get, so they cannot see whether+# the body is still unread when safe_get returns. safe_get reads the body+# itself unless stream=True is passed, and a ChunkedEncodingError raised+# there is neither a ConnectionError nor a Timeout, so it would escape the+# wrapper unretried. These run the real safe_get over a real requests+# transport against a server on 127.0.0.1.++_BODY = b"0123456789abcdef" * 256 # 4 KiB+++def loopback():+ """Start a 127.0.0.1 server that answers from a script.++ ``serve(script)`` returns ``(url, seen)``. ``script`` names one+ response per request, in order, and its last entry repeats:+ ``"ok-length"`` / ``"ok-chunked"`` send ``_BODY`` with a+ Content-Length or chunked framing, ``"broken-length"`` /+ ``"broken-chunked"`` promise all of it but send a third and hang up+ (the client raises ``ChunkedEncodingError`` mid-body), and+ ``"busy"`` sends ``_BODY`` with status 503. ``seen`` lists the+ request paths received.+ """+ servers = []++ def serve(script):+ seen = []+ lock = threading.Lock()++ class Handler(BaseHTTPRequestHandler):+ protocol_version = "HTTP/1.1"++ def do_GET(self):+ with lock:+ seen.append(self.path)+ kind = script[min(len(seen), len(script)) - 1]+ self.close_connection = True+ body = _BODY[: len(_BODY) // 3] if "broken" in kind else _BODY+ self.send_response(503 if kind == "busy" else 200)+ self.send_header("Connection", "close")+ if kind.endswith("chunked"):+ self.send_header("Transfer-Encoding", "chunked")+ self.end_headers()+ self.wfile.write(b"%x\r\n" % len(_BODY) + body)+ if "broken" not in kind:+ self.wfile.write(b"\r\n0\r\n\r\n")+ else:+ self.send_header("Content-Length", str(len(_BODY)))+ self.end_headers()+ self.wfile.write(body)+ self.wfile.flush()++ def log_message(self, *args):+ pass++ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)+ thread = threading.Thread(target=server.serve_forever, daemon=True)+ thread.start()+ servers.append((server, thread))+ return f"http://127.0.0.1:{server.server_port}/data", seen++ yield serve+ for server, thread in servers:+ server.shutdown()+ server.server_close()+ thread.join(timeout=2)++[email protected]("framing", ["length", "chunked"])+def test_consume_body_retries_mid_body_break_through_real_safe_get(+ loopback, framing+):+ """A body that breaks mid-read is fetched again and returned.++ Red if the wrapper stops passing ``stream=True``: safe_get then reads+ the body before returning and the ``ChunkedEncodingError`` escapes+ after one request.+ """+ url, seen = loopback([f"broken-{framing}", f"ok-{framing}"])++ response = safe_get_with_retries(+ url,+ allow_localhost=True,+ timeout=5,+ consume_body=True,+ backoff_times=(0,),+ )++ assert len(seen) == 2+ assert response.status_code == 200+ # Read and cached inside the wrapper: closing the transport loses+ # nothing.+ response.close()+ assert response.content == _BODY+++def test_consume_body_final_retryable_status_returns_read_body(loopback):+ """The last attempt's 503 comes back with its body already read.++ ``stream=True`` makes safe_get hand over an unread body, so a+ retryable status returned before the consume_body read would leave+ it unread on the response.+ """+ url, seen = loopback(["busy"])++ response = safe_get_with_retries(+ url,+ allow_localhost=True,+ timeout=5,+ consume_body=True,+ max_retries=1,+ backoff_times=(0,),+ )++ assert len(seen) == 2+ assert response.status_code == 503+ response.close()+ assert response.content == _BODYdiff --git a/tests/security/test_ssrf_response_size.py b/tests/security/test_ssrf_response_size.pyindex 66cc6e6252..36503b5eb9 100644--- a/tests/security/test_ssrf_response_size.py+++ b/tests/security/test_ssrf_response_size.py@@ -4,6 +4,8 @@ Verifies try/except/else pattern ensures ValueError from int() parsingdoes not mask ValueError from legitimate size-exceeded checks."""+import socket+import requestsimport pytestfrom unittest.mock import patch, MagicMock@@ -11,7 +13,7 @@ from unittest.mock import patch, MagicMockfrom io import BytesIOfrom types import SimpleNamespace-from local_deep_research.security import ssrf_validator+from local_deep_research.security import dns_pinning, ssrf_validatorfrom local_deep_research.security.safe_requests import (safe_get,safe_post,@@ -23,7 +25,26 @@ from local_deep_research.security.safe_requests import (@pytest.fixture-def mock_validate_url():+def mock_validate_url(monkeypatch):+ # These cases mock HTTP responses, but the real pinning guard still+ # resolves the target before the mocked request. Keep that lookup local+ # so a DNS outage cannot prevent the size-limit assertions from running.+ def resolve(host, port, *args, **kwargs):+ if host != "example.com":+ pytest.fail(+ f"Unexpected DNS lookup in response-size test: {host!r}"+ )+ return [+ (+ socket.AF_INET,+ socket.SOCK_STREAM,+ socket.IPPROTO_TCP,+ "",+ ("93.184.216.34", port),+ )+ ]++ monkeypatch.setattr(dns_pinning, "_real_getaddrinfo", resolve)with patch.object(ssrf_validator,"validate_url",@@ -414,6 +435,15 @@ class TestSafeSessionResponseSize:class TestBodySizeEnforcement:"""Verify _install_body_guard enforces size limits on responses without CL."""+ @pytest.fixture(autouse=True)+ def small_body_limit(self, monkeypatch):+ # Exercise the same overflow and streaming boundaries without+ # allocating GiB-sized bodies in each test worker.+ monkeypatch.setitem(globals(), "MAX_RESPONSE_SIZE", 4096)+ monkeypatch.setattr(+ "local_deep_research.security.safe_requests.MAX_RESPONSE_SIZE", 4096+ )+def _make_response(self, body: bytes, headers: dict | None = None):"""Create a mock response with a real readable raw body."""resp = MagicMock(spec=requests.Response)diff --git a/tests/web_search_engines/engines/test_arxiv_pdf_egress_gating.py b/tests/web_search_engines/engines/test_arxiv_pdf_egress_gating.pyindex 18ced2022f..9c3cdf7420 100644--- a/tests/web_search_engines/engines/test_arxiv_pdf_egress_gating.py+++ b/tests/web_search_engines/engines/test_arxiv_pdf_egress_gating.py@@ -154,8 +154,9 @@ class TestFullContentUsesGatedPath:HTML leg answers 404, so ``download_full_text`` falls backto ``BaseDownloader._download_pdf``, whose response is a craftedbody decoding past ``MAX_RESPONSE_SIZE`` while claiming a valid,- under-cap ``Content-Length`` — the exact framing SafeSession- leaves unguarded. The abort can only come from+ under-cap ``Content-Length``. ``session.get`` is stubbed, so+ ``SafeSession.send`` (which guards that framing too) never runs:+ the abort can only come from``BaseDownloader._ensure_decoded_body_cap``'s guard install;reverting it lets the finite bomb through as a "successful"download and the ResponseBodyTooLarge outcome is never recorded.