Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

61.7 KB
Raw Blame
"""
Safe HTTP Requests Wrapper

Wraps requests library to add SSRF protection and security best practices.
"""

import datetime
import email.utils
import functools
import inspect
import threading
import time
import types
import weakref
from typing import TYPE_CHECKING, Any, Callable, Optional
from urllib.parse import urljoin

import requests
from loguru import logger

from . import dns_pinning
from . import ssrf_validator
from ..constants import USER_AGENT
from ..utilities.resource_utils import safe_close

if TYPE_CHECKING:
    from .egress.policy import EgressContext

# Log lines and raised exception texts below carry only scheme://host:port:
# the texts flow into caller logs (loguru renders ``ExcType: str(e)``) and
# into DB error columns, and the URLs can hold userinfo passwords or
# presigned query tokens.
_redact = ssrf_validator.redact_url_for_log


# Default timeout for all HTTP requests (prevents hanging)
DEFAULT_TIMEOUT = 30  # seconds

# Maximum response size to prevent memory exhaustion (1GB)
# Set high to accommodate large documents (annual reports, PDFs, datasets).
# This is a local research tool — users intentionally download these files.
MAX_RESPONSE_SIZE = 1024 * 1024 * 1024

# HTTP status codes that indicate a redirect
_REDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})

# Maximum number of redirects to follow
_MAX_REDIRECTS = 10

# Prefix of the requests library's default User-Agent
# (e.g. "python-requests/2.32.5"). Used by `SafeSession.request` to detect
# whether the session is using the upstream default — if so, we override
# it with the project `USER_AGENT` so academic API endpoints (arXiv,
# OpenAlex, PubMed, …) can identify us. Promoting the literal to a
# constant so a future requests rename only requires a one-line edit.
_DEFAULT_REQUESTS_UA_PREFIX = "python-requests"

# Instance attribute of a size-guarded raw: the list of _BodyGuard objects
# its class-level read()/read_chunked() count against. See
# _install_body_guard.
_BODY_GUARDS_ATTR = "_ldr_body_guards"


class _BodyGuard:
    """One installed body guard: the byte budget of a single response.

    It holds the response only through a weak reference and never refers
    to ``raw``, so storing it on ``raw`` forms no reference cycle.
    """

    __slots__ = ("bytes_read", "response_ref")

    def __init__(self, response: requests.Response) -> None:
        self.bytes_read = 0
        self.response_ref = weakref.ref(response)

    def limit(self, amt):
        """Return the read size to request, at most one byte past the cap.

        raw.read(None) can consume an entire hostile response in one call.
        Limit even direct raw readers to one byte past the remaining cap,
        enough to detect overflow without buffering the whole body.
        """
        allowance = MAX_RESPONSE_SIZE - self.bytes_read + 1
        if amt is None or amt < 0 or amt > allowance:
            return allowance
        return amt

    def count(self, raw: Any, data: bytes) -> None:
        """Add ``data`` to the budget; past the cap, close and raise.

        Raises:
            ValueError: Once more than MAX_RESPONSE_SIZE bytes were read.
        """
        self.bytes_read += len(data)
        if self.bytes_read > MAX_RESPONSE_SIZE:
            owner = self.response_ref()
            if owner is not None:
                owner.close()
            else:
                # Only raw is still referenced. Close it the way
                # Response.close() closes an unread body.
                raw.close()
                release_conn = getattr(raw, "release_conn", None)
                if release_conn is not None:
                    release_conn()
            raise ValueError(
                f"Response body too large: >{self.bytes_read} bytes "
                f"(max {MAX_RESPONSE_SIZE})"
            )


def _is_plain_method(cls: type, name: str) -> bool:
    """Whether ``cls`` defines ``name`` as an ordinary instance method."""
    attr = inspect.getattr_static(cls, name, None)
    return isinstance(attr, (types.FunctionType, types.MethodDescriptorType))


@functools.lru_cache(maxsize=32)
def _size_guarded_class(base: type[Any], with_read_chunked: bool) -> type:
    """Return a subclass of ``base`` whose reads count against its guards.

    Its read(), and read_chunked() when ``with_read_chunked``, call
    ``base``'s method (looked up when called) and count the result
    against every _BodyGuard in the instance's ``_BODY_GUARDS_ATTR``
    list, in install order, as nested wrappers would. It adds no slots,
    so an instance of ``base`` can be switched to it by assigning
    ``__class__``. Cached, so the raws of one class share one subclass.
    """

    class SizeGuarded(base):
        __slots__ = ()
        _ldr_size_guarded = True

        def read(self, amt=None, *args, **kwargs):
            guards = tuple(getattr(self, _BODY_GUARDS_ATTR))
            # The last guard installed clamps first, as the outermost
            # wrapper would.
            for guard in reversed(guards):
                amt = guard.limit(amt)
            data = super().read(amt, *args, **kwargs)
            for guard in guards:
                guard.count(self, data)
            return data

        if with_read_chunked:

            def read_chunked(self, amt=None, *args, **kwargs):
                guards = tuple(getattr(self, _BODY_GUARDS_ATTR))
                for guard in reversed(guards):
                    amt = guard.limit(amt)
                for chunk in super().read_chunked(amt, *args, **kwargs):
                    for guard in guards:
                        guard.count(self, chunk)
                    yield chunk

    SizeGuarded.__name__ = "".join(("SizeGuarded", base.__name__))
    SizeGuarded.__qualname__ = SizeGuarded.__name__
    return SizeGuarded


def _class_guard_list(raw: Any) -> Optional[list]:
    """Return the guard list that raw's class-level reads count against.

    On the first guard this switches raw's class to the
    ``_size_guarded_class`` of its class and gives raw an empty list. It
    returns None, leaving raw unchanged, when raw cannot take that
    subclass: read() or read_chunked() is set on the instance, the class
    does not define read() as an ordinary method or defines
    read_chunked() as something else, raw has no instance ``__dict__``
    or its class overrides ``__setattr__``, or the ``__class__``
    assignment is refused. That is the case for test doubles such as
    mocks, ``SimpleNamespace`` and ``io.BytesIO``. requests' own
    adapters always return urllib3 responses, which take the subclass.
    """
    cls: type = type(raw)
    if getattr(cls, "_ldr_size_guarded", False):
        existing: list = vars(raw).setdefault(_BODY_GUARDS_ATTR, [])
        return existing
    # A class with its own __setattr__ (a mock, for one) may not treat the
    # __class__ assignment below as a plain attribute write.
    if any("__setattr__" in vars(klass) for klass in cls.__mro__[:-1]):
        return None
    try:
        own = vars(raw)
    except TypeError:
        return None
    if "read" in own or "read_chunked" in own:
        return None
    if not _is_plain_method(cls, "read"):
        return None
    with_read_chunked = (
        inspect.getattr_static(cls, "read_chunked", None) is not None
    )
    if with_read_chunked and not _is_plain_method(cls, "read_chunked"):
        return None
    try:
        raw.__class__ = _size_guarded_class(cls, with_read_chunked)
    except TypeError:
        return None
    guards: list = []
    setattr(raw, _BODY_GUARDS_ATTR, guards)
    return guards


def _wrap_instance_reads(raw: Any, guard: _BodyGuard) -> None:
    """Wrap raw's read() and read_chunked() on the instance.

    The fallback for a raw that cannot take ``_size_guarded_class`` (see
    ``_class_guard_list``). A second guard wraps the first guard's
    wrappers, so each counts every byte. The wrappers hold ``raw``, as
    the bound read() they replace does.
    """
    original_read = raw.read

    def bounded_read(amt=None, *args, **kwargs):
        data = original_read(guard.limit(amt), *args, **kwargs)
        guard.count(raw, data)
        return data

    raw.read = bounded_read

    original_read_chunked = getattr(raw, "read_chunked", None)
    if original_read_chunked is None:
        return

    def bounded_read_chunked(amt=None, *args, **kwargs):
        for chunk in original_read_chunked(guard.limit(amt), *args, **kwargs):
            guard.count(raw, chunk)
            yield chunk

    raw.read_chunked = bounded_read_chunked


def _install_body_guard(response: requests.Response) -> None:
    """Install a bounded reader that enforces MAX_RESPONSE_SIZE.

    Counts the bytes response.raw.read() returns and raises ValueError
    once more than MAX_RESPONSE_SIZE have been read during body
    consumption. This transparently protects both streamed
    (.iter_content) and non-streamed (.text, .json(), .content) access
    patterns, including decompressed bodies whose wire Content-Length
    was small.

    A Transfer-Encoding: chunked body never reaches raw.read():
    urllib3's stream() dispatches it to read_chunked(), which pulls the
    socket through raw._fp. That generator is counted against the same
    budget, so the two paths share one.

    The guard (``_BodyGuard``) refers to the response only weakly and
    never to ``raw``. A urllib3 response's ``raw`` is switched to a
    cached subclass of its own class whose read() and read_chunked()
    apply the guards listed on ``raw`` (``_size_guarded_class``), so
    nothing stored on ``raw`` refers back to ``raw`` or to the response.
    A dropped response, its ``raw`` and its cached body are therefore
    freed by reference counting, however many guards are stacked. A
    cycle through the guard would keep them, and urllib3's decoder
    state, in memory until the cyclic garbage collector ran. Because
    read() is found on the class, a call such as
    ``safe_get(url, stream=True).raw.read()`` keeps ``raw`` alive while
    it runs. If the response is gone when the cap is crossed, ``raw`` is
    closed directly instead.

    A raw that cannot take the subclass, such as a test double (a mock,
    ``SimpleNamespace`` or ``io.BytesIO``; see ``_class_guard_list``),
    gets instance-level wrappers instead. They hold ``raw``, as the
    bound read() they replace does, so such a raw is freed only by the
    cyclic garbage collector.

    Always installs. The callers (_check_response_size, and
    BaseDownloader._ensure_decoded_body_cap for a second guard) decide
    when to call it.
    """
    raw = response.raw
    # Synthetic responses used by callers/tests may have a cached body
    # without a transport. Real streamed responses always have raw set.
    if raw is None:
        return

    guard = _BodyGuard(response)
    guards = _class_guard_list(raw)
    if guards is not None:
        guards.append(guard)
    else:
        _wrap_instance_reads(raw, guard)


def _check_response_size(response: requests.Response) -> None:
    """Reject responses whose Content-Length exceeds MAX_RESPONSE_SIZE.

    Handles comma-separated values per RFC 7230 §3.3.2: identical
    duplicates (from proxies) are normalized; differing values are
    rejected as invalid framing. Empty parts from malformed headers
    (trailing/doubled commas) are filtered before parsing. Non-integer
    or negative values cause the header to be treated as absent.

    Every accepted response gets a body guard. Content-Length counts wire
    bytes; decompression can make the delivered body much larger, and a
    server may understate its length.

    Must be called before returning a response to the caller. On
    rejection the response is closed to avoid leaking the connection.

    Raises:
        ValueError: If Content-Length values conflict or exceed
            MAX_RESPONSE_SIZE.
    """
    content_length = response.headers.get("Content-Length")
    if content_length:
        try:
            # Handle comma-separated Content-Length values (RFC 7230 §3.3.2).
            # Multiple identical values may be sent by proxies; differing
            # values indicate invalid framing and must be rejected.
            raw_parts = [v.strip() for v in content_length.split(",")]
            parts = [p for p in raw_parts if p]
            if not parts:
                _install_body_guard(response)
                return  # Only commas/whitespace — treat as absent
            sizes = [int(p) for p in parts]
        except (ValueError, TypeError):
            _install_body_guard(response)
            return  # Content-Length not a valid number
        if len(set(sizes)) > 1:
            response.close()
            raise ValueError(
                f"Conflicting Content-Length values: {content_length}"
            )
        size = sizes[0]
        if size < 0:
            _install_body_guard(response)
            return  # Malformed Content-Length, treat as absent
        if size > MAX_RESPONSE_SIZE:
            response.close()
            raise ValueError(
                f"Response too large: {size} bytes (max {MAX_RESPONSE_SIZE})"
            )
        # Valid wire length is only a precheck; decoded bytes still need
        # the body guard during consumption.
        _install_body_guard(response)
        return

    # No Content-Length header at all — install body guard
    _install_body_guard(response)


def _finish_response(
    response: requests.Response, *, preload: bool
) -> requests.Response:
    """Install the size guard before optionally buffering the body."""
    _check_response_size(response)
    if preload:
        try:
            _ = response.content
        except Exception:
            safe_close(response, "response")
            raise
    return response


def _discard_redirect_body(response: requests.Response) -> None:
    """Size-check a redirect response, then drop its body unread.

    Requests reads ``response.content`` -- the whole body, decoded -- at the
    start of ``Session.resolve_redirects``, which ``Session.send`` runs on
    every redirect: to follow it, and with ``allow_redirects=False`` to
    prepare ``Response.next``. A redirect body is not needed to follow
    ``Location``, so close the connection and cache an empty body before
    that read can happen.

    Raises:
        ValueError: If the redirect's Content-Length exceeds
            MAX_RESPONSE_SIZE or conflicts (see ``_check_response_size``).
    """
    _check_response_size(response)
    # Close before marking the body consumed: Response.close() releases an
    # unread connection only while this flag is false. A synthetic response
    # with no transport (raw=None) has no connection to release and
    # Response.close() would crash on it; skip straight to discarding the
    # (absent) body.
    if response.raw is not None:
        response.close()
    response._content = b""
    response._content_consumed = True


def _discard_redirect_body_hook(response: requests.Response, **_kwargs):
    """Requests response hook: discard a redirect's body (see above).

    ``Session.send`` dispatches response hooks before it calls
    ``resolve_redirects``, so this runs ahead of the body read there.
    """
    if response.is_redirect:
        _discard_redirect_body(response)


def _with_redirect_body_hook(hooks: Optional[dict]) -> dict:
    """Return ``hooks`` with ``_discard_redirect_body_hook`` run first.

    ``safe_get``/``safe_post`` send each hop through a plain
    ``requests.get``/``requests.post``, whose throwaway ``requests.Session``
    would otherwise buffer every redirect body (see
    ``_discard_redirect_body``). Caller-supplied hooks are kept and run
    after it, so they see the redirect's status and headers but an empty
    body. The caller's dict is not mutated.
    """
    merged = dict(hooks or {})
    caller_response_hooks = merged.get("response")
    if caller_response_hooks is None:
        caller_response_hooks = []
    elif callable(caller_response_hooks):
        caller_response_hooks = [caller_response_hooks]
    merged["response"] = [
        _discard_redirect_body_hook,
        *caller_response_hooks,
    ]
    return merged


def _resolve_redirect_method(method: str, status_code: int) -> str:
    """Determine HTTP method after redirect, per RFC 7231."""
    if status_code == 303 and method != "HEAD":
        method = "GET"
    elif status_code == 302 and method == "POST":
        method = "GET"
    elif status_code == 301 and method == "POST":
        method = "GET"
    # 307, 308: preserve original method (no change needed)
    return method


# Credential headers: the HTTP-standard set, plus the vendor key/token family.
# The suffixes cover the shapes vendors ship (X-Api-Key, Zotero-API-Key,
# X-Goog-Api-Key, X-Subscription-Token, Ocp-Apim-Subscription-Key, Private-Token),
# so a new vendor needs no edit here. No non-credential header this project
# sends ends in one of them.
_CREDENTIAL_HEADERS = frozenset(
    {"authorization", "proxy-authorization", "cookie"}
)
_CREDENTIAL_HEADER_SUFFIXES = ("key", "token", "secret")

# requests owns the "does this hop leave the credential's scope" rule, so
# reuse it rather than restate it in the hand-rolled loops below.
_redirect_scope = requests.sessions.SessionRedirectMixin()


def _is_credential_header(name: str) -> bool:
    """Whether a request header carries a caller credential."""
    lowered = name.lower()
    return lowered in _CREDENTIAL_HEADERS or lowered.endswith(
        _CREDENTIAL_HEADER_SUFFIXES
    )


def _headers_for_redirect(
    headers: Optional[dict], from_url: str, to_url: str
) -> Optional[dict]:
    """Return the headers to send on a redirect hop.

    Credential headers are dropped when the hop leaves the scope they were
    issued for, so a key stays with the host the caller addressed. Returns a
    new dict when it strips, so the caller's own headers object is untouched.
    A chain that leaves scope and later returns to the original host does not
    get the credential back, which is what requests does on its own hops.
    """
    if not headers or not _redirect_scope.should_strip_auth(from_url, to_url):
        return headers
    return {k: v for k, v in headers.items() if not _is_credential_header(k)}


def safe_get(
    url: str,
    params: Optional[dict] = None,
    timeout: int = DEFAULT_TIMEOUT,
    allow_localhost: bool = False,
    allow_private_ips: bool = False,
    require_https: bool = False,
    **kwargs,
) -> requests.Response:
    """
    Make a safe HTTP GET request with SSRF protection.

    Args:
        url: URL to request
        params: URL parameters
        timeout: Request timeout in seconds
        allow_localhost: Whether to allow localhost/loopback addresses.
            Set to True for trusted internal services like self-hosted
            search engines (e.g., searxng). Default False.
        allow_private_ips: Whether to allow all private/internal IPs plus localhost.
            This includes RFC1918 (10.x, 172.16-31.x, 192.168.x), CGNAT (100.64.x.x
            used by Podman/rootless containers), link-local (169.254.x.x), and IPv6
            private ranges (fc00::/7, fe80::/10). Use for trusted self-hosted services
            like SearXNG or Ollama in containerized environments.
            Note: cloud metadata endpoints (AWS / Azure / GCP / OCI /
            Scaleway / Akamai / OpenStack / Hetzner / DigitalOcean /
            AlibabaCloud / Tencent / Oracle Compute Classic, and the GCP /
            IBM Cloud classic metadata hostnames) are ALWAYS blocked — see
            ``ssrf_validator.ALWAYS_BLOCKED_METADATA_IPS`` and
            ``ssrf_validator.ALWAYS_BLOCKED_METADATA_HOSTNAMES``.
        require_https: Refuse the fetch (and any redirect hop) whose URL
            is not https://. Opt-in per caller for integrity-sensitive
            downloads (e.g. the journal-quality datasets): a hop to
            cleartext passes SSRF validation, letting a compromised
            upstream or open-redirect move the fetch onto a tamperable
            path. Redirect hops are judged after resolving the Location
            against the current URL, so relative and protocol-relative
            Locations inherit https. Combining it with ``verify=False``
            raises ValueError — https without certificate verification
            is not an authenticated transport. NOT global — localhost
            SearXNG instances are legitimately http://.
        **kwargs: Additional arguments to pass to requests.get()

    Returns:
        Response object. A redirect response's body (any 301/302/303/307/308
        with a ``Location``) is discarded unread, whether the hop is followed
        or returned because ``allow_redirects=False``: its status and headers
        stay available, ``content`` is ``b""``. Caller ``hooks`` still run,
        after that discard.

    Raises:
        ValueError: If URL fails SSRF validation, or (with
            ``require_https``) the URL or a redirect hop is not https, or
            ``verify=False`` is passed, or a response's Content-Length
            (a redirect's included) exceeds MAX_RESPONSE_SIZE or conflicts,
            or the decoded body read before returning exceeds
            MAX_RESPONSE_SIZE (with ``stream=True`` the caller's own body
            read raises that instead)
        requests.RequestException: If request fails
    """
    if require_https:
        # Pinning the scheme is meaningless if the certificate is not
        # checked: an on-path attacker can terminate TLS with any cert.
        # requests disables verification for any falsy ``verify``
        # (``False``, ``0``, ``""``); ``None`` merges to the session
        # default (verified) and is allowed.
        verify = kwargs.get("verify")
        if verify is not None and not verify:
            raise ValueError(
                "require_https cannot be combined with verify=False"
            )
        if not url.lower().startswith("https://"):
            raise ValueError(
                f"URL must use https when require_https is set: {_redact(url)}"
            )
    # Validate URL to prevent SSRF
    if not ssrf_validator.validate_url(
        url,
        allow_localhost=allow_localhost,
        allow_private_ips=allow_private_ips,
    ):
        raise ValueError(
            f"URL failed security validation (possible SSRF): {_redact(url)}"
        )

    # Ensure timeout is set
    if "timeout" not in kwargs:
        kwargs["timeout"] = timeout

    # Inject the project User-Agent if the caller didn't supply one.
    # Mutates a copy of any caller-supplied headers dict so we never
    # touch their object.
    headers = dict(kwargs.get("headers") or {})
    if not any(k.lower() == "user-agent" for k in headers):
        headers["User-Agent"] = USER_AGENT
        kwargs["headers"] = headers

    # Intercept allow_redirects — we handle redirects manually to validate
    # each redirect target against SSRF rules
    caller_wants_redirects = kwargs.pop("allow_redirects", True)
    kwargs["allow_redirects"] = False
    caller_stream = kwargs.get("stream", False)
    # Requests otherwise buffers the body inside requests.get(), before
    # _check_response_size can install its guard. Preserve the caller's
    # normal preloaded-response contract after that guard is in place.
    kwargs["stream"] = True
    # Each hop's plain requests call would still buffer a redirect body
    # while preparing Response.next; drop it unread instead.
    kwargs["hooks"] = _with_redirect_body_hook(kwargs.get("hooks"))

    current_url = url
    try:
        # Pin the validated IP for the actual connection: the request below
        # connects to the address resolved+validated here, closing the
        # resolve-vs-connect gap where requests/urllib3 would otherwise
        # re-resolve the hostname independently.
        with dns_pinning.pinned_request(
            url, allow_localhost, allow_private_ips
        ):
            # The wrapper sets timeout in kwargs before the first request and retains it for redirects.
            response = requests.get(url, params=params, **kwargs)  # nosec B113

        # Follow redirects manually with SSRF validation on each hop.
        # Each hop uses a fresh requests.get() call without a session,
        # so cookies set by intermediate responses are not carried
        # forward. This is acceptable for current callers (all stateless).
        # Callers needing cookie persistence across redirects should use
        # SafeSession instead, which preserves cookies via its cookie jar.
        if caller_wants_redirects:
            redirects_followed = 0
            while (
                response.status_code in _REDIRECT_STATUS_CODES
                and redirects_followed < _MAX_REDIRECTS
            ):
                redirect_url = (response.headers.get("Location") or "").strip()
                if not redirect_url:
                    break

                # Resolve relative redirects
                redirect_url = urljoin(
                    response.url or current_url, redirect_url
                )

                # Scheme pin: refuse a hop that would move the fetch to
                # cleartext. Checked after urljoin so relative and
                # protocol-relative Locations inherit the https scheme
                # before being judged.
                if require_https and not redirect_url.lower().startswith(
                    "https://"
                ):
                    logger.warning(
                        f"Redirect to {_redact(redirect_url)} would downgrade "
                        f"to cleartext (from {_redact(url)}, hop "
                        f"{redirects_followed + 1})"
                    )
                    response.close()
                    raise ValueError(
                        "Redirect would downgrade to cleartext: "
                        f"{_redact(redirect_url)}"
                    )

                # Validate redirect target against SSRF rules
                if not ssrf_validator.validate_url(
                    redirect_url,
                    allow_localhost=allow_localhost,
                    allow_private_ips=allow_private_ips,
                ):
                    logger.warning(
                        f"Redirect to {_redact(redirect_url)} blocked by SSRF "
                        f"validation (from {_redact(url)}, hop "
                        f"{redirects_followed + 1})"
                    )
                    response.close()
                    raise ValueError(
                        f"Redirect target failed SSRF validation: {_redact(redirect_url)}"
                    )

                # Credentials do not follow the hop out of their own scope.
                kwargs["headers"] = _headers_for_redirect(
                    kwargs.get("headers"), current_url, redirect_url
                )
                current_url = redirect_url
                response.close()
                # Note: params are intentionally NOT forwarded to redirect
                # hops. Per HTTP spec, the server's Location header contains
                # the complete target URL. Re-appending original query params
                # would corrupt it.
                # Re-pin per hop: the address validated for this redirect
                # target is the address connected to.
                with dns_pinning.pinned_request(
                    redirect_url, allow_localhost, allow_private_ips
                ):
                    # The wrapper sets timeout in kwargs before the first request and retains it for redirects.
                    response = requests.get(redirect_url, **kwargs)  # nosec B113
                redirects_followed += 1

            if (
                response.status_code in _REDIRECT_STATUS_CODES
                and redirects_followed >= _MAX_REDIRECTS
            ):
                response.close()
                # Note: raises ValueError here, while SafeSession raises
                # requests.TooManyRedirects (delegated to the base class).
                # Callers should catch ValueError for standalone functions.
                raise ValueError(
                    f"Too many redirects ({_MAX_REDIRECTS}) from {_redact(url)}"
                )

        return _finish_response(response, preload=not caller_stream)

    except requests.Timeout:
        logger.warning(
            f"Request timeout after {timeout}s: {_redact(current_url)}"
        )
        raise
    except requests.RequestException:
        logger.warning(f"Request failed for {_redact(current_url)}")
        raise


def safe_post(
    url: str,
    data: Optional[Any] = None,
    json: Optional[dict] = None,
    timeout: int = DEFAULT_TIMEOUT,
    allow_localhost: bool = False,
    allow_private_ips: bool = False,
    **kwargs,
) -> requests.Response:
    """
    Make a safe HTTP POST request with SSRF protection.

    Args:
        url: URL to request
        data: Data to send in request body
        json: JSON data to send in request body
        timeout: Request timeout in seconds
        allow_localhost: Whether to allow localhost/loopback addresses.
            Set to True for trusted internal services like self-hosted
            search engines (e.g., searxng). Default False.
        allow_private_ips: Whether to allow all private/internal IPs plus localhost.
            This includes RFC1918 (10.x, 172.16-31.x, 192.168.x), CGNAT (100.64.x.x
            used by Podman/rootless containers), link-local (169.254.x.x), and IPv6
            private ranges (fc00::/7, fe80::/10). Use for trusted self-hosted services
            like SearXNG or Ollama in containerized environments.
            Note: cloud metadata endpoints (AWS / Azure / GCP / OCI /
            Scaleway / Akamai / OpenStack / Hetzner / DigitalOcean /
            AlibabaCloud / Tencent / Oracle Compute Classic, and the GCP /
            IBM Cloud classic metadata hostnames) are ALWAYS blocked — see
            ``ssrf_validator.ALWAYS_BLOCKED_METADATA_IPS`` and
            ``ssrf_validator.ALWAYS_BLOCKED_METADATA_HOSTNAMES``.
        **kwargs: Additional arguments to pass to requests.post()

    Returns:
        Response object. Redirect bodies are discarded unread, as in
        ``safe_get``.

    Raises:
        ValueError: If the URL fails SSRF validation, if a 307/308
            redirect would re-send the request body outside the scope of
            the URL the caller addressed, if a response's
            Content-Length (a redirect's included) exceeds
            MAX_RESPONSE_SIZE or conflicts, or if the decoded body read
            before returning exceeds MAX_RESPONSE_SIZE (with
            ``stream=True`` the caller's own body read raises that
            instead)
        requests.RequestException: If request fails
    """
    # Validate URL to prevent SSRF
    if not ssrf_validator.validate_url(
        url,
        allow_localhost=allow_localhost,
        allow_private_ips=allow_private_ips,
    ):
        raise ValueError(
            f"URL failed security validation (possible SSRF): {_redact(url)}"
        )

    # Ensure timeout is set
    if "timeout" not in kwargs:
        kwargs["timeout"] = timeout

    # Inject the project User-Agent if the caller didn't supply one.
    # Mutates a copy of any caller-supplied headers dict so we never
    # touch their object.
    headers = dict(kwargs.get("headers") or {})
    if not any(k.lower() == "user-agent" for k in headers):
        headers["User-Agent"] = USER_AGENT
        kwargs["headers"] = headers

    # Intercept allow_redirects — we handle redirects manually to validate
    # each redirect target against SSRF rules
    caller_wants_redirects = kwargs.pop("allow_redirects", True)
    kwargs["allow_redirects"] = False
    caller_stream = kwargs.get("stream", False)
    kwargs["stream"] = True
    # Each hop's plain requests call would still buffer a redirect body
    # while preparing Response.next; drop it unread instead.
    kwargs["hooks"] = _with_redirect_body_hook(kwargs.get("hooks"))

    current_url = url
    try:
        # Pin the validated IP for the actual connection (see safe_get).
        with dns_pinning.pinned_request(
            url, allow_localhost, allow_private_ips
        ):
            # The wrapper sets timeout in kwargs before the first request and retains it for redirects.
            response = requests.post(url, data=data, json=json, **kwargs)  # nosec B113

        # Follow redirects manually with SSRF validation on each hop.
        # Each hop uses a fresh request without a session, so cookies
        # set by intermediate responses are not carried forward. Callers
        # needing cookie persistence should use SafeSession instead.
        if caller_wants_redirects:
            redirect_method = "POST"
            redirects_followed = 0
            while (
                response.status_code in _REDIRECT_STATUS_CODES
                and redirects_followed < _MAX_REDIRECTS
            ):
                redirect_url = (response.headers.get("Location") or "").strip()
                if not redirect_url:
                    break

                # Resolve relative redirects
                redirect_url = urljoin(
                    response.url or current_url, redirect_url
                )

                # Validate redirect target against SSRF rules
                if not ssrf_validator.validate_url(
                    redirect_url,
                    allow_localhost=allow_localhost,
                    allow_private_ips=allow_private_ips,
                ):
                    logger.warning(
                        f"Redirect to {_redact(redirect_url)} blocked by SSRF "
                        f"validation (from {_redact(url)}, hop "
                        f"{redirects_followed + 1})"
                    )
                    response.close()
                    raise ValueError(
                        f"Redirect target failed SSRF validation: {_redact(redirect_url)}"
                    )

                redirect_method = _resolve_redirect_method(
                    redirect_method, response.status_code
                )
                # 307/308 keep the body, and it cannot be dropped the way the
                # credential headers below are without sending a request the
                # server did not ask for, so refuse the hop instead.
                body_would_follow = (
                    data is not None
                    or json is not None
                    or kwargs.get("files") is not None
                )
                if (
                    redirect_method != "GET"
                    and body_would_follow
                    and _redirect_scope.should_strip_auth(
                        current_url, redirect_url
                    )
                ):
                    logger.warning(
                        f"Redirect to {_redact(redirect_url)} not followed: it "
                        f"would re-send the request body outside the scope "
                        f"of {_redact(current_url)} (hop "
                        f"{redirects_followed + 1})"
                    )
                    response.close()
                    raise ValueError(
                        "Redirect would re-send the request body outside "
                        f"the requested scope: {_redact(redirect_url)}"
                    )
                # Credentials do not follow the hop out of their own scope.
                kwargs["headers"] = _headers_for_redirect(
                    kwargs.get("headers"), current_url, redirect_url
                )
                current_url = redirect_url
                response.close()

                # Re-pin per hop: the address validated for this redirect
                # target is the address connected to.
                with dns_pinning.pinned_request(
                    redirect_url, allow_localhost, allow_private_ips
                ):
                    if redirect_method == "GET":
                        # 301/302/303: convert to GET, drop body
                        data = None
                        json = None
                        # The wrapper sets timeout in kwargs before the first request and retains it for redirects.
                        response = requests.get(redirect_url, **kwargs)  # nosec B113
                    else:
                        # 307/308: preserve current method and body
                        # The wrapper sets timeout in kwargs before the first request and retains it for redirects.
                        response = requests.post(  # nosec B113
                            redirect_url, data=data, json=json, **kwargs
                        )
                redirects_followed += 1

            if (
                response.status_code in _REDIRECT_STATUS_CODES
                and redirects_followed >= _MAX_REDIRECTS
            ):
                response.close()
                # Note: raises ValueError here, while SafeSession raises
                # requests.TooManyRedirects (delegated to the base class).
                # Callers should catch ValueError for standalone functions.
                raise ValueError(
                    f"Too many redirects ({_MAX_REDIRECTS}) from {_redact(url)}"
                )

        return _finish_response(response, preload=not caller_stream)

    except requests.Timeout:
        logger.warning(
            f"Request timeout after {timeout}s: {_redact(current_url)}"
        )
        raise
    except requests.RequestException:
        logger.warning(f"Request failed for {_redact(current_url)}")
        raise


# Create a safe session class
class SafeSession(requests.Session):
    """
    Session with built-in SSRF protection.

    Redirect validation relies on ``requests.Session.resolve_redirects()``
    calling ``self.send()`` for each hop — an internal implementation detail
    of the ``requests`` library.  This is simpler than re-implementing the
    redirect loop (as ``safe_get``/``safe_post`` do) and keeps session-level
    features (cookies, auth) working.  The trade-off is coupling to the
    ``requests`` internals; if a future version stops routing hops through
    ``send()``, redirect targets would no longer be validated.

    Redirect response bodies are discarded before Requests' redirect loop
    would buffer them, including its ``Response.next`` preparation when
    ``allow_redirects=False``. Callers can inspect redirect status and
    headers, but not its body.

    Usage:
        with SafeSession() as session:
            response = session.get(url)

        # For trusted internal services (e.g., searxng on localhost):
        with SafeSession(allow_localhost=True) as session:
            response = session.get(url)

        # For trusted internal services on any private network IP:
        with SafeSession(allow_private_ips=True) as session:
            response = session.get(url)

    Raises:
        ValueError: If a URL (initial or redirect target) fails SSRF
            validation, an enabled HTTPS downgrade guard refuses a redirect
            or disabled certificate verification, a 307/308 redirect would
            re-send the request body outside the scope of the URL the caller
            addressed, the response Content-Length exceeds MAX_RESPONSE_SIZE,
            or the decoded body read before a request returns exceeds it
            (with ``stream=True`` the caller's own body read raises that
            instead). Note: ``safe_get``/``safe_post`` also raise
            ``ValueError`` for too-many-redirects, but ``SafeSession`` raises
            ``requests.TooManyRedirects`` for that case since it delegates
            redirect counting to the ``requests`` library.
        requests.RequestException: On transport-level failures.
    """

    def __init__(
        self,
        allow_localhost: bool = False,
        allow_private_ips: bool = False,
        block_link_local: bool = False,
        block_https_downgrade: bool = False,
        egress_context: Optional["EgressContext"] = None,
    ):
        """
        Initialize SafeSession.

        Args:
            allow_localhost: Whether to allow localhost/loopback addresses.
            allow_private_ips: Whether to allow all private/internal IPs plus localhost.
                This includes RFC1918, CGNAT (100.64.x.x used by Podman), link-local, and
                IPv6 private ranges. Use for trusted self-hosted services like SearXNG or
                Ollama in containerized environments.
                Note: cloud metadata endpoints (AWS / Azure / GCP / OCI /
                Scaleway / Akamai / OpenStack / Hetzner / DigitalOcean /
                AlibabaCloud / Tencent / Oracle Compute Classic, and the
                GCP / IBM Cloud classic metadata hostnames) are ALWAYS
                blocked — see ``ssrf_validator.ALWAYS_BLOCKED_METADATA_IPS``
                and ``ssrf_validator.ALWAYS_BLOCKED_METADATA_HOSTNAMES``.
            block_link_local: Keep the whole link-local range (IPv4
                ``169.254.0.0/16``, IPv6 ``fe80::/10`` and their NAT64-wrapped
                forms) blocked EVEN under ``allow_private_ips=True``, on the
                initial request, every redirect hop, and the connect-time DNS
                re-validation. Set it when relaxing ``allow_private_ips`` for a
                trusted target that is never link-local (cloud metadata lives
                there beyond the always-blocked literals). No effect without
                ``allow_private_ips``; forwarded to ``validate_url``.
            block_https_downgrade: Refuse a redirect from HTTPS to HTTP,
                including a downgrade after earlier redirect hops. HTTP-only
                starting URLs remain supported. HTTPS requests must verify
                their certificates when this is enabled.
            egress_context: Optional per-run policy enforced on each initial
                request, redirect hop, and pinned connect-time address.
        """
        super().__init__()
        self.max_redirects = _MAX_REDIRECTS
        self.allow_localhost = allow_localhost
        self.allow_private_ips = allow_private_ips
        self.block_link_local = block_link_local
        self.url_guard: Optional[Callable[[str], None]] = None
        self.block_https_downgrade = block_https_downgrade
        self.egress_context = egress_context
        # Per-thread, because one session is shared across search engines.
        self._redirect_local = threading.local()

    @property
    def _preparing_next_only(self) -> bool:
        """True while ``resolve_redirects`` is only building ``Response.next``."""
        return getattr(self._redirect_local, "preparing_next_only", False)

    def resolve_redirects(
        self,
        resp,
        req,
        stream=False,
        timeout=None,
        verify=True,
        cert=None,
        proxies=None,
        yield_requests=False,
        **adapter_kwargs,
    ):
        """Record the redirect mode and discard bodies before Requests drains them.

        ``Session.send`` calls this with ``yield_requests=True`` even when the
        caller passed ``allow_redirects=False``, to populate ``Response.next``.
        ``rebuild_auth`` runs before that yield and cannot see which mode it is
        in, so the flag is published here for it to read.

        Requests reads ``resp.content`` at the start of its redirect loop,
        before control returns to ``SafeSession.send`` for the size check.
        A redirect can therefore buffer an unbounded body (including decoded
        compressed content) unless it is closed here. Redirect bodies are not
        needed to follow ``Location``; keep an empty cached body so Requests
        does not read the closed socket while preparing the next request.

        The parameter list mirrors ``Session.resolve_redirects`` positionally,
        so a caller passing ``stream`` third still gets streamed responses.
        """
        previous = self._preparing_next_only
        self._redirect_local.preparing_next_only = yield_requests
        try:
            if self.get_redirect_target(resp):
                _discard_redirect_body(resp)
            yield from super().resolve_redirects(
                resp,
                req,
                stream=stream,
                timeout=timeout,
                verify=verify,
                cert=cert,
                proxies=proxies,
                yield_requests=yield_requests,
                **adapter_kwargs,
            )
        finally:
            self._redirect_local.preparing_next_only = previous

    def rebuild_auth(
        self, prepared_request: requests.PreparedRequest, response
    ) -> None:
        """Drop vendor API-key headers too when a hop leaves auth scope, and
        refuse the hop when it would carry the request body along.

        The base implementation removes ``Authorization`` and nothing else, so
        a session default such as the Semantic Scholar engine's ``x-api-key``
        would otherwise be replayed to the redirect target. The strip runs
        before ``super()`` because the base method reapplies netrc credentials
        for the new host, and those belong to it.

        ``resolve_redirects`` clears ``body`` before calling this method for
        every status that downgrades the method, and keeps it for 307 and 308,
        so a body still present here is one the hop would re-send.

        The refusal is raised only when the hop is about to be followed. With
        ``allow_redirects=False`` nothing is sent, so raising would deny the
        caller the 3xx response it asked for; the credential strip still runs,
        which is what ``Response.next`` needs.
        """
        following_the_hop = not self._preparing_next_only
        if (
            self.block_https_downgrade
            and following_the_hop
            and (response.request.url or "").lower().startswith("https://")
            and (prepared_request.url or "").lower().startswith("http://")
        ):
            raise ValueError(
                "Redirect would downgrade to cleartext: "
                f"{ssrf_validator.redact_url_for_log(prepared_request.url)}"
            )

        if self.should_strip_auth(response.request.url, prepared_request.url):
            if prepared_request.body is not None and following_the_hop:
                logger.warning(
                    f"Redirect to {_redact(prepared_request.url)} not "
                    f"followed: it would re-send the request body outside "
                    f"the scope of {_redact(response.request.url)}"
                )
                raise ValueError(
                    "Redirect would re-send the request body outside "
                    f"the requested scope: {_redact(prepared_request.url)}"
                )
            for name in [
                header
                for header in prepared_request.headers
                if _is_credential_header(header)
            ]:
                del prepared_request.headers[name]
        super().rebuild_auth(prepared_request, response)

    def request(self, method: str, url: str, **kwargs) -> requests.Response:  # type: ignore[override]
        """Override request method to add SSRF validation."""
        # Validate URL
        if not ssrf_validator.validate_url(
            url,
            allow_localhost=self.allow_localhost,
            allow_private_ips=self.allow_private_ips,
            block_link_local=self.block_link_local,
        ):
            raise ValueError(
                f"URL failed security validation (possible SSRF): {_redact(url)}"
            )

        # Ensure timeout is set
        if "timeout" not in kwargs:
            kwargs["timeout"] = DEFAULT_TIMEOUT

        # Inject project User-Agent if the caller didn't already set one.
        # Session-level User-Agent (self.headers) is left alone — only
        # per-request headers are copied so we never mutate caller state.
        headers = dict(kwargs.get("headers") or {})
        session_ua = self.headers.get("User-Agent", "")
        has_per_request_ua = any(k.lower() == "user-agent" for k in headers)
        if not has_per_request_ua and (
            not session_ua or session_ua.startswith(_DEFAULT_REQUESTS_UA_PREFIX)
        ):
            headers["User-Agent"] = USER_AGENT
            kwargs["headers"] = headers

        return super().request(method, url, **kwargs)

    def send(
        self, request: requests.PreparedRequest, **kwargs
    ) -> requests.Response:
        """Validate every outgoing request against SSRF and run egress scope.

        This runs on **all** calls — both the initial request (routed
        here by ``requests.Session.request()``) and each redirect hop
        (routed here by ``resolve_redirects()``).  The initial URL is
        therefore validated twice (once in ``request()``, once here);
        this is intentional defense-in-depth.
        """
        url_guard = getattr(self, "url_guard", None)
        if url_guard is not None:
            url_guard(request.url or "")
        if (
            self.block_https_downgrade
            and request.url
            and request.url.lower().startswith("https://")
        ):
            # Session.send inherits only when the argument is absent.
            # An explicit None reaches the adapter with verification disabled.
            verify = kwargs.get("verify", self.verify)
            if not verify:
                raise ValueError(
                    "block_https_downgrade cannot be combined with verify=False"
                )
        if request.url and not ssrf_validator.validate_url(
            request.url,
            allow_localhost=self.allow_localhost,
            allow_private_ips=self.allow_private_ips,
            block_link_local=self.block_link_local,
        ):
            logger.warning(
                f"Request to {_redact(request.url)} blocked by SSRF validation"
            )
            # Note: This error says "security validation" while safe_get/
            # safe_post say "SSRF validation". The difference indicates the
            # source (session vs standalone function) in logs.
            raise ValueError(
                "Redirect target failed security validation (possible "
                f"SSRF): {_redact(request.url)}"
            )

        if request.url and self.egress_context is not None:
            from .egress.policy import PolicyDeniedError, evaluate_url

            decision = evaluate_url(request.url, self.egress_context)
            if not decision.allowed:
                raise PolicyDeniedError(decision, request.url)

        # Pin the validated IP for the connection this send performs. send()
        # runs for the initial request AND for every redirect hop
        # (resolve_redirects re-enters here), so each hop is independently
        # resolved, validated, and pinned — the address validated is the
        # address connected to.
        # Restore the resolve_redirects flag here rather than relying on the
        # generator's own finally: Session.send takes one item from it and
        # drops it, so that finally waits on collection. A leaked True would
        # disarm the body-scope refusal for the rest of the thread.
        preparing = self._preparing_next_only
        caller_stream = kwargs.get("stream", self.stream)
        # Session.send() preloads non-stream responses before returning.
        # Force header-first transport, then read through our size guard.
        kwargs["stream"] = True
        try:
            with dns_pinning.pinned_request(
                request.url or "",
                allow_localhost=self.allow_localhost,
                allow_private_ips=self.allow_private_ips,
                block_link_local=self.block_link_local,
            ) as pinned_addresses:
                if pinned_addresses and self.egress_context is not None:
                    from .egress.policy import (
                        Decision,
                        PolicyDeniedError,
                        evaluate_url,
                        url_host_is_declared_local,
                    )

                    # The URL-level decision above honors an operator-declared
                    # local hostname. Replacing that name with its pinned IP
                    # would lose the declaration, so inspect only the
                    # unconditional metadata block in that case. Otherwise
                    # enforce the run scope on every address that can connect.
                    declared_local = url_host_is_declared_local(
                        request.url, self.egress_context
                    )
                    for address_info in pinned_addresses:
                        address = str(address_info[4][0])
                        if declared_local:
                            if ssrf_validator.is_ip_blocked(
                                address,
                                allow_localhost=True,
                                allow_private_ips=True,
                            ):
                                # The pinning layer normally rejects this
                                # first. Audit without incrementing the run's
                                # quota: this is one already-refused request,
                                # not an additional user URL evaluation.
                                logger.bind(policy_audit=True).warning(
                                    "policy denied URL fetch (pinned address) "
                                    f"pinned_address={address} "
                                    "reason=blocked_metadata_ip "
                                    f"scope={self.egress_context.scope.value} "
                                    "counted=False"
                                )
                                raise PolicyDeniedError(
                                    Decision(False, "blocked_metadata_ip"),
                                    request.url or "",
                                )
                            continue
                        host = f"[{address}]" if ":" in address else address
                        decision = evaluate_url(
                            f"http://{host}/", self.egress_context
                        )
                        if not decision.allowed:
                            raise PolicyDeniedError(decision, request.url or "")
                response = super().send(request, **kwargs)
        finally:
            self._redirect_local.preparing_next_only = preparing
        return _finish_response(response, preload=not caller_stream)


# Exponential backoff schedule (seconds). Kept short: journal-quality
# downloads are run from a user request or a scheduled job, not from a
# time-sensitive hot path, so three retries over ~7 seconds is plenty
# without adding real latency.
_RETRY_BACKOFF_SECONDS = (1, 2, 4)

# HTTP status codes worth retrying (transient server / rate-limit errors).
_RETRYABLE_STATUS_CODES = frozenset({429, 500, 502, 503, 504})

# Upper bound on a honored Retry-After (seconds). RFC 7231 puts no
# ceiling on the header, so a hostile or misconfigured upstream could
# pin a worker via an arbitrarily large value. Cap here to bound the
# damage; legitimate waits (seconds to low minutes) pass through.
_MAX_RETRY_AFTER_SECONDS = 300


def _parse_retry_after(retry_after_raw: Optional[str]) -> Optional[int]:
    """Parse a ``Retry-After`` header value, clamped to ``[0, MAX]``.

    Returns ``None`` if the header is missing or unparseable, so the
    caller can fall back to the exponential-backoff schedule. Accepts
    both RFC 7231 forms: delay-seconds (integer) and HTTP-date. An
    HTTP-date that names no zone is read as UTC.
    """
    if retry_after_raw is None:
        return None
    try:
        seconds = int(retry_after_raw)
    except ValueError:
        try:
            retry_dt = email.utils.parsedate_to_datetime(retry_after_raw)
        except (ValueError, TypeError, OverflowError):
            # OverflowError: a numeric field too large for datetime().
            logger.debug(
                f"Unparseable Retry-After {retry_after_raw!r}; "
                f"using backoff schedule"
            )
            return None
        if retry_dt.tzinfo is None:
            # "-0000", a missing zone and the asctime form parse naive,
            # and naive minus aware raises TypeError. An HTTP-date is
            # always UTC (RFC 9110 section 5.6.7).
            retry_dt = retry_dt.replace(tzinfo=datetime.timezone.utc)
        now_utc = datetime.datetime.now(datetime.timezone.utc)
        seconds = int((retry_dt - now_utc).total_seconds())
    return max(0, min(seconds, _MAX_RETRY_AFTER_SECONDS))


def safe_get_with_retries(
    url: str,
    params: Optional[dict] = None,
    timeout: int = DEFAULT_TIMEOUT,
    allow_localhost: bool = False,
    allow_private_ips: bool = False,
    max_retries: int = 3,
    backoff_times: tuple = _RETRY_BACKOFF_SECONDS,
    consume_body: bool = False,
    **kwargs,
) -> requests.Response:
    """`safe_get` plus exponential-backoff retry on transient errors.

    Retries on:
      * ``requests.ConnectionError``
      * ``requests.Timeout``
      * HTTP ``429`` (rate limit) and ``5xx`` (server error)
      * (when ``consume_body=True``) body-read failures —
        ``ChunkedEncodingError``, ``ReadTimeout``, mid-stream
        ``ConnectionError``

    Honors the ``Retry-After`` header when present (falls back to the
    backoff schedule otherwise). SSRF-validation errors (``ValueError``)
    and non-retryable HTTP 4xx responses are not retried.

    Without ``consume_body``, ``safe_get`` reads the guarded body before
    it returns (unless the caller passes ``stream=True``), so a body-read
    failure is raised by ``safe_get`` itself and retried only if it is a
    ``ConnectionError`` or ``Timeout``; a ``ChunkedEncodingError`` is
    neither and propagates. With ``consume_body=True``, this wrapper
    passes ``stream=True`` and reads the body inside the retry loop, so
    transient mid-body failures such as ``ChunkedEncodingError`` are also
    retried. The cached body is still available via ``response.content``
    after return.

    Args:
        url: Target URL.
        params: Query parameters.
        timeout: Per-attempt socket timeout.
        allow_localhost: Forwarded to ``safe_get``.
        allow_private_ips: Forwarded to ``safe_get``.
        max_retries: Maximum retry attempts after the initial try.
        backoff_times: Per-attempt sleep seconds.
        consume_body: If True, read ``response.content`` inside the
            retry loop so body-read transients are retried. Use for
            large or chunk-transferred bodies (~MB+) where mid-stream
            disconnects are realistic. The body-guard's ``ValueError``
            (oversized body) is NOT retried — it propagates immediately.
        **kwargs: Forwarded to ``safe_get``.

    Returns:
        The first successful (or final-attempt) ``requests.Response``.
        When ``consume_body=True``, the body has already been read and
        is cached on the response.

    Raises:
        ValueError: If SSRF validation fails or the body guard rejects an
            oversized response. Retries do not help in either case.
        requests.RequestException: If every attempt fails.
    """
    attempt = 0
    if consume_body:
        # Keep body-read failures inside this retry loop. Without
        # stream=True, safe_get reads the body before returning, and a
        # ChunkedEncodingError raised there is neither a ConnectionError
        # nor a Timeout, so it would escape the loop unretried.
        kwargs["stream"] = True
    while True:
        try:
            response = safe_get(
                url,
                params=params,
                timeout=timeout,
                allow_localhost=allow_localhost,
                allow_private_ips=allow_private_ips,
                **kwargs,
            )
        except (requests.ConnectionError, requests.Timeout) as exc:
            if attempt >= max_retries:
                raise
            wait = backoff_times[min(attempt, len(backoff_times) - 1)]
            logger.warning(
                f"{exc.__class__.__name__} on {_redact(url)}; "
                f"retrying in {wait}s "
                f"(attempt {attempt + 1}/{max_retries})"
            )
            time.sleep(wait)
            attempt += 1
            continue

        # On the last attempt a retryable status falls through, so with
        # consume_body its body is read below like any other response's.
        if (
            response.status_code in _RETRYABLE_STATUS_CODES
            and attempt < max_retries
        ):
            parsed = _parse_retry_after(response.headers.get("Retry-After"))
            wait = (
                parsed
                if parsed is not None
                else backoff_times[min(attempt, len(backoff_times) - 1)]
            )
            logger.warning(
                f"HTTP {response.status_code} on {_redact(url)}; "
                f"retrying in {wait}s "
                f"(attempt {attempt + 1}/{max_retries})"
            )
            response.close()
            time.sleep(wait)
            attempt += 1
            continue

        if consume_body:
            try:
                # Force body read while still inside the retry loop.
                # ChunkedEncodingError / ReadTimeout / mid-stream
                # ConnectionError can fire here on large responses
                # from flaky upstreams. ReadTimeout is a Timeout
                # subclass but NOT a ConnectionError subclass, so the
                # except must list both ConnectionError and Timeout
                # — listing Timeout alone would miss ConnectError, and
                # listing ConnectionError alone would miss ReadTimeout.
                _ = response.content
            except (
                requests.exceptions.ChunkedEncodingError,
                requests.exceptions.ConnectionError,
                requests.exceptions.Timeout,
            ) as exc:
                # safe_close instead of bare close: a close() that
                # raises here would mask the original body-read error
                # we actually want to surface / retry on.
                safe_close(response, "response")
                if attempt >= max_retries:
                    raise
                wait = backoff_times[min(attempt, len(backoff_times) - 1)]
                logger.warning(
                    f"{exc.__class__.__name__} reading body of {_redact(url)}; "
                    f"retrying in {wait}s "
                    f"(attempt {attempt + 1}/{max_retries})"
                )
                time.sleep(wait)
                attempt += 1
                continue

        return response