Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
"""SSRF-safe HTTP fetching for crewai-tools.
``safe_get`` validates each URL and redirect hop, then fetches through a
session that pins TCP to the checked IP and ignores environment proxies.
"""
from __future__ import annotations
from typing import Any
from urllib.parse import urljoin, urlparse
import requests
from crewai_tools.security.safe_path import (
_BYPASS_HINT,
_is_escape_hatch_enabled,
validate_url,
)
from crewai_tools.security.ssrf_adapter import SSRFProtectedAdapter
_REDIRECT_STATUS_CODES = {301, 302, 303, 307, 308}
_STREAM_CHUNK_SIZE = 65536
_SENSITIVE_HEADER_NAMES = {
"authorization",
"cookie",
"proxy-authorization",
"x-api-key",
}
_SENSITIVE_HEADER_FRAGMENTS = ("api-key", "apikey", "secret", "token")
def _same_origin(previous_url: str, next_url: str) -> bool:
previous = urlparse(previous_url)
next_ = urlparse(next_url)
return (previous.scheme, previous.netloc) == (next_.scheme, next_.netloc)
def _is_sensitive_header(header: str) -> bool:
normalized = header.lower()
return (
normalized in _SENSITIVE_HEADER_NAMES
or normalized.startswith("authorization-")
or any(fragment in normalized for fragment in _SENSITIVE_HEADER_FRAGMENTS)
)
def _strip_cross_origin_credentials(request_kwargs: dict[str, Any]) -> dict[str, Any]:
sanitized = {**request_kwargs}
headers = sanitized.get("headers")
if headers:
sanitized["headers"] = {
key: value
for key, value in headers.items()
if not _is_sensitive_header(str(key))
}
sanitized.pop("cookies", None)
return sanitized
class _SafeSession(requests.Session):
"""Session that does not follow redirects unless the caller opts in."""
def get(self, url: str | bytes, **kwargs: Any) -> requests.Response:
kwargs.setdefault("allow_redirects", False)
return super().get(url, **kwargs)
def options(self, url: str | bytes, **kwargs: Any) -> requests.Response:
kwargs.setdefault("allow_redirects", False)
return super().options(url, **kwargs)
def request(
self, method: str | bytes, url: str | bytes, *args: Any, **kwargs: Any
) -> requests.Response:
kwargs.setdefault("allow_redirects", False)
return super().request(method, url, *args, **kwargs)
def create_safe_session() -> requests.Session:
"""Return a session that pins TCP to a validated peer and does not follow redirects."""
session = _SafeSession()
session.trust_env = False
session.proxies = {}
adapter = SSRFProtectedAdapter()
session.mount("http://", adapter)
session.mount("https://", adapter)
return session
def _reject_proxies(kwargs: dict[str, Any]) -> None:
proxies = kwargs.pop("proxies", None)
if proxies and not _is_escape_hatch_enabled():
raise ValueError(f"Proxies are not allowed for safe_get. {_BYPASS_HINT}")
kwargs["proxies"] = {}
def _attach_session(response: requests.Response, session: requests.Session) -> None:
"""Keep *session* alive until *response* is closed (needed for ``stream=True``)."""
original_close = response.close
def close_with_session() -> None:
try:
original_close()
finally:
session.close()
response.close = close_with_session # type: ignore[method-assign]
def _raw_get(url: str, **kwargs: Any) -> requests.Response:
"""GET through an SSRF-protected session."""
session = create_safe_session()
owns_session = True
try:
response = session.get(url, **kwargs)
if kwargs.get("stream"):
_attach_session(response, session)
owns_session = False
return response
finally:
if owns_session:
session.close()
def safe_get(url: str, *, max_redirects: int = 10, **kwargs: Any) -> requests.Response:
"""GET a URL while validating each redirect target before following it.
On success the hops are attached to the returned response's ``history`` and
are the caller's to close. On failure they are closed here: a caller given
an exception has no handle on them, and a streamed hop holds its connection
until its body is read or closed.
"""
current_url = validate_url(url)
_reject_proxies(kwargs)
request_kwargs = {**kwargs, "allow_redirects": False}
timeout = request_kwargs.pop("timeout", 30)
history: list[requests.Response] = []
redirects_followed = 0
try:
while True:
response = _raw_get(current_url, timeout=timeout, **request_kwargs)
if (
response.status_code not in _REDIRECT_STATUS_CODES
or "Location" not in response.headers
):
response.history = history
return response
if redirects_followed >= max_redirects:
response.close()
raise ValueError(f"Too many redirects while fetching URL: {url}")
location = response.headers.get("Location")
if not location:
response.history = history
return response
try:
redirect_url = validate_url(urljoin(response.url, location))
except ValueError:
response.close()
raise
if not _same_origin(current_url, redirect_url):
request_kwargs = _strip_cross_origin_credentials(request_kwargs)
history.append(response)
current_url = redirect_url
redirects_followed += 1
except BaseException:
for hop in history:
hop.close()
raise
def safe_get_bounded(
url: str,
*,
max_bytes: int,
timeout: float | tuple[float, float] = 30,
headers: dict[str, str] | None = None,
max_redirects: int = 10,
) -> tuple[bytes, str, str]:
"""GET a URL through :func:`safe_get`, refusing bodies over *max_bytes*.
The body is streamed and abandoned as soon as it crosses the limit, so an
oversized response costs one chunk of memory instead of all of it. The cap
counts decoded bytes, which is what a compressed response expands into --
``Content-Length`` describes the wire size and cannot bound that.
Args:
url: The URL to fetch.
max_bytes: Largest body to accept, in decoded bytes.
timeout: Request timeout, passed through to requests.
headers: Request headers.
max_redirects: Hops to follow before giving up.
Returns:
A ``(body, content_type, final_url)`` tuple, where *final_url* is the
last validated URL in the redirect chain.
Raises:
ValueError: If *max_bytes* is not positive, URL validation fails, the
redirect chain is too long, or the body exceeds *max_bytes*.
requests.RequestException: If the request fails or returns an error
status.
"""
if max_bytes <= 0:
raise ValueError(f"max_bytes must be positive, got {max_bytes}.")
response = safe_get(
url,
max_redirects=max_redirects,
headers=headers,
timeout=timeout,
stream=True,
)
try:
response.raise_for_status()
chunks: list[bytes] = []
total = 0
for chunk in response.iter_content(chunk_size=_STREAM_CHUNK_SIZE):
if not chunk:
continue
total += len(chunk)
if total > max_bytes:
# Names the URL that served the body, which after a redirect is
# not the one that was requested.
raise ValueError(
f"Response body from '{response.url}' exceeds the "
f"{max_bytes} byte limit."
)
chunks.append(chunk)
return (
b"".join(chunks),
response.headers.get("Content-Type", ""),
response.url,
)
finally:
# Under stream=True each hop holds its connection until the body is read,
# so the redirects need closing too, not just the response we return.
for hop in response.history:
hop.close()
response.close()