Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
Fix SSRF redirect bypass in scraping fetches (#6331)
Rip&Tear5d4851eac797
diff --git a/lib/crewai-tools/src/crewai_tools/rag/loaders/docs_site_loader.py b/lib/crewai-tools/src/crewai_tools/rag/loaders/docs_site_loader.pyindex 87b97266e..3f60bc35f 100644--- a/lib/crewai-tools/src/crewai_tools/rag/loaders/docs_site_loader.py+++ b/lib/crewai-tools/src/crewai_tools/rag/loaders/docs_site_loader.py@@ -8,6 +8,7 @@ import requestsfrom crewai_tools.rag.base_loader import BaseLoader, LoaderResultfrom crewai_tools.rag.source_content import SourceContent+from crewai_tools.security.safe_requests import safe_getclass DocsSiteLoader(BaseLoader):@@ -26,7 +27,7 @@ class DocsSiteLoader(BaseLoader):docs_url = source.sourcetry:- response = requests.get(docs_url, timeout=30)+ response = safe_get(docs_url, timeout=30)response.raise_for_status()except requests.RequestException as e:raise ValueError(diff --git a/lib/crewai-tools/src/crewai_tools/rag/loaders/docx_loader.py b/lib/crewai-tools/src/crewai_tools/rag/loaders/docx_loader.pyindex 201bc12c2..df736e348 100644--- a/lib/crewai-tools/src/crewai_tools/rag/loaders/docx_loader.py+++ b/lib/crewai-tools/src/crewai_tools/rag/loaders/docx_loader.py@@ -2,10 +2,9 @@ import osimport tempfilefrom typing import Any-import requests-from crewai_tools.rag.base_loader import BaseLoader, LoaderResultfrom crewai_tools.rag.source_content import SourceContent+from crewai_tools.security.safe_requests import safe_getclass DOCXLoader(BaseLoader):@@ -43,7 +42,7 @@ class DOCXLoader(BaseLoader):)try:- response = requests.get(url, headers=headers, timeout=30)+ response = safe_get(url, headers=headers, timeout=30)response.raise_for_status()# Create temporary file to save the DOCX contentdiff --git a/lib/crewai-tools/src/crewai_tools/rag/loaders/pdf_loader.py b/lib/crewai-tools/src/crewai_tools/rag/loaders/pdf_loader.pyindex 774de92d9..259181481 100644--- a/lib/crewai-tools/src/crewai_tools/rag/loaders/pdf_loader.py+++ b/lib/crewai-tools/src/crewai_tools/rag/loaders/pdf_loader.py@@ -6,10 +6,9 @@ import tempfilefrom typing import Anyfrom urllib.parse import urlparse-import requests-from crewai_tools.rag.base_loader import BaseLoader, LoaderResultfrom crewai_tools.rag.source_content import SourceContent+from crewai_tools.security.safe_requests import safe_getclass PDFLoader(BaseLoader):@@ -47,7 +46,7 @@ class PDFLoader(BaseLoader):)try:- response = requests.get(url, headers=headers, timeout=30)+ response = safe_get(url, headers=headers, timeout=30)response.raise_for_status()with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as temp_file:diff --git a/lib/crewai-tools/src/crewai_tools/rag/loaders/utils.py b/lib/crewai-tools/src/crewai_tools/rag/loaders/utils.pyindex f944070c8..f239a0a05 100644--- a/lib/crewai-tools/src/crewai_tools/rag/loaders/utils.py+++ b/lib/crewai-tools/src/crewai_tools/rag/loaders/utils.py@@ -23,7 +23,7 @@ def load_from_url(Raises:ValueError: If there's an error fetching the URL"""- import requests+ from crewai_tools.security.safe_requests import safe_getheaders = kwargs.get("headers",@@ -34,7 +34,7 @@ def load_from_url()try:- response = requests.get(url, headers=headers, timeout=30)+ response = safe_get(url, headers=headers, timeout=30)response.raise_for_status()return response.textexcept Exception as e:diff --git a/lib/crewai-tools/src/crewai_tools/rag/loaders/webpage_loader.py b/lib/crewai-tools/src/crewai_tools/rag/loaders/webpage_loader.pyindex 5d9a2d180..a22c09e66 100644--- a/lib/crewai-tools/src/crewai_tools/rag/loaders/webpage_loader.py+++ b/lib/crewai-tools/src/crewai_tools/rag/loaders/webpage_loader.py@@ -2,10 +2,10 @@ import refrom typing import Any, Finalfrom bs4 import BeautifulSoup-import requestsfrom crewai_tools.rag.base_loader import BaseLoader, LoaderResultfrom crewai_tools.rag.source_content import SourceContent+from crewai_tools.security.safe_requests import safe_get_SPACES_PATTERN: Final[re.Pattern[str]] = re.compile(r"[ \t]+")@@ -25,7 +25,7 @@ class WebPageLoader(BaseLoader):)try:- response = requests.get(url, timeout=15, headers=headers)+ response = safe_get(url, timeout=15, headers=headers)response.encoding = response.apparent_encodingsoup = BeautifulSoup(response.text, "html.parser")diff --git a/lib/crewai-tools/src/crewai_tools/security/safe_requests.py b/lib/crewai-tools/src/crewai_tools/security/safe_requests.pynew file mode 100644index 000000000..505a5cdb6--- /dev/null+++ b/lib/crewai-tools/src/crewai_tools/security/safe_requests.py@@ -0,0 +1,88 @@+"""HTTP helpers that preserve crewai-tools URL safety checks."""++from __future__ import annotations++from typing import Any+from urllib.parse import urljoin, urlparse++import requests++from crewai_tools.security.safe_path import validate_url+++_REDIRECT_STATUS_CODES = {301, 302, 303, 307, 308}+_SENSITIVE_HEADER_NAMES = {+ "authorization",+ "cookie",+ "proxy-authorization",+ "x-api-key",+}+_SENSITIVE_HEADER_FRAGMENTS = ("api-key", "apikey", "secret", "token")+++def _same_origin(previous_url: str, next_url: str) -> bool:+ previous = urlparse(previous_url)+ next_ = urlparse(next_url)+ return (previous.scheme, previous.netloc) == (next_.scheme, next_.netloc)+++def _is_sensitive_header(header: str) -> bool:+ normalized = header.lower()+ return (+ normalized in _SENSITIVE_HEADER_NAMES+ or normalized.startswith("authorization-")+ or any(fragment in normalized for fragment in _SENSITIVE_HEADER_FRAGMENTS)+ )+++def _strip_cross_origin_credentials(request_kwargs: dict[str, Any]) -> dict[str, Any]:+ sanitized = {**request_kwargs}+ headers = sanitized.get("headers")+ if headers:+ sanitized["headers"] = {+ key: value+ for key, value in headers.items()+ if not _is_sensitive_header(str(key))+ }+ sanitized.pop("cookies", None)+ return sanitized+++def safe_get(url: str, *, max_redirects: int = 10, **kwargs: Any) -> requests.Response:+ """GET a URL while validating each redirect target before following it."""+ current_url = validate_url(url)+ request_kwargs = {**kwargs, "allow_redirects": False}+ timeout = request_kwargs.pop("timeout", 30)+ history: list[requests.Response] = []+ redirects_followed = 0++ while True:+ response = requests.get(current_url, timeout=timeout, **request_kwargs)+ if (+ response.status_code not in _REDIRECT_STATUS_CODES+ or "Location" not in response.headers+ ):+ response.history = history+ return response++ if redirects_followed >= max_redirects:+ response.close()+ raise ValueError(f"Too many redirects while fetching URL: {url}")++ location = response.headers.get("Location")+ if not location:+ response.history = history+ return response++ try:+ redirect_url = validate_url(urljoin(response.url, location))+ except ValueError:+ response.close()+ raise++ if not _same_origin(current_url, redirect_url):+ request_kwargs = _strip_cross_origin_credentials(request_kwargs)++ history.append(response)+ current_url = redirect_url+ redirects_followed += 1diff --git a/lib/crewai-tools/src/crewai_tools/tools/scrape_element_from_website/scrape_element_from_website.py b/lib/crewai-tools/src/crewai_tools/tools/scrape_element_from_website/scrape_element_from_website.pyindex 7bba12b72..535bbf9d5 100644--- a/lib/crewai-tools/src/crewai_tools/tools/scrape_element_from_website/scrape_element_from_website.py+++ b/lib/crewai-tools/src/crewai_tools/tools/scrape_element_from_website/scrape_element_from_website.py@@ -3,9 +3,8 @@ from typing import Anyfrom crewai.tools import BaseToolfrom pydantic import BaseModel, Field-import requests-from crewai_tools.security.safe_path import validate_url+from crewai_tools.security.safe_requests import safe_gettry:@@ -83,8 +82,7 @@ class ScrapeElementFromWebsiteTool(BaseTool):if website_url is None or css_element is None:raise ValueError("Both website_url and css_element must be provided.")- website_url = validate_url(website_url)- page = requests.get(+ page = safe_get(website_url,headers=self.headers,cookies=self.cookies if self.cookies else {},diff --git a/lib/crewai-tools/src/crewai_tools/tools/scrape_website_tool/scrape_website_tool.py b/lib/crewai-tools/src/crewai_tools/tools/scrape_website_tool/scrape_website_tool.pyindex d297dfe08..62bd7b0be 100644--- a/lib/crewai-tools/src/crewai_tools/tools/scrape_website_tool/scrape_website_tool.py+++ b/lib/crewai-tools/src/crewai_tools/tools/scrape_website_tool/scrape_website_tool.py@@ -3,9 +3,8 @@ import refrom typing import Anyfrom pydantic import Field-import requests-from crewai_tools.security.safe_path import validate_url+from crewai_tools.security.safe_requests import safe_gettry:@@ -75,8 +74,7 @@ class ScrapeWebsiteTool(BaseTool):if website_url is None:raise ValueError("Website URL must be provided.")- website_url = validate_url(website_url)- page = requests.get(+ page = safe_get(website_url,timeout=15,headers=self.headers,diff --git a/lib/crewai-tools/tests/rag/conftest.py b/lib/crewai-tools/tests/rag/conftest.pynew file mode 100644index 000000000..a0301ae87--- /dev/null+++ b/lib/crewai-tools/tests/rag/conftest.py@@ -0,0 +1,28 @@+from __future__ import annotations++import socket+from typing import Any++import pytest++[email protected](autouse=True)+def public_example_dns(monkeypatch: pytest.MonkeyPatch) -> None:+ original_getaddrinfo = socket.getaddrinfo++ def fake_getaddrinfo(+ host: str, port: int, *args: Any, **kwargs: Any+ ) -> list[tuple[Any, ...]]:+ if host in {"example.com", "api.example.com"}:+ return [+ (+ socket.AF_INET,+ socket.SOCK_STREAM,+ 6,+ "",+ ("93.184.216.34", port),+ )+ ]+ return original_getaddrinfo(host, port, *args, **kwargs)++ monkeypatch.setattr(socket, "getaddrinfo", fake_getaddrinfo)diff --git a/lib/crewai-tools/tests/utilities/test_safe_requests.py b/lib/crewai-tools/tests/utilities/test_safe_requests.pynew file mode 100644index 000000000..f45dd86c6--- /dev/null+++ b/lib/crewai-tools/tests/utilities/test_safe_requests.py@@ -0,0 +1,177 @@+"""Tests for redirect-aware safe HTTP helpers."""++from __future__ import annotations++import socket+from io import BytesIO+from typing import Any++import pytest+import requests++from crewai_tools.security.safe_requests import safe_get+++def _response(url: str, status_code: int, *, location: str | None = None) -> requests.Response:+ response = requests.Response()+ response.status_code = status_code+ response.url = url+ response._content = b"ok"+ response.raw = BytesIO()+ if location is not None:+ response.headers["Location"] = location+ return response+++def public_dns(monkeypatch: pytest.MonkeyPatch) -> None:+ original_getaddrinfo = socket.getaddrinfo++ def fake_getaddrinfo(+ host: str, port: int, *args: Any, **kwargs: Any+ ) -> list[tuple[Any, ...]]:+ if host in {"public.example", "safe.example"}:+ return [+ (+ socket.AF_INET,+ socket.SOCK_STREAM,+ 6,+ "",+ ("93.184.216.34", port),+ )+ ]+ return original_getaddrinfo(host, port, *args, **kwargs)++ monkeypatch.setattr(socket, "getaddrinfo", fake_getaddrinfo)+++def test_safe_get_blocks_direct_internal_url() -> None:+ with pytest.raises(ValueError, match="private/reserved IP"):+ safe_get("http://127.0.0.1/admin", timeout=15)+++def _mock_get(monkeypatch: pytest.MonkeyPatch, get_response: Any) -> None:+ monkeypatch.setattr(+ "crewai_tools.security.safe_requests.requests.get",+ get_response,+ )+++def test_safe_get_blocks_redirect_to_internal_url(+ monkeypatch: pytest.MonkeyPatch, public_dns: None+) -> None:+ requested_urls: list[str] = []++ def fake_get(url: str, **kwargs: Any) -> requests.Response:+ requested_urls.append(url)+ assert kwargs["allow_redirects"] is False+ return _response(url, 302, location="http://127.0.0.1/admin")++ _mock_get(monkeypatch, fake_get)++ with pytest.raises(ValueError, match="private/reserved IP"):+ safe_get("http://public.example/start", timeout=15)++ assert requested_urls == ["http://public.example/start"]+++def test_safe_get_follows_safe_relative_redirect(+ monkeypatch: pytest.MonkeyPatch, public_dns: None+) -> None:+ requested_urls: list[str] = []++ def fake_get(url: str, **kwargs: Any) -> requests.Response:+ requested_urls.append(url)+ assert kwargs["allow_redirects"] is False+ if url == "http://public.example/start":+ return _response(url, 302, location="/final")+ return _response(url, 200)++ _mock_get(monkeypatch, fake_get)++ response = safe_get("http://public.example/start", timeout=15)++ assert response.status_code == 200+ assert response.url == "http://public.example/final"+ assert requested_urls == [+ "http://public.example/start",+ "http://public.example/final",+ ]+ assert len(response.history) == 1+++def test_safe_get_fails_closed_after_too_many_redirects(+ monkeypatch: pytest.MonkeyPatch, public_dns: None+) -> None:+ def fake_get(url: str, **kwargs: Any) -> requests.Response:+ return _response(url, 302, location="http://safe.example/again")++ _mock_get(monkeypatch, fake_get)++ with pytest.raises(ValueError, match="Too many redirects"):+ safe_get("http://public.example/start", max_redirects=1, timeout=15)+++def test_safe_get_strips_credentials_on_cross_origin_redirect(+ monkeypatch: pytest.MonkeyPatch, public_dns: None+) -> None:+ requests_made: list[tuple[str, dict[str, Any]]] = []++ def fake_get(url: str, **kwargs: Any) -> requests.Response:+ requests_made.append((url, kwargs))+ if url == "http://public.example/start":+ return _response(url, 302, location="http://safe.example/final")+ return _response(url, 200)++ _mock_get(monkeypatch, fake_get)++ response = safe_get(+ "http://public.example/start",+ timeout=15,+ headers={+ "Authorization": "Bearer token",+ "Authorization-Custom": "secret token",+ "Cookie": "session=abc",+ "X-API-Key": "api key",+ "X-CrewAI-Token": "crewai token",+ "User-Agent": "crewai-test",+ },+ cookies={"session": "abc"},+ )++ assert response.status_code == 200+ assert requests_made[0][1]["headers"] == {+ "Authorization": "Bearer token",+ "Authorization-Custom": "secret token",+ "Cookie": "session=abc",+ "X-API-Key": "api key",+ "X-CrewAI-Token": "crewai token",+ "User-Agent": "crewai-test",+ }+ assert requests_made[0][1]["cookies"] == {"session": "abc"}+ assert requests_made[1][1]["headers"] == {"User-Agent": "crewai-test"}+ assert "cookies" not in requests_made[1][1]+++def test_safe_get_preserves_credentials_on_same_origin_redirect(+ monkeypatch: pytest.MonkeyPatch, public_dns: None+) -> None:+ requests_made: list[tuple[str, dict[str, Any]]] = []++ def fake_get(url: str, **kwargs: Any) -> requests.Response:+ requests_made.append((url, kwargs))+ if url == "http://public.example/start":+ return _response(url, 302, location="/final")+ return _response(url, 200)++ _mock_get(monkeypatch, fake_get)++ safe_get(+ "http://public.example/start",+ timeout=15,+ headers={"Authorization": "Bearer token"},+ cookies={"session": "abc"},+ )++ assert requests_made[1][1]["headers"] == {"Authorization": "Bearer token"}+ assert requests_made[1][1]["cookies"] == {"session": "abc"}