Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

feat(tools): add URLReadTool for reading arbitrary URLs (#6834)

João Moura0c74f23596b5
diff --git a/lib/crewai-tools/src/crewai_tools/__init__.py b/lib/crewai-tools/src/crewai_tools/__init__.py
index 1ffdf96b4..a50444888 100644
--- a/lib/crewai-tools/src/crewai_tools/__init__.py
+++ b/lib/crewai-tools/src/crewai_tools/__init__.py
@@ -209,6 +209,7 @@ from crewai_tools.tools.tavily_research_tool.tavily_research_tool import (
)
from crewai_tools.tools.tavily_search_tool.tavily_search_tool import TavilySearchTool
from crewai_tools.tools.txt_search_tool.txt_search_tool import TXTSearchTool
+from crewai_tools.tools.url_read_tool.url_read_tool import URLReadTool
from crewai_tools.tools.vision_tool.vision_tool import VisionTool
from crewai_tools.tools.wait_tool.wait_tool import WaitTool
from crewai_tools.tools.weaviate_tool.vector_search import WeaviateVectorSearchTool
@@ -327,6 +328,7 @@ __all__ = [
"TavilyGetResearchTool",
"TavilyResearchTool",
"TavilySearchTool",
+ "URLReadTool",
"VisionTool",
"WaitTool",
"WeaviateVectorSearchTool",
diff --git a/lib/crewai-tools/src/crewai_tools/rag/loaders/pdf_loader.py b/lib/crewai-tools/src/crewai_tools/rag/loaders/pdf_loader.py
index 259181481..84fd45d96 100644
--- a/lib/crewai-tools/src/crewai_tools/rag/loaders/pdf_loader.py
+++ b/lib/crewai-tools/src/crewai_tools/rag/loaders/pdf_loader.py
@@ -2,13 +2,17 @@
import os
from pathlib import Path
-import tempfile
from typing import Any
from urllib.parse import urlparse
from crewai_tools.rag.base_loader import BaseLoader, LoaderResult
from crewai_tools.rag.source_content import SourceContent
-from crewai_tools.security.safe_requests import safe_get
+from crewai_tools.security.safe_requests import safe_get_bounded
+
+
+# Remote PDFs are held in memory for the whole extraction, so the download needs
+# a ceiling. Override per call with the ``max_bytes`` kwarg to ``load``.
+DEFAULT_MAX_PDF_BYTES = 50 * 1024 * 1024
class PDFLoader(BaseLoader):
@@ -24,18 +28,24 @@ class PDFLoader(BaseLoader):
return False
@staticmethod
- def _download_from_url(url: str, kwargs: dict[str, Any]) -> str:
- """Download PDF from a URL to a temporary file and return its path.
+ def _fetch_from_url(url: str, kwargs: dict[str, Any]) -> bytes:
+ """Download a PDF from a URL and return its bytes.
+
+ The content stays in memory rather than going to a temporary file: the
+ whole body has to be buffered either way, and a temp file would need
+ unlinking on every error path to avoid leaving files behind. Because it
+ is held in memory, the download is capped.
Args:
url: The URL to download from.
- kwargs: Optional dict that may contain custom headers.
+ kwargs: Optional dict that may contain custom ``headers`` and a
+ ``max_bytes`` ceiling for the download.
Returns:
- Path to the temporary file containing the PDF.
+ The raw PDF content.
Raises:
- ValueError: If the download fails.
+ ValueError: If the download fails or exceeds the size ceiling.
"""
headers = kwargs.get(
"headers",
@@ -46,12 +56,13 @@ class PDFLoader(BaseLoader):
)
try:
- response = safe_get(url, headers=headers, timeout=30)
- response.raise_for_status()
-
- with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as temp_file:
- temp_file.write(response.content)
- return temp_file.name
+ body, _content_type, _final_url = safe_get_bounded(
+ url,
+ max_bytes=kwargs.get("max_bytes", DEFAULT_MAX_PDF_BYTES),
+ headers=headers,
+ timeout=30,
+ )
+ return body
except Exception as e:
raise ValueError(f"Failed to download PDF from {url}: {e!s}") from e
@@ -93,21 +104,25 @@ class PDFLoader(BaseLoader):
try:
if is_url:
- local_path = self._download_from_url(file_path, kwargs)
- doc = pymupdf.open(local_path)
+ doc = pymupdf.open(
+ stream=self._fetch_from_url(file_path, kwargs), filetype="pdf"
+ )
else:
if not os.path.isfile(file_path):
raise FileNotFoundError(f"PDF file not found: {file_path}")
doc = pymupdf.open(file_path)
- metadata["num_pages"] = len(doc)
-
- for page_num, page in enumerate(doc, 1):
- page_text = page.get_text()
- if page_text.strip():
- text_content.append(f"Page {page_num}:\n{page_text}")
-
- doc.close()
+ # Closed in a finally so a failure mid-extraction still releases the
+ # document handle.
+ try:
+ metadata["num_pages"] = len(doc)
+
+ for page_num, page in enumerate(doc, 1):
+ page_text = page.get_text()
+ if page_text.strip():
+ text_content.append(f"Page {page_num}:\n{page_text}")
+ finally:
+ doc.close()
except FileNotFoundError:
raise
except Exception as e:
diff --git a/lib/crewai-tools/src/crewai_tools/security/safe_requests.py b/lib/crewai-tools/src/crewai_tools/security/safe_requests.py
index 505a5cdb6..12765b022 100644
--- a/lib/crewai-tools/src/crewai_tools/security/safe_requests.py
+++ b/lib/crewai-tools/src/crewai_tools/security/safe_requests.py
@@ -11,6 +11,7 @@ from crewai_tools.security.safe_path import validate_url
_REDIRECT_STATUS_CODES = {301, 302, 303, 307, 308}
+_STREAM_CHUNK_SIZE = 65536
_SENSITIVE_HEADER_NAMES = {
"authorization",
"cookie",
@@ -49,40 +50,124 @@ def _strip_cross_origin_credentials(request_kwargs: dict[str, Any]) -> dict[str,
def safe_get(url: str, *, max_redirects: int = 10, **kwargs: Any) -> requests.Response:
- """GET a URL while validating each redirect target before following it."""
+ """GET a URL while validating each redirect target before following it.
+
+ On success the hops are attached to the returned response's ``history`` and
+ are the caller's to close. On failure they are closed here: a caller given
+ an exception has no handle on them, and a streamed hop holds its connection
+ until its body is read or closed.
+ """
current_url = validate_url(url)
request_kwargs = {**kwargs, "allow_redirects": False}
timeout = request_kwargs.pop("timeout", 30)
history: list[requests.Response] = []
redirects_followed = 0
- while True:
- response = requests.get(current_url, timeout=timeout, **request_kwargs)
- if (
- response.status_code not in _REDIRECT_STATUS_CODES
- or "Location" not in response.headers
- ):
- response.history = history
- return response
-
- if redirects_followed >= max_redirects:
- response.close()
- raise ValueError(f"Too many redirects while fetching URL: {url}")
-
- location = response.headers.get("Location")
- if not location:
- response.history = history
- return response
-
- try:
- redirect_url = validate_url(urljoin(response.url, location))
- except ValueError:
- response.close()
- raise
-
- if not _same_origin(current_url, redirect_url):
- request_kwargs = _strip_cross_origin_credentials(request_kwargs)
-
- history.append(response)
- current_url = redirect_url
- redirects_followed += 1
+ try:
+ while True:
+ response = requests.get(current_url, timeout=timeout, **request_kwargs)
+ if (
+ response.status_code not in _REDIRECT_STATUS_CODES
+ or "Location" not in response.headers
+ ):
+ response.history = history
+ return response
+
+ if redirects_followed >= max_redirects:
+ response.close()
+ raise ValueError(f"Too many redirects while fetching URL: {url}")
+
+ location = response.headers.get("Location")
+ if not location:
+ response.history = history
+ return response
+
+ try:
+ redirect_url = validate_url(urljoin(response.url, location))
+ except ValueError:
+ response.close()
+ raise
+
+ if not _same_origin(current_url, redirect_url):
+ request_kwargs = _strip_cross_origin_credentials(request_kwargs)
+
+ history.append(response)
+ current_url = redirect_url
+ redirects_followed += 1
+ except BaseException:
+ for hop in history:
+ hop.close()
+ raise
+
+
+def safe_get_bounded(
+ url: str,
+ *,
+ max_bytes: int,
+ timeout: float | tuple[float, float] = 30,
+ headers: dict[str, str] | None = None,
+ max_redirects: int = 10,
+) -> tuple[bytes, str, str]:
+ """GET a URL through :func:`safe_get`, refusing bodies over *max_bytes*.
+
+ The body is streamed and abandoned as soon as it crosses the limit, so an
+ oversized response costs one chunk of memory instead of all of it. The cap
+ counts decoded bytes, which is what a compressed response expands into --
+ ``Content-Length`` describes the wire size and cannot bound that.
+
+ Args:
+ url: The URL to fetch.
+ max_bytes: Largest body to accept, in decoded bytes.
+ timeout: Request timeout, passed through to requests.
+ headers: Request headers.
+ max_redirects: Hops to follow before giving up.
+
+ Returns:
+ A ``(body, content_type, final_url)`` tuple, where *final_url* is the
+ last validated URL in the redirect chain.
+
+ Raises:
+ ValueError: If *max_bytes* is not positive, URL validation fails, the
+ redirect chain is too long, or the body exceeds *max_bytes*.
+ requests.RequestException: If the request fails or returns an error
+ status.
+ """
+ if max_bytes <= 0:
+ raise ValueError(f"max_bytes must be positive, got {max_bytes}.")
+
+ response = safe_get(
+ url,
+ max_redirects=max_redirects,
+ headers=headers,
+ timeout=timeout,
+ stream=True,
+ )
+ try:
+ response.raise_for_status()
+
+ chunks: list[bytes] = []
+ total = 0
+ for chunk in response.iter_content(chunk_size=_STREAM_CHUNK_SIZE):
+ if not chunk:
+ continue
+ total += len(chunk)
+ if total > max_bytes:
+ # Names the URL that served the body, which after a redirect is
+ # not the one that was requested.
+ raise ValueError(
+ f"Response body from '{response.url}' exceeds the "
+ f"{max_bytes} byte limit."
+ )
+ chunks.append(chunk)
+
+ return (
+ b"".join(chunks),
+ response.headers.get("Content-Type", ""),
+ response.url,
+ )
+ finally:
+ # Under stream=True each hop holds its connection until the body is read,
+ # so the redirects need closing too, not just the response we return.
+ for hop in response.history:
+ hop.close()
+ response.close()
diff --git a/lib/crewai-tools/src/crewai_tools/tools/__init__.py b/lib/crewai-tools/src/crewai_tools/tools/__init__.py
index 4651bbdd3..2653490f7 100644
--- a/lib/crewai-tools/src/crewai_tools/tools/__init__.py
+++ b/lib/crewai-tools/src/crewai_tools/tools/__init__.py
@@ -196,6 +196,7 @@ from crewai_tools.tools.tavily_research_tool.tavily_research_tool import (
)
from crewai_tools.tools.tavily_search_tool.tavily_search_tool import TavilySearchTool
from crewai_tools.tools.txt_search_tool.txt_search_tool import TXTSearchTool
+from crewai_tools.tools.url_read_tool.url_read_tool import URLReadTool
from crewai_tools.tools.vision_tool.vision_tool import VisionTool
from crewai_tools.tools.wait_tool.wait_tool import WaitTool
from crewai_tools.tools.weaviate_tool.vector_search import WeaviateVectorSearchTool
@@ -310,6 +311,7 @@ __all__ = [
"TavilyGetResearchTool",
"TavilyResearchTool",
"TavilySearchTool",
+ "URLReadTool",
"VisionTool",
"WaitTool",
"WeaviateVectorSearchTool",
diff --git a/lib/crewai-tools/src/crewai_tools/tools/url_read_tool/__init__.py b/lib/crewai-tools/src/crewai_tools/tools/url_read_tool/__init__.py
new file mode 100644
index 000000000..37efdfc9b
--- /dev/null
+++ b/lib/crewai-tools/src/crewai_tools/tools/url_read_tool/__init__.py
@@ -0,0 +1,4 @@
+from crewai_tools.tools.url_read_tool.url_read_tool import URLReadTool
+
+
+__all__ = ["URLReadTool"]
diff --git a/lib/crewai-tools/src/crewai_tools/tools/url_read_tool/url_read_tool.py b/lib/crewai-tools/src/crewai_tools/tools/url_read_tool/url_read_tool.py
new file mode 100644
index 000000000..5a2c83288
--- /dev/null
+++ b/lib/crewai-tools/src/crewai_tools/tools/url_read_tool/url_read_tool.py
@@ -0,0 +1,379 @@
+"""Tool for reading the content at an arbitrary URL as text."""
+
+from __future__ import annotations
+
+from io import BytesIO
+from itertools import islice
+import re
+from typing import Any, Final
+from urllib.parse import urlparse
+
+from crewai.tools import BaseTool
+from pydantic import BaseModel, Field
+import requests
+
+from crewai_tools.security.safe_path import format_error_for_display
+from crewai_tools.security.safe_requests import safe_get_bounded
+
+
+_DEFAULT_MAX_BYTES: Final[int] = 5 * 1024 * 1024
+_DEFAULT_TIMEOUT: Final[int] = 30
+
+_PDF_TYPE: Final[str] = "application/pdf"
+_DOCX_TYPE: Final[str] = (
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+)
+_HTML_TYPES: Final[frozenset[str]] = frozenset({"text/html", "application/xhtml+xml"})
+_TEXT_TYPES: Final[frozenset[str]] = frozenset(
+ {
+ "application/csv",
+ "application/javascript",
+ "application/json",
+ "application/sql",
+ "application/x-ndjson",
+ "application/x-yaml",
+ "application/xml",
+ "application/yaml",
+ }
+)
+_TEXT_TYPE_SUFFIXES: Final[tuple[str, ...]] = ("+json", "+xml", "+yaml")
+
+# Servers commonly serve static files as octet-stream, or send no type at all,
+# so the extension is consulted when the header carries no usable answer.
+_UNINFORMATIVE_TYPES: Final[frozenset[str]] = frozenset(
+ {"", "application/octet-stream", "binary/octet-stream"}
+)
+_EXTENSION_TYPES: Final[dict[str, str]] = {
+ ".csv": "text/csv",
+ ".docx": _DOCX_TYPE,
+ ".htm": "text/html",
+ ".html": "text/html",
+ ".json": "application/json",
+ ".md": "text/markdown",
+ ".pdf": _PDF_TYPE,
+ ".txt": "text/plain",
+ ".xml": "application/xml",
+ ".yaml": "application/yaml",
+ ".yml": "application/yaml",
+}
+
+_SPACES_PATTERN: Final[re.Pattern[str]] = re.compile(r"[ \t]+")
+_NEWLINE_PATTERN: Final[re.Pattern[str]] = re.compile(r"\s+\n\s+")
+
+
+def _charset_from_content_type(content_type: str) -> str | None:
+ """Return the charset parameter of a Content-Type header, if it has one."""
+ for parameter in content_type.split(";")[1:]:
+ name, _, value = parameter.partition("=")
+ if name.strip().lower() == "charset":
+ return value.strip().strip('"') or None
+ return None
+
+
+class URLReadToolSchema(BaseModel):
+ """Input for URLReadTool."""
+
+ url: str = Field(
+ ...,
+ description=(
+ "The http:// or https:// URL to read. Addresses that resolve to "
+ "private or internal networks are refused."
+ ),
+ )
+ start_line: int | None = Field(
+ 1, ge=1, description="Line number to start reading from (1-indexed)"
+ )
+ line_count: int | None = Field(
+ None,
+ ge=1,
+ description="Number of lines to read. If None, reads the entire content",
+ )
+
+
+class URLReadTool(BaseTool):
+ """Read the content at an arbitrary URL and return it as text.
+
+ Unlike :class:`~crewai_tools.tools.file_read_tool.file_read_tool.FileReadTool`,
+ which is confined to the local filesystem, this tool performs network
+ requests to addresses the caller -- often an LLM -- chooses at runtime. It
+ is a separate tool for exactly that reason: granting it is granting network
+ egress, and that should be a deliberate choice rather than a flag on a
+ filesystem tool.
+
+ Responses are decoded to text according to their content type. PDF and DOCX
+ bodies have their text extracted, HTML is stripped to visible text, and
+ text-shaped types (plain text, Markdown, JSON, XML, YAML, CSV) are decoded
+ as-is. Any other type is refused rather than returned as base64, keeping
+ this tool's output text-only.
+
+ Security:
+ Requests go through :func:`~crewai_tools.security.safe_requests.safe_get_bounded`,
+ which resolves each hostname and rejects it when any resolved address is
+ private, loopback, link-local, or otherwise reserved -- covering cloud
+ metadata endpoints and internal services. Redirects are never followed
+ automatically: every hop is revalidated, and credentials are dropped on
+ cross-origin hops. Bodies over ``max_bytes`` are abandoned mid-stream.
+
+ Two risks are not closed here. Validation resolves the hostname and
+ requests resolves it again when connecting, so a DNS entry that changes
+ between those lookups can still redirect the connection (DNS
+ rebinding); closing that requires pinning the connection to the
+ validated address. And the returned text is untrusted remote content
+ flowing into an agent's context -- a fetched page can attempt to
+ instruct the agent. Neither is addressable by input validation alone;
+ network egress policy and prompt-level handling cover them.
+
+ Args:
+ max_bytes (int): Largest response body to accept, in decoded bytes.
+ Defaults to 5 MiB.
+ timeout (float): Per-request timeout in seconds. Defaults to 30.
+ headers (Optional[dict[str, str]]): Extra request headers. Developer
+ supplied, not chosen by the model.
+ encoding (Optional[str]): Force a text encoding instead of honoring the
+ charset the server declares.
+ **kwargs: Additional keyword arguments passed to BaseTool.
+
+ Example:
+ >>> tool = URLReadTool()
+ >>> content = tool.run(url="https://example.com/report.pdf")
+ >>> head = tool.run(url="https://example.com/data.csv", line_count=20)
+ """
+
+ name: str = "Read content from a URL"
+ description: str = (
+ "A tool that reads the content at a URL and returns it as text. To use "
+ "this tool, provide a 'url' parameter with an http:// or https:// "
+ "address. PDF, DOCX, HTML, JSON, XML, CSV and plain-text responses are "
+ "converted to text; other binary types are rejected. URLs that resolve "
+ "to private or internal network addresses are refused, as are responses "
+ "over the tool's size limit. Optionally provide 'start_line' and "
+ "'line_count' to read only part of the content."
+ )
+ args_schema: type[BaseModel] = URLReadToolSchema
+ max_bytes: int = _DEFAULT_MAX_BYTES
+ timeout: float = _DEFAULT_TIMEOUT
+ headers: dict[str, str] | None = None
+ encoding: str | None = None
+
+ def __init__(
+ self,
+ max_bytes: int = _DEFAULT_MAX_BYTES,
+ timeout: float = _DEFAULT_TIMEOUT,
+ headers: dict[str, str] | None = None,
+ encoding: str | None = None,
+ **kwargs: Any,
+ ) -> None:
+ """Initialize the URLReadTool.
+
+ Args:
+ max_bytes: Largest response body to accept, in decoded bytes.
+ timeout: Per-request timeout in seconds.
+ headers: Extra request headers.
+ encoding: Force a text encoding instead of the server's charset.
+ **kwargs: Additional keyword arguments passed to BaseTool.
+ """
+ super().__init__(**kwargs)
+ self.max_bytes = max_bytes
+ self.timeout = timeout
+ self.headers = headers
+ self.encoding = encoding
+
+ def _request_headers(self) -> dict[str, str]:
+ """Return the headers to send, with caller headers taking precedence."""
+ return {
+ "Accept": "*/*",
+ "User-Agent": "Mozilla/5.0 (compatible; crewai-tools URLReadTool)",
+ **(self.headers or {}),
+ }
+
+ @staticmethod
+ def _classify(media_type: str) -> str | None:
+ """Map a media type onto the extractor that handles it."""
+ if media_type == _PDF_TYPE:
+ return "pdf"
+ if media_type == _DOCX_TYPE:
+ return "docx"
+ if media_type in _HTML_TYPES:
+ return "html"
+ if (
+ media_type.startswith("text/")
+ or media_type in _TEXT_TYPES
+ or media_type.endswith(_TEXT_TYPE_SUFFIXES)
+ ):
+ return "text"
+ return None
+
+ def _resolve_kind(self, content_type: str, *urls: str) -> str | None:
+ """Decide how to extract text, by content type then by URL extension.
+
+ Args:
+ content_type: The raw Content-Type header value.
+ *urls: URLs to consult for an extension, most authoritative first.
+ A ``.pdf`` link that redirects to an extensionless CDN or
+ presigned path only carries its type on the requested URL, so
+ both ends of the chain are worth checking.
+
+ Returns:
+ The extractor name, or None when the content type is unsupported.
+ """
+ declared = content_type.split(";", 1)[0].strip().lower()
+ if declared not in _UNINFORMATIVE_TYPES:
+ return self._classify(declared)
+
+ for url in urls:
+ path = urlparse(url).path.lower()
+ for extension, media_type in _EXTENSION_TYPES.items():
+ if path.endswith(extension):
+ return self._classify(media_type)
+ return None
+
+ def _decode(self, body: bytes, content_type: str) -> str:
+ """Decode *body* using the configured, declared, or default encoding.
+
+ Falls back to a replacing UTF-8 decode rather than failing: partially
+ readable text is more useful to an agent than an error.
+ """
+ encoding = self.encoding or _charset_from_content_type(content_type) or "utf-8"
+ try:
+ return body.decode(encoding)
+ except (LookupError, UnicodeDecodeError):
+ return body.decode("utf-8", errors="replace")
+
+ @staticmethod
+ def _extract_pdf(body: bytes) -> str:
+ """Extract text from PDF bytes, page by page."""
+ try:
+ import pymupdf # type: ignore[import-untyped]
+ except ImportError as e:
+ raise ImportError(
+ "Reading PDF URLs requires pymupdf. Install with: uv add pymupdf"
+ ) from e
+
+ # Opened from memory: the bytes are already in hand, and a temp file
+ # would need cleaning up on every error path.
+ document = pymupdf.open(stream=body, filetype="pdf")
+ try:
+ pages = [
+ f"Page {number}:\n{text}"
+ for number, page in enumerate(document, 1)
+ if (text := page.get_text().strip())
+ ]
+ finally:
+ document.close()
+
+ if not pages:
+ return "[PDF with no extractable text]"
+ return "\n\n".join(pages)
+
+ @staticmethod
+ def _extract_docx(body: bytes) -> str:
+ """Extract paragraph text from DOCX bytes."""
+ try:
+ from docx import Document
+ except ImportError as e:
+ raise ImportError(
+ "Reading DOCX URLs requires python-docx. Install with: "
+ "uv add python-docx"
+ ) from e
+
+ document = Document(BytesIO(body))
+ return "\n".join(
+ paragraph.text
+ for paragraph in document.paragraphs
+ if paragraph.text.strip()
+ )
+
+ def _extract_html(self, body: bytes, content_type: str) -> str:
+ """Strip HTML bytes down to visible text."""
+ try:
+ from bs4 import BeautifulSoup
+ except ImportError as e:
+ raise ImportError(
+ "Reading HTML URLs requires beautifulsoup4. Install with: "
+ "uv add beautifulsoup4"
+ ) from e
+
+ soup = BeautifulSoup(self._decode(body, content_type), "html.parser")
+ for element in soup(["script", "style"]):
+ element.decompose()
+
+ text = _SPACES_PATTERN.sub(" ", soup.get_text(" "))
+ return _NEWLINE_PATTERN.sub("\n", text).strip()
+
+ def _extract(self, body: bytes, kind: str, content_type: str) -> str:
+ """Dispatch to the extractor named by *kind*."""
+ if kind == "pdf":
+ return self._extract_pdf(body)
+ if kind == "docx":
+ return self._extract_docx(body)
+ if kind == "html":
+ return self._extract_html(body, content_type)
+ return self._decode(body, content_type)
+
+ @staticmethod
+ def _window(text: str, start_line: int, line_count: int | None) -> str:
+ """Return the requested line window of *text*.
+
+ The whole body has already been fetched by this point, so unlike the
+ filesystem equivalent this only trims output -- it saves no transfer.
+
+ The bounds are clamped rather than trusted: the args schema rejects
+ anything below 1 before it gets here, but islice raises on a negative
+ stop index, and this runs outside the caller's error handling.
+ """
+ if start_line == 1 and line_count is None:
+ return text
+
+ start_index = max(start_line - 1, 0)
+ stop_index = None if line_count is None else start_index + max(line_count, 0)
+ selected = list(islice(text.splitlines(keepends=True), start_index, stop_index))
+
+ if not selected and start_index > 0:
+ return (
+ f"Error: Start line {start_line} exceeds the number of lines in "
+ f"the content."
+ )
+ return "".join(selected)
+
+ def _run(
+ self,
+ url: str,
+ start_line: int | None = 1,
+ line_count: int | None = None,
+ ) -> str:
+ """Fetch a URL and return its content, or a window of it, as text."""
+ start_line = start_line or 1
+ line_count = line_count or None
+
+ try:
+ body, content_type, final_url = safe_get_bounded(
+ url,
+ max_bytes=self.max_bytes,
+ timeout=self.timeout,
+ headers=self._request_headers(),
+ )
+ except ValueError as e:
+ return f"Error: {e}"
+ except requests.RequestException as e:
+ return f"Error: Failed to fetch '{url}'. {format_error_for_display(e)}"
+
+ kind = self._resolve_kind(content_type, final_url, url)
+ if kind is None:
+ return (
+ f"Error: Unsupported content type "
+ f"'{content_type.split(';', 1)[0].strip() or 'unknown'}' at "
+ f"'{url}'. This tool reads text, HTML, JSON, XML, CSV, PDF and "
+ f"DOCX responses."
+ )
+
+ try:
+ text = self._extract(body, kind, content_type)
+ except ImportError as e:
+ return f"Error: {e}"
+ except Exception as e:
+ return (
+ f"Error: Failed to read {kind.upper()} content from '{url}'. "
+ f"{format_error_for_display(e)}"
+ )
+
+ return self._window(text, start_line, line_count)
diff --git a/lib/crewai-tools/tests/rag/test_pdf_loader.py b/lib/crewai-tools/tests/rag/test_pdf_loader.py
new file mode 100644
index 000000000..9bae12151
--- /dev/null
+++ b/lib/crewai-tools/tests/rag/test_pdf_loader.py
@@ -0,0 +1,156 @@
+import tempfile
+from unittest.mock import patch
+
+from crewai_tools.rag.base_loader import LoaderResult
+from crewai_tools.rag.loaders.pdf_loader import PDFLoader
+from crewai_tools.rag.source_content import SourceContent
+import pytest
+
+
+pymupdf = pytest.importorskip("pymupdf")
+
+# Patched at the loader's seam rather than at requests.get: safe_get_bounded
+# resolves the hostname before issuing a request, which would make these tests
+# depend on DNS for example.com.
+FETCH = "crewai_tools.rag.loaders.pdf_loader.safe_get_bounded"
+
+
+def build_pdf(text: str = "Quarterly revenue was 42") -> bytes:
+ """Return the bytes of a one-page PDF containing *text*."""
+ document = pymupdf.open()
+ document.new_page().insert_text((72, 72), text)
+ try:
+ return document.tobytes()
+ finally:
+ document.close()
+
+
+def fetch_result(body: bytes, url: str = "https://example.com/report.pdf"):
+ """Build the (body, content_type, final_url) tuple safe_get_bounded returns."""
+ return body, "application/pdf", url
+
+
+class TestPDFLoader:
+ def test_load_pdf_from_file(self):
+ """A PDF on disk has its text extracted with page markers."""
+ with tempfile.NamedTemporaryFile(suffix=".pdf") as f:
+ f.write(build_pdf())
+ f.flush()
+
+ result = PDFLoader().load(SourceContent(f.name))
+
+ assert isinstance(result, LoaderResult)
+ assert "Page 1:" in result.content
+ assert "Quarterly revenue was 42" in result.content
+ assert result.metadata["num_pages"] == 1
+ assert result.metadata["file_type"] == "pdf"
+
+ def test_load_pdf_from_url(self):
+ """A PDF fetched from a URL is extracted and attributed to that URL."""
+ with patch(FETCH) as fetch:
+ fetch.return_value = fetch_result(build_pdf("Content from URL"))
+ result = PDFLoader().load(SourceContent("https://example.com/report.pdf"))
+
+ assert "Content from URL" in result.content
+ assert result.source == "https://example.com/report.pdf"
+ assert result.metadata["file_name"] == "report.pdf"
+
+ headers = fetch.call_args.kwargs["headers"]
+ assert headers["Accept"] == "application/pdf"
+ assert "crewai-tools PDFLoader" in headers["User-Agent"]
+
+ def test_load_pdf_from_url_leaves_no_temp_file(self):
+ """The URL path must not write a temp file it never cleans up.
+
+ It previously used NamedTemporaryFile(delete=False) without unlinking,
+ so every PDF ingested from a URL left a file behind.
+ """
+ with (
+ patch(FETCH) as fetch,
+ patch("tempfile.NamedTemporaryFile") as mock_tempfile,
+ ):
+ fetch.return_value = fetch_result(build_pdf())
+ PDFLoader().load(SourceContent("https://example.com/report.pdf"))
+
+ mock_tempfile.assert_not_called()
+
+ def test_load_pdf_from_url_is_size_bounded(self):
+ """The download is capped, since the body is held in memory."""
+ with patch(FETCH) as fetch:
+ fetch.return_value = fetch_result(build_pdf())
+ PDFLoader().load(SourceContent("https://example.com/report.pdf"))
+
+ assert fetch.call_args.kwargs["max_bytes"] == 50 * 1024 * 1024
+
+ def test_load_pdf_from_url_accepts_a_custom_size_limit(self):
+ """Callers can lower or raise the ceiling per load."""
+ with patch(FETCH) as fetch:
+ fetch.return_value = fetch_result(build_pdf())
+ PDFLoader().load(
+ SourceContent("https://example.com/report.pdf"), max_bytes=1024
+ )
+
+ assert fetch.call_args.kwargs["max_bytes"] == 1024
+
+ def test_load_pdf_from_url_with_custom_headers(self):
+ """Caller-supplied headers replace the loader's defaults."""
+ custom_headers = {"Authorization": "Bearer token"}
+
+ with patch(FETCH) as fetch:
+ fetch.return_value = fetch_result(build_pdf())
+ PDFLoader().load(
+ SourceContent("https://example.com/report.pdf"), headers=custom_headers
+ )
+
+ assert fetch.call_args.kwargs["headers"] == custom_headers
+
+ def test_load_pdf_url_download_error(self):
+ """A failed download surfaces as a ValueError naming the URL."""
+ with patch(FETCH, side_effect=Exception("Network error")):
+ with pytest.raises(ValueError, match="Failed to download PDF"):
+ PDFLoader().load(SourceContent("https://example.com/report.pdf"))
+
+ def test_load_pdf_url_over_size_limit(self):
+ """An oversized body is reported rather than partially parsed."""
+ with patch(FETCH, side_effect=ValueError("exceeds the 1024 byte limit")):
+ with pytest.raises(ValueError, match="Failed to download PDF"):
+ PDFLoader().load(SourceContent("https://example.com/huge.pdf"))
+
+ def test_load_pdf_missing_file(self):
+ """A missing local path raises FileNotFoundError, not ValueError."""
+ with pytest.raises(FileNotFoundError, match="PDF file not found"):
+ PDFLoader().load(SourceContent("/nonexistent/report.pdf"))
+
+ def test_load_corrupt_pdf_raises_value_error(self):
+ """Bytes that are not a parseable PDF produce a read error."""
+ with tempfile.NamedTemporaryFile(suffix=".pdf") as f:
+ f.write(b"%PDF-1.4 not really a pdf")
+ f.flush()
+
+ with pytest.raises(ValueError, match="Error reading PDF"):
+ PDFLoader().load(SourceContent(f.name))
+
+ def test_pdf_with_no_extractable_text(self):
+ """A PDF whose pages hold no text says so instead of returning empty."""
+ document = pymupdf.open()
+ document.new_page()
+ blank = document.tobytes()
+ document.close()
+
+ with tempfile.NamedTemporaryFile(suffix=".pdf") as f:
+ f.write(blank)
+ f.flush()
+
+ result = PDFLoader().load(SourceContent(f.name))
+
+ assert "no extractable text" in result.content
+
+ def test_pdf_doc_id_is_stable(self):
+ """The same source yields the same doc_id across loads."""
+ with tempfile.NamedTemporaryFile(suffix=".pdf") as f:
+ f.write(build_pdf())
+ f.flush()
+
+ loader = PDFLoader()
+ source = SourceContent(f.name)
+ assert loader.load(source).doc_id == loader.load(source).doc_id
diff --git a/lib/crewai-tools/tests/url_read_tool_test.py b/lib/crewai-tools/tests/url_read_tool_test.py
new file mode 100644
index 000000000..9d48beb80
--- /dev/null
+++ b/lib/crewai-tools/tests/url_read_tool_test.py
@@ -0,0 +1,415 @@
+from unittest.mock import patch
+
+import pytest
+import requests
+
+from crewai_tools import URLReadTool
+from crewai_tools.security.safe_requests import safe_get_bounded
+
+
+TOOL_MODULE = "crewai_tools.tools.url_read_tool.url_read_tool"
+
+
+class FakeResponse:
+ """Minimal stand-in for a streamed requests.Response."""
+
+ def __init__(
+ self,
+ body: bytes = b"",
+ content_type: str = "text/plain",
+ url: str = "https://example.com/file.txt",
+ status_code: int = 200,
+ chunk_size: int | None = None,
+ ):
+ self._body = body
+ self._chunk_size = chunk_size
+ self.headers = {"Content-Type": content_type} if content_type else {}
+ self.url = url
+ self.status_code = status_code
+ self.history: list["FakeResponse"] = []
+ self.closed = False
+
+ def raise_for_status(self) -> None:
+ """Mimic requests' error-status behavior."""
+ if self.status_code >= 400:
+ raise requests.HTTPError(f"{self.status_code} error")
+
+ def iter_content(self, chunk_size: int = 65536):
+ """Yield the body in chunks, like a streamed response."""
+ size = self._chunk_size or chunk_size
+ for index in range(0, len(self._body), size):
+ yield self._body[index : index + size]
+
+ def close(self) -> None:
+ """Record that the response was closed."""
+ self.closed = True
+
+
+def build_pdf(text: str = "Quarterly revenue was 42") -> bytes:
+ """Return the bytes of a one-page PDF containing *text*."""
+ pymupdf = pytest.importorskip("pymupdf")
+ document = pymupdf.open()
+ document.new_page().insert_text((72, 72), text)
+ try:
+ return document.tobytes()
+ finally:
+ document.close()
+
+
+def fetch_result(
+ body: bytes,
+ content_type: str = "text/plain",
+ url: str = "https://example.com/f.txt",
+):
+ """Build the (body, content_type, final_url) tuple safe_get_bounded returns."""
+ return body, content_type, url
+
+
+def test_reads_plain_text():
+ """A text response is returned as-is, with the configured limits applied."""
+ tool = URLReadTool()
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ fetch.return_value = fetch_result(b"hello world")
+ assert tool.run(url="https://example.com/f.txt") == "hello world"
+
+ assert fetch.call_args.kwargs["max_bytes"] == 5 * 1024 * 1024
+ assert fetch.call_args.kwargs["timeout"] == 30
+
+
+def test_honors_declared_charset():
+ """The charset in the Content-Type header drives decoding."""
+ tool = URLReadTool()
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ fetch.return_value = fetch_result(
+ "café".encode("latin-1"), "text/plain; charset=iso-8859-1"
+ )
+ assert tool.run(url="https://example.com/f.txt") == "café"
+
+
+def test_encoding_override_wins_over_server_charset():
+ """An explicit encoding beats whatever the server declares."""
+ tool = URLReadTool(encoding="latin-1")
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ fetch.return_value = fetch_result(
+ "café".encode("latin-1"), "text/plain; charset=utf-8"
+ )
+ assert tool.run(url="https://example.com/f.txt") == "café"
+
+
+def test_undecodable_bytes_fall_back_instead_of_failing():
+ """Partially readable text beats an error for the agent."""
+ tool = URLReadTool()
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ fetch.return_value = fetch_result(b"\xff\xfe bad bytes", "text/plain")
+ result = tool.run(url="https://example.com/f.txt")
+
+ assert "bad bytes" in result
+ assert not result.startswith("Error:")
+
+
+def test_line_window():
+ """start_line and line_count select a window of the extracted text."""
+ tool = URLReadTool()
+ body = b"one\ntwo\nthree\nfour\nfive\n"
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ fetch.return_value = fetch_result(body)
+ result = tool.run(url="https://example.com/f.txt", start_line=2, line_count=2)
+
+ assert result == "two\nthree\n"
+
+
+def test_start_line_past_end_reports_error():
+ """Asking past the end of the content is reported, not silently empty."""
+ tool = URLReadTool()
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ fetch.return_value = fetch_result(b"one\ntwo\n")
+ result = tool.run(url="https://example.com/f.txt", start_line=99)
+
+ assert "exceeds the number of lines" in result
+
+
+ "line_args",
+ [{"line_count": -5}, {"line_count": 0}, {"start_line": 0}, {"start_line": -5}],
+)
+def test_line_arguments_below_one_are_refused(line_args):
+ """Out-of-range line arguments are rejected before any request is made.
+
+ islice raises on a negative stop index, and the windowing runs outside the
+ tool's error handling, so these have to be refused at validation time.
+ """
+ tool = URLReadTool()
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ with pytest.raises(ValueError, match="greater than or equal to 1"):
+ tool.run(url="https://example.com/f.txt", **line_args)
+
+ fetch.assert_not_called()
+
+
+def test_json_is_returned_verbatim():
+ """JSON is passed through undecorated so callers can parse it."""
+ tool = URLReadTool()
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ fetch.return_value = fetch_result(b'{"a": 1}', "application/json")
+ assert tool.run(url="https://example.com/data.json") == '{"a": 1}'
+
+
+def test_structured_suffix_type_is_treated_as_text():
+ """A +json vendor type is text, not an unsupported binary type."""
+ tool = URLReadTool()
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ fetch.return_value = fetch_result(b'{"a": 1}', "application/vnd.api+json")
+ assert tool.run(url="https://example.com/data") == '{"a": 1}'
+
+
+def test_html_is_stripped_to_visible_text():
+ """HTML returns visible text with script and style content removed."""
+ tool = URLReadTool()
+ body = b"<html><head><style>p{color:red}</style></head><body><p>Hi</p><script>x=1</script></body></html>"
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ fetch.return_value = fetch_result(body, "text/html; charset=utf-8")
+ result = tool.run(url="https://example.com/page")
+
+ assert "Hi" in result
+ assert "x=1" not in result
+ assert "color:red" not in result
+
+
+def test_binary_content_type_is_rejected():
+ """An unsupported type is refused rather than returned as base64."""
+ tool = URLReadTool()
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ fetch.return_value = fetch_result(b"\x89PNG\r\n", "image/png")
+ result = tool.run(url="https://example.com/logo.png")
+
+ assert "Unsupported content type 'image/png'" in result
+
+
+def test_octet_stream_pdf_falls_back_to_url_extension():
+ """A PDF served as octet-stream is still extracted, via its extension."""
+ tool = URLReadTool()
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ fetch.return_value = fetch_result(
+ build_pdf("Fallback worked"),
+ "application/octet-stream",
+ "https://example.com/a/b.pdf",
+ )
+ result = tool.run(url="https://example.com/a/b.pdf")
+
+ assert "Fallback worked" in result
+
+
+def test_missing_content_type_falls_back_to_url_extension():
+ """No Content-Type at all still reads as text when the path says .csv."""
+ tool = URLReadTool()
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ fetch.return_value = fetch_result(b"a,b\n1,2\n", "", "https://example.com/b.csv")
+ assert tool.run(url="https://example.com/b.csv") == "a,b\n1,2\n"
+
+
+def test_query_string_does_not_break_extension_fallback():
+ """A presigned-style query string does not hide the path's extension."""
+ tool = URLReadTool()
+ url = "https://example.com/b.csv?X-Amz-Signature=abc"
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ fetch.return_value = fetch_result(b"a,b\n", "application/octet-stream", url)
+ assert tool.run(url=url) == "a,b\n"
+
+
+def test_extension_from_requested_url_survives_a_redirect():
+ """A .pdf link that redirects to an extensionless path is still extracted.
+
+ Presigned CDN targets routinely drop the extension and serve octet-stream,
+ so the requested URL is the only place the type survives.
+ """
+ tool = URLReadTool()
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ fetch.return_value = fetch_result(
+ build_pdf("Survived the redirect"),
+ "application/octet-stream",
+ "https://cdn.example.com/objects/9f8a7b6c5d",
+ )
+ result = tool.run(url="https://example.com/report.pdf")
+
+ assert "Survived the redirect" in result
+
+
+def test_octet_stream_with_unknown_extension_is_rejected():
+ """With neither a usable type nor a known extension, the read is refused."""
+ tool = URLReadTool()
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ fetch.return_value = fetch_result(
+ b"\x00\x01", "application/octet-stream", "https://example.com/a/b.bin"
+ )
+ result = tool.run(url="https://example.com/a/b.bin")
+
+ assert "Unsupported content type" in result
+
+
+def test_validation_failure_is_returned_as_error():
+ """An SSRF rejection reaches the agent as an error string, not an exception."""
+ tool = URLReadTool()
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ fetch.side_effect = ValueError(
+ "URL 'http://169.254.169.254/' resolves to private/reserved IP 169.254.169.254."
+ )
+ result = tool.run(url="http://169.254.169.254/")
+
+ assert result.startswith("Error:")
+ assert "private/reserved IP" in result
+
+
+def test_request_failure_is_returned_as_error():
+ """A transport failure is reported without raising out of the tool."""
+ tool = URLReadTool()
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ fetch.side_effect = requests.ConnectionError("connection refused")
+ result = tool.run(url="https://example.com/f.txt")
+
+ assert result.startswith("Error: Failed to fetch")
+
+
+def test_custom_headers_are_merged_over_defaults():
+ """Caller headers win, but the default User-Agent survives."""
+ tool = URLReadTool(headers={"Authorization": "Bearer x", "Accept": "text/plain"})
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ fetch.return_value = fetch_result(b"ok")
+ tool.run(url="https://example.com/f.txt")
+
+ headers = fetch.call_args.kwargs["headers"]
+ assert headers["Authorization"] == "Bearer x"
+ assert headers["Accept"] == "text/plain"
+ assert "crewai-tools URLReadTool" in headers["User-Agent"]
+
+
+def test_reads_a_real_pdf_end_to_end():
+ """Real PDF bytes are extracted page by page."""
+ pdf_bytes = build_pdf()
+
+ tool = URLReadTool()
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ fetch.return_value = fetch_result(
+ pdf_bytes, "application/pdf", "https://example.com/report.pdf"
+ )
+ result = tool.run(url="https://example.com/report.pdf")
+
+ assert "Page 1:" in result
+ assert "Quarterly revenue was 42" in result
+
+
+def test_corrupt_pdf_reports_error_without_raising():
+ """A malformed PDF becomes an error string, not a traceback."""
+ pytest.importorskip("pymupdf")
+
+ tool = URLReadTool()
+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:
+ fetch.return_value = fetch_result(
+ b"%PDF-1.4 not really a pdf", "application/pdf"
+ )
+ result = tool.run(url="https://example.com/report.pdf")
+
+ assert result.startswith("Error: Failed to read PDF content")
+
+
+class TestSafeGetBounded:
+ """Tests for the bounded-fetch helper itself."""
+
+ def test_returns_body_content_type_and_final_url(self):
+ """The helper reports the body alongside where it ended up."""
+ response = FakeResponse(b"payload", "text/plain", "https://example.com/final")
+ with patch(
+ "crewai_tools.security.safe_requests.safe_get", return_value=response
+ ):
+ body, content_type, final_url = safe_get_bounded(
+ "https://example.com/start", max_bytes=1024
+ )
+
+ assert body == b"payload"
+ assert content_type == "text/plain"
+ assert final_url == "https://example.com/final"
+ assert response.closed
+
+ def test_rejects_body_over_the_limit(self):
+ """Crossing max_bytes raises rather than truncating silently."""
+ response = FakeResponse(b"x" * 100, chunk_size=10)
+ with patch(
+ "crewai_tools.security.safe_requests.safe_get", return_value=response
+ ):
+ with pytest.raises(ValueError, match="exceeds the 25 byte limit"):
+ safe_get_bounded("https://example.com/big", max_bytes=25)
+
+ assert response.closed
+
+ def test_oversized_error_names_the_url_that_served_the_body(self):
+ """After a redirect the requested URL is not the one that sent it."""
+ response = FakeResponse(
+ b"x" * 100, url="https://cdn.example.com/final", chunk_size=10
+ )
+ with patch(
+ "crewai_tools.security.safe_requests.safe_get", return_value=response
+ ):
+ with pytest.raises(ValueError, match="https://cdn.example.com/final"):
+ safe_get_bounded("https://example.com/start", max_bytes=25)
+
+ @pytest.mark.parametrize("max_bytes", [0, -1])
+ def test_non_positive_max_bytes_fails_before_requesting(self, max_bytes):
+ """A misconfigured cap is caught without issuing a request."""
+ with patch("crewai_tools.security.safe_requests.safe_get") as safe_get:
+ with pytest.raises(ValueError, match="max_bytes must be positive"):
+ safe_get_bounded("https://example.com/f", max_bytes=max_bytes)
+
+ safe_get.assert_not_called()
+
+ def test_stops_reading_once_the_limit_is_crossed(self):
+ """The cap must abandon the stream, not buffer the whole body first."""
+ chunks_yielded = 0
+
+ class CountingResponse(FakeResponse):
+ def iter_content(self, chunk_size: int = 65536):
+ nonlocal chunks_yielded
+ for _ in range(1000):
+ chunks_yielded += 1
+ yield b"x" * 10
+
+ response = CountingResponse()
+ with patch(
+ "crewai_tools.security.safe_requests.safe_get", return_value=response
+ ):
+ with pytest.raises(ValueError):
+ safe_get_bounded("https://example.com/huge", max_bytes=25)
+
+ assert chunks_yielded == 3
+
+ def test_error_status_raises(self):
+ """An error status propagates as an HTTPError."""
+ response = FakeResponse(b"nope", status_code=404)
+ with patch(
+ "crewai_tools.security.safe_requests.safe_get", return_value=response
+ ):
+ with pytest.raises(requests.HTTPError):
+ safe_get_bounded("https://example.com/missing", max_bytes=1024)
+
+ assert response.closed
+
+ def test_closes_redirect_hops(self):
+ """Streamed redirect hops hold connections until closed."""
+ hop = FakeResponse(b"", status_code=302)
+ response = FakeResponse(b"done")
+ response.history = [hop]
+ with patch(
+ "crewai_tools.security.safe_requests.safe_get", return_value=response
+ ):
+ safe_get_bounded("https://example.com/start", max_bytes=1024)
+
+ assert hop.closed
+ assert response.closed
+
+ def test_requests_are_streamed(self):
+ """Streaming is what lets an oversized body be abandoned early."""
+ response = FakeResponse(b"ok")
+ with patch(
+ "crewai_tools.security.safe_requests.safe_get", return_value=response
+ ) as safe_get:
+ safe_get_bounded("https://example.com/f", max_bytes=1024)
+
+ assert safe_get.call_args.kwargs["stream"] is True
diff --git a/lib/crewai-tools/tests/utilities/test_safe_requests.py b/lib/crewai-tools/tests/utilities/test_safe_requests.py
index f45dd86c6..895c0044c 100644
--- a/lib/crewai-tools/tests/utilities/test_safe_requests.py
+++ b/lib/crewai-tools/tests/utilities/test_safe_requests.py
@@ -112,6 +112,80 @@ def test_safe_get_fails_closed_after_too_many_redirects(
safe_get("http://public.example/start", max_redirects=1, timeout=15)
+def _closable_response(
+ url: str, status_code: int, *, location: str | None = None, closed: list[str]
+) -> requests.Response:
+ """Build a response that records its own URL when closed."""
+ response = _response(url, status_code, location=location)
+ response.close = lambda: closed.append(url) # type: ignore[method-assign]
+ return response
+
+
+def test_safe_get_closes_earlier_hops_after_too_many_redirects(
+ monkeypatch: pytest.MonkeyPatch, public_dns: None
+) -> None:
+ """Hops accumulated before the failure must not be left open.
+
+ Under stream=True each hop holds its connection until its body is read or
+ closed, and a caller handed an exception has no handle on them.
+ """
+ closed: list[str] = []
+
+ def fake_get(url: str, **kwargs: Any) -> requests.Response:
+ return _closable_response(
+ url, 302, location="http://safe.example/again", closed=closed
+ )
+
+ _mock_get(monkeypatch, fake_get)
+
+ with pytest.raises(ValueError, match="Too many redirects"):
+ safe_get("http://public.example/start", max_redirects=2, timeout=15, stream=True)
+
+ assert len(closed) == 3
+
+
+def test_safe_get_closes_earlier_hops_when_a_redirect_is_rejected(
+ monkeypatch: pytest.MonkeyPatch, public_dns: None
+) -> None:
+ """A hop rejected mid-chain still releases the connections already open."""
+ closed: list[str] = []
+
+ def fake_get(url: str, **kwargs: Any) -> requests.Response:
+ if url == "http://public.example/start":
+ return _closable_response(
+ url, 302, location="http://safe.example/next", closed=closed
+ )
+ return _closable_response(
+ url, 302, location="http://169.254.169.254/latest", closed=closed
+ )
+
+ _mock_get(monkeypatch, fake_get)
+
+ with pytest.raises(ValueError, match="private/reserved IP"):
+ safe_get("http://public.example/start", timeout=15, stream=True)
+
+ assert closed == ["http://safe.example/next", "http://public.example/start"]
+
+
+def test_safe_get_leaves_hops_open_on_success(
+ monkeypatch: pytest.MonkeyPatch, public_dns: None
+) -> None:
+ """On success the hops belong to the caller, via response.history."""
+ closed: list[str] = []
+
+ def fake_get(url: str, **kwargs: Any) -> requests.Response:
+ if url == "http://public.example/start":
+ return _closable_response(url, 302, location="/final", closed=closed)
+ return _closable_response(url, 200, closed=closed)
+
+ _mock_get(monkeypatch, fake_get)
+
+ response = safe_get("http://public.example/start", timeout=15, stream=True)
+
+ assert closed == []
+ assert len(response.history) == 1
+
+
def test_safe_get_strips_credentials_on_cross_origin_redirect(
monkeypatch: pytest.MonkeyPatch, public_dns: None
) -> None:
diff --git a/lib/crewai-tools/tool.specs.json b/lib/crewai-tools/tool.specs.json
index a03513308..6380dc7c6 100644
--- a/lib/crewai-tools/tool.specs.json
+++ b/lib/crewai-tools/tool.specs.json
@@ -26885,6 +26885,148 @@
"type": "object"
}
},
+ {
+ "description": "A tool that reads the content at a URL and returns it as text. To use this tool, provide a 'url' parameter with an http:// or https:// address. PDF, DOCX, HTML, JSON, XML, CSV and plain-text responses are converted to text; other binary types are rejected. URLs that resolve to private or internal network addresses are refused, as are responses over the tool's size limit. Optionally provide 'start_line' and 'line_count' to read only part of the content.",
+ "env_vars": [],
+ "humanized_name": "Read content from a URL",
+ "init_params_schema": {
+ "$defs": {
+ "EnvVar": {
+ "properties": {
+ "default": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "default": null,
+ "title": "Default"
+ },
+ "description": {
+ "title": "Description",
+ "type": "string"
+ },
+ "name": {
+ "title": "Name",
+ "type": "string"
+ },
+ "required": {
+ "default": true,
+ "title": "Required",
+ "type": "boolean"
+ }
+ },
+ "required": [
+ "name",
+ "description"
+ ],
+ "title": "EnvVar",
+ "type": "object"
+ },
+ "ToolFailurePolicy": {
+ "description": "How an agent reacts when one of its tools reports a failure.",
+ "enum": [
+ "ignore",
+ "warn",
+ "raise"
+ ],
+ "title": "ToolFailurePolicy",
+ "type": "string"
+ }
+ },
+ "description": "Read the content at an arbitrary URL and return it as text.\n\nUnlike :class:`~crewai_tools.tools.file_read_tool.file_read_tool.FileReadTool`,\nwhich is confined to the local filesystem, this tool performs network\nrequests to addresses the caller -- often an LLM -- chooses at runtime. It\nis a separate tool for exactly that reason: granting it is granting network\negress, and that should be a deliberate choice rather than a flag on a\nfilesystem tool.\n\nResponses are decoded to text according to their content type. PDF and DOCX\nbodies have their text extracted, HTML is stripped to visible text, and\ntext-shaped types (plain text, Markdown, JSON, XML, YAML, CSV) are decoded\nas-is. Any other type is refused rather than returned as base64, keeping\nthis tool's output text-only.\n\nSecurity:\n Requests go through :func:`~crewai_tools.security.safe_requests.safe_get_bounded`,\n which resolves each hostname and rejects it when any resolved address is\n private, loopback, link-local, or otherwise reserved -- covering cloud\n metadata endpoints and internal services. Redirects are never followed\n automatically: every hop is revalidated, and credentials are dropped on\n cross-origin hops. Bodies over ``max_bytes`` are abandoned mid-stream.\n\n Two risks are not closed here. Validation resolves the hostname and\n requests resolves it again when connecting, so a DNS entry that changes\n between those lookups can still redirect the connection (DNS\n rebinding); closing that requires pinning the connection to the\n validated address. And the returned text is untrusted remote content\n flowing into an agent's context -- a fetched page can attempt to\n instruct the agent. Neither is addressable by input validation alone;\n network egress policy and prompt-level handling cover them.\n\nArgs:\n max_bytes (int): Largest response body to accept, in decoded bytes.\n Defaults to 5 MiB.\n timeout (float): Per-request timeout in seconds. Defaults to 30.\n headers (Optional[dict[str, str]]): Extra request headers. Developer\n supplied, not chosen by the model.\n encoding (Optional[str]): Force a text encoding instead of honoring the\n charset the server declares.\n **kwargs: Additional keyword arguments passed to BaseTool.\n\nExample:\n >>> tool = URLReadTool()\n >>> content = tool.run(url=\"https://example.com/report.pdf\")\n >>> head = tool.run(url=\"https://example.com/data.csv\", line_count=20)",
+ "properties": {
+ "encoding": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "default": null,
+ "title": "Encoding"
+ },
+ "headers": {
+ "anyOf": [
+ {
+ "additionalProperties": {
+ "type": "string"
+ },
+ "type": "object"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "default": null,
+ "title": "Headers"
+ },
+ "max_bytes": {
+ "default": 5242880,
+ "title": "Max Bytes",
+ "type": "integer"
+ },
+ "timeout": {
+ "default": 30,
+ "title": "Timeout",
+ "type": "number"
+ }
+ },
+ "required": [],
+ "title": "URLReadTool",
+ "type": "object"
+ },
+ "name": "URLReadTool",
+ "package_dependencies": [],
+ "run_params_schema": {
+ "description": "Input for URLReadTool.",
+ "properties": {
+ "line_count": {
+ "anyOf": [
+ {
+ "minimum": 1,
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "default": null,
+ "description": "Number of lines to read. If None, reads the entire content",
+ "title": "Line Count"
+ },
+ "start_line": {
+ "anyOf": [
+ {
+ "minimum": 1,
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "default": 1,
+ "description": "Line number to start reading from (1-indexed)",
+ "title": "Start Line"
+ },
+ "url": {
+ "description": "The http:// or https:// URL to read. Addresses that resolve to private or internal networks are refused.",
+ "title": "Url",
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "URLReadToolSchema",
+ "type": "object"
+ }
+ },
{
"description": "This tool uses OpenAI's Vision API to describe the contents of an image.",
"env_vars": [