Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
feat(tools): add URLReadTool for reading arbitrary URLs (#6834)
João Moura0c74f23596b5
diff --git a/lib/crewai-tools/src/crewai_tools/__init__.py b/lib/crewai-tools/src/crewai_tools/__init__.pyindex 1ffdf96b4..a50444888 100644--- a/lib/crewai-tools/src/crewai_tools/__init__.py+++ b/lib/crewai-tools/src/crewai_tools/__init__.py@@ -209,6 +209,7 @@ from crewai_tools.tools.tavily_research_tool.tavily_research_tool import ()from crewai_tools.tools.tavily_search_tool.tavily_search_tool import TavilySearchToolfrom crewai_tools.tools.txt_search_tool.txt_search_tool import TXTSearchTool+from crewai_tools.tools.url_read_tool.url_read_tool import URLReadToolfrom crewai_tools.tools.vision_tool.vision_tool import VisionToolfrom crewai_tools.tools.wait_tool.wait_tool import WaitToolfrom crewai_tools.tools.weaviate_tool.vector_search import WeaviateVectorSearchTool@@ -327,6 +328,7 @@ __all__ = ["TavilyGetResearchTool","TavilyResearchTool","TavilySearchTool",+ "URLReadTool","VisionTool","WaitTool","WeaviateVectorSearchTool",diff --git a/lib/crewai-tools/src/crewai_tools/rag/loaders/pdf_loader.py b/lib/crewai-tools/src/crewai_tools/rag/loaders/pdf_loader.pyindex 259181481..84fd45d96 100644--- a/lib/crewai-tools/src/crewai_tools/rag/loaders/pdf_loader.py+++ b/lib/crewai-tools/src/crewai_tools/rag/loaders/pdf_loader.py@@ -2,13 +2,17 @@import osfrom pathlib import Path-import tempfilefrom typing import Anyfrom urllib.parse import urlparsefrom crewai_tools.rag.base_loader import BaseLoader, LoaderResultfrom crewai_tools.rag.source_content import SourceContent-from crewai_tools.security.safe_requests import safe_get+from crewai_tools.security.safe_requests import safe_get_bounded+++# Remote PDFs are held in memory for the whole extraction, so the download needs+# a ceiling. Override per call with the ``max_bytes`` kwarg to ``load``.+DEFAULT_MAX_PDF_BYTES = 50 * 1024 * 1024class PDFLoader(BaseLoader):@@ -24,18 +28,24 @@ class PDFLoader(BaseLoader):return False@staticmethod- def _download_from_url(url: str, kwargs: dict[str, Any]) -> str:- """Download PDF from a URL to a temporary file and return its path.+ def _fetch_from_url(url: str, kwargs: dict[str, Any]) -> bytes:+ """Download a PDF from a URL and return its bytes.++ The content stays in memory rather than going to a temporary file: the+ whole body has to be buffered either way, and a temp file would need+ unlinking on every error path to avoid leaving files behind. Because it+ is held in memory, the download is capped.Args:url: The URL to download from.- kwargs: Optional dict that may contain custom headers.+ kwargs: Optional dict that may contain custom ``headers`` and a+ ``max_bytes`` ceiling for the download.Returns:- Path to the temporary file containing the PDF.+ The raw PDF content.Raises:- ValueError: If the download fails.+ ValueError: If the download fails or exceeds the size ceiling."""headers = kwargs.get("headers",@@ -46,12 +56,13 @@ class PDFLoader(BaseLoader):)try:- response = safe_get(url, headers=headers, timeout=30)- response.raise_for_status()-- with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as temp_file:- temp_file.write(response.content)- return temp_file.name+ body, _content_type, _final_url = safe_get_bounded(+ url,+ max_bytes=kwargs.get("max_bytes", DEFAULT_MAX_PDF_BYTES),+ headers=headers,+ timeout=30,+ )+ return bodyexcept Exception as e:raise ValueError(f"Failed to download PDF from {url}: {e!s}") from e@@ -93,21 +104,25 @@ class PDFLoader(BaseLoader):try:if is_url:- local_path = self._download_from_url(file_path, kwargs)- doc = pymupdf.open(local_path)+ doc = pymupdf.open(+ stream=self._fetch_from_url(file_path, kwargs), filetype="pdf"+ )else:if not os.path.isfile(file_path):raise FileNotFoundError(f"PDF file not found: {file_path}")doc = pymupdf.open(file_path)- metadata["num_pages"] = len(doc)-- for page_num, page in enumerate(doc, 1):- page_text = page.get_text()- if page_text.strip():- text_content.append(f"Page {page_num}:\n{page_text}")-- doc.close()+ # Closed in a finally so a failure mid-extraction still releases the+ # document handle.+ try:+ metadata["num_pages"] = len(doc)++ for page_num, page in enumerate(doc, 1):+ page_text = page.get_text()+ if page_text.strip():+ text_content.append(f"Page {page_num}:\n{page_text}")+ finally:+ doc.close()except FileNotFoundError:raiseexcept Exception as e:diff --git a/lib/crewai-tools/src/crewai_tools/security/safe_requests.py b/lib/crewai-tools/src/crewai_tools/security/safe_requests.pyindex 505a5cdb6..12765b022 100644--- a/lib/crewai-tools/src/crewai_tools/security/safe_requests.py+++ b/lib/crewai-tools/src/crewai_tools/security/safe_requests.py@@ -11,6 +11,7 @@ from crewai_tools.security.safe_path import validate_url_REDIRECT_STATUS_CODES = {301, 302, 303, 307, 308}+_STREAM_CHUNK_SIZE = 65536_SENSITIVE_HEADER_NAMES = {"authorization","cookie",@@ -49,40 +50,124 @@ def _strip_cross_origin_credentials(request_kwargs: dict[str, Any]) -> dict[str,def safe_get(url: str, *, max_redirects: int = 10, **kwargs: Any) -> requests.Response:- """GET a URL while validating each redirect target before following it."""+ """GET a URL while validating each redirect target before following it.++ On success the hops are attached to the returned response's ``history`` and+ are the caller's to close. On failure they are closed here: a caller given+ an exception has no handle on them, and a streamed hop holds its connection+ until its body is read or closed.+ """current_url = validate_url(url)request_kwargs = {**kwargs, "allow_redirects": False}timeout = request_kwargs.pop("timeout", 30)history: list[requests.Response] = []redirects_followed = 0- while True:- response = requests.get(current_url, timeout=timeout, **request_kwargs)- if (- response.status_code not in _REDIRECT_STATUS_CODES- or "Location" not in response.headers- ):- response.history = history- return response-- if redirects_followed >= max_redirects:- response.close()- raise ValueError(f"Too many redirects while fetching URL: {url}")-- location = response.headers.get("Location")- if not location:- response.history = history- return response-- try:- redirect_url = validate_url(urljoin(response.url, location))- except ValueError:- response.close()- raise-- if not _same_origin(current_url, redirect_url):- request_kwargs = _strip_cross_origin_credentials(request_kwargs)-- history.append(response)- current_url = redirect_url- redirects_followed += 1+ try:+ while True:+ response = requests.get(current_url, timeout=timeout, **request_kwargs)+ if (+ response.status_code not in _REDIRECT_STATUS_CODES+ or "Location" not in response.headers+ ):+ response.history = history+ return response++ if redirects_followed >= max_redirects:+ response.close()+ raise ValueError(f"Too many redirects while fetching URL: {url}")++ location = response.headers.get("Location")+ if not location:+ response.history = history+ return response++ try:+ redirect_url = validate_url(urljoin(response.url, location))+ except ValueError:+ response.close()+ raise++ if not _same_origin(current_url, redirect_url):+ request_kwargs = _strip_cross_origin_credentials(request_kwargs)++ history.append(response)+ current_url = redirect_url+ redirects_followed += 1+ except BaseException:+ for hop in history:+ hop.close()+ raise+++def safe_get_bounded(+ url: str,+ *,+ max_bytes: int,+ timeout: float | tuple[float, float] = 30,+ headers: dict[str, str] | None = None,+ max_redirects: int = 10,+) -> tuple[bytes, str, str]:+ """GET a URL through :func:`safe_get`, refusing bodies over *max_bytes*.++ The body is streamed and abandoned as soon as it crosses the limit, so an+ oversized response costs one chunk of memory instead of all of it. The cap+ counts decoded bytes, which is what a compressed response expands into --+ ``Content-Length`` describes the wire size and cannot bound that.++ Args:+ url: The URL to fetch.+ max_bytes: Largest body to accept, in decoded bytes.+ timeout: Request timeout, passed through to requests.+ headers: Request headers.+ max_redirects: Hops to follow before giving up.++ Returns:+ A ``(body, content_type, final_url)`` tuple, where *final_url* is the+ last validated URL in the redirect chain.++ Raises:+ ValueError: If *max_bytes* is not positive, URL validation fails, the+ redirect chain is too long, or the body exceeds *max_bytes*.+ requests.RequestException: If the request fails or returns an error+ status.+ """+ if max_bytes <= 0:+ raise ValueError(f"max_bytes must be positive, got {max_bytes}.")++ response = safe_get(+ url,+ max_redirects=max_redirects,+ headers=headers,+ timeout=timeout,+ stream=True,+ )+ try:+ response.raise_for_status()++ chunks: list[bytes] = []+ total = 0+ for chunk in response.iter_content(chunk_size=_STREAM_CHUNK_SIZE):+ if not chunk:+ continue+ total += len(chunk)+ if total > max_bytes:+ # Names the URL that served the body, which after a redirect is+ # not the one that was requested.+ raise ValueError(+ f"Response body from '{response.url}' exceeds the "+ f"{max_bytes} byte limit."+ )+ chunks.append(chunk)++ return (+ b"".join(chunks),+ response.headers.get("Content-Type", ""),+ response.url,+ )+ finally:+ # Under stream=True each hop holds its connection until the body is read,+ # so the redirects need closing too, not just the response we return.+ for hop in response.history:+ hop.close()+ response.close()diff --git a/lib/crewai-tools/src/crewai_tools/tools/__init__.py b/lib/crewai-tools/src/crewai_tools/tools/__init__.pyindex 4651bbdd3..2653490f7 100644--- a/lib/crewai-tools/src/crewai_tools/tools/__init__.py+++ b/lib/crewai-tools/src/crewai_tools/tools/__init__.py@@ -196,6 +196,7 @@ from crewai_tools.tools.tavily_research_tool.tavily_research_tool import ()from crewai_tools.tools.tavily_search_tool.tavily_search_tool import TavilySearchToolfrom crewai_tools.tools.txt_search_tool.txt_search_tool import TXTSearchTool+from crewai_tools.tools.url_read_tool.url_read_tool import URLReadToolfrom crewai_tools.tools.vision_tool.vision_tool import VisionToolfrom crewai_tools.tools.wait_tool.wait_tool import WaitToolfrom crewai_tools.tools.weaviate_tool.vector_search import WeaviateVectorSearchTool@@ -310,6 +311,7 @@ __all__ = ["TavilyGetResearchTool","TavilyResearchTool","TavilySearchTool",+ "URLReadTool","VisionTool","WaitTool","WeaviateVectorSearchTool",diff --git a/lib/crewai-tools/src/crewai_tools/tools/url_read_tool/__init__.py b/lib/crewai-tools/src/crewai_tools/tools/url_read_tool/__init__.pynew file mode 100644index 000000000..37efdfc9b--- /dev/null+++ b/lib/crewai-tools/src/crewai_tools/tools/url_read_tool/__init__.py@@ -0,0 +1,4 @@+from crewai_tools.tools.url_read_tool.url_read_tool import URLReadTool+++__all__ = ["URLReadTool"]diff --git a/lib/crewai-tools/src/crewai_tools/tools/url_read_tool/url_read_tool.py b/lib/crewai-tools/src/crewai_tools/tools/url_read_tool/url_read_tool.pynew file mode 100644index 000000000..5a2c83288--- /dev/null+++ b/lib/crewai-tools/src/crewai_tools/tools/url_read_tool/url_read_tool.py@@ -0,0 +1,379 @@+"""Tool for reading the content at an arbitrary URL as text."""++from __future__ import annotations++from io import BytesIO+from itertools import islice+import re+from typing import Any, Final+from urllib.parse import urlparse++from crewai.tools import BaseTool+from pydantic import BaseModel, Field+import requests++from crewai_tools.security.safe_path import format_error_for_display+from crewai_tools.security.safe_requests import safe_get_bounded+++_DEFAULT_MAX_BYTES: Final[int] = 5 * 1024 * 1024+_DEFAULT_TIMEOUT: Final[int] = 30++_PDF_TYPE: Final[str] = "application/pdf"+_DOCX_TYPE: Final[str] = (+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document"+)+_HTML_TYPES: Final[frozenset[str]] = frozenset({"text/html", "application/xhtml+xml"})+_TEXT_TYPES: Final[frozenset[str]] = frozenset(+ {+ "application/csv",+ "application/javascript",+ "application/json",+ "application/sql",+ "application/x-ndjson",+ "application/x-yaml",+ "application/xml",+ "application/yaml",+ }+)+_TEXT_TYPE_SUFFIXES: Final[tuple[str, ...]] = ("+json", "+xml", "+yaml")++# Servers commonly serve static files as octet-stream, or send no type at all,+# so the extension is consulted when the header carries no usable answer.+_UNINFORMATIVE_TYPES: Final[frozenset[str]] = frozenset(+ {"", "application/octet-stream", "binary/octet-stream"}+)+_EXTENSION_TYPES: Final[dict[str, str]] = {+ ".csv": "text/csv",+ ".docx": _DOCX_TYPE,+ ".htm": "text/html",+ ".html": "text/html",+ ".json": "application/json",+ ".md": "text/markdown",+ ".pdf": _PDF_TYPE,+ ".txt": "text/plain",+ ".xml": "application/xml",+ ".yaml": "application/yaml",+ ".yml": "application/yaml",+}++_SPACES_PATTERN: Final[re.Pattern[str]] = re.compile(r"[ \t]+")+_NEWLINE_PATTERN: Final[re.Pattern[str]] = re.compile(r"\s+\n\s+")+++def _charset_from_content_type(content_type: str) -> str | None:+ """Return the charset parameter of a Content-Type header, if it has one."""+ for parameter in content_type.split(";")[1:]:+ name, _, value = parameter.partition("=")+ if name.strip().lower() == "charset":+ return value.strip().strip('"') or None+ return None+++class URLReadToolSchema(BaseModel):+ """Input for URLReadTool."""++ url: str = Field(+ ...,+ description=(+ "The http:// or https:// URL to read. Addresses that resolve to "+ "private or internal networks are refused."+ ),+ )+ start_line: int | None = Field(+ 1, ge=1, description="Line number to start reading from (1-indexed)"+ )+ line_count: int | None = Field(+ None,+ ge=1,+ description="Number of lines to read. If None, reads the entire content",+ )+++class URLReadTool(BaseTool):+ """Read the content at an arbitrary URL and return it as text.++ Unlike :class:`~crewai_tools.tools.file_read_tool.file_read_tool.FileReadTool`,+ which is confined to the local filesystem, this tool performs network+ requests to addresses the caller -- often an LLM -- chooses at runtime. It+ is a separate tool for exactly that reason: granting it is granting network+ egress, and that should be a deliberate choice rather than a flag on a+ filesystem tool.++ Responses are decoded to text according to their content type. PDF and DOCX+ bodies have their text extracted, HTML is stripped to visible text, and+ text-shaped types (plain text, Markdown, JSON, XML, YAML, CSV) are decoded+ as-is. Any other type is refused rather than returned as base64, keeping+ this tool's output text-only.++ Security:+ Requests go through :func:`~crewai_tools.security.safe_requests.safe_get_bounded`,+ which resolves each hostname and rejects it when any resolved address is+ private, loopback, link-local, or otherwise reserved -- covering cloud+ metadata endpoints and internal services. Redirects are never followed+ automatically: every hop is revalidated, and credentials are dropped on+ cross-origin hops. Bodies over ``max_bytes`` are abandoned mid-stream.++ Two risks are not closed here. Validation resolves the hostname and+ requests resolves it again when connecting, so a DNS entry that changes+ between those lookups can still redirect the connection (DNS+ rebinding); closing that requires pinning the connection to the+ validated address. And the returned text is untrusted remote content+ flowing into an agent's context -- a fetched page can attempt to+ instruct the agent. Neither is addressable by input validation alone;+ network egress policy and prompt-level handling cover them.++ Args:+ max_bytes (int): Largest response body to accept, in decoded bytes.+ Defaults to 5 MiB.+ timeout (float): Per-request timeout in seconds. Defaults to 30.+ headers (Optional[dict[str, str]]): Extra request headers. Developer+ supplied, not chosen by the model.+ encoding (Optional[str]): Force a text encoding instead of honoring the+ charset the server declares.+ **kwargs: Additional keyword arguments passed to BaseTool.++ Example:+ >>> tool = URLReadTool()+ >>> content = tool.run(url="https://example.com/report.pdf")+ >>> head = tool.run(url="https://example.com/data.csv", line_count=20)+ """++ name: str = "Read content from a URL"+ description: str = (+ "A tool that reads the content at a URL and returns it as text. To use "+ "this tool, provide a 'url' parameter with an http:// or https:// "+ "address. PDF, DOCX, HTML, JSON, XML, CSV and plain-text responses are "+ "converted to text; other binary types are rejected. URLs that resolve "+ "to private or internal network addresses are refused, as are responses "+ "over the tool's size limit. Optionally provide 'start_line' and "+ "'line_count' to read only part of the content."+ )+ args_schema: type[BaseModel] = URLReadToolSchema+ max_bytes: int = _DEFAULT_MAX_BYTES+ timeout: float = _DEFAULT_TIMEOUT+ headers: dict[str, str] | None = None+ encoding: str | None = None++ def __init__(+ self,+ max_bytes: int = _DEFAULT_MAX_BYTES,+ timeout: float = _DEFAULT_TIMEOUT,+ headers: dict[str, str] | None = None,+ encoding: str | None = None,+ **kwargs: Any,+ ) -> None:+ """Initialize the URLReadTool.++ Args:+ max_bytes: Largest response body to accept, in decoded bytes.+ timeout: Per-request timeout in seconds.+ headers: Extra request headers.+ encoding: Force a text encoding instead of the server's charset.+ **kwargs: Additional keyword arguments passed to BaseTool.+ """+ super().__init__(**kwargs)+ self.max_bytes = max_bytes+ self.timeout = timeout+ self.headers = headers+ self.encoding = encoding++ def _request_headers(self) -> dict[str, str]:+ """Return the headers to send, with caller headers taking precedence."""+ return {+ "Accept": "*/*",+ "User-Agent": "Mozilla/5.0 (compatible; crewai-tools URLReadTool)",+ **(self.headers or {}),+ }++ @staticmethod+ def _classify(media_type: str) -> str | None:+ """Map a media type onto the extractor that handles it."""+ if media_type == _PDF_TYPE:+ return "pdf"+ if media_type == _DOCX_TYPE:+ return "docx"+ if media_type in _HTML_TYPES:+ return "html"+ if (+ media_type.startswith("text/")+ or media_type in _TEXT_TYPES+ or media_type.endswith(_TEXT_TYPE_SUFFIXES)+ ):+ return "text"+ return None++ def _resolve_kind(self, content_type: str, *urls: str) -> str | None:+ """Decide how to extract text, by content type then by URL extension.++ Args:+ content_type: The raw Content-Type header value.+ *urls: URLs to consult for an extension, most authoritative first.+ A ``.pdf`` link that redirects to an extensionless CDN or+ presigned path only carries its type on the requested URL, so+ both ends of the chain are worth checking.++ Returns:+ The extractor name, or None when the content type is unsupported.+ """+ declared = content_type.split(";", 1)[0].strip().lower()+ if declared not in _UNINFORMATIVE_TYPES:+ return self._classify(declared)++ for url in urls:+ path = urlparse(url).path.lower()+ for extension, media_type in _EXTENSION_TYPES.items():+ if path.endswith(extension):+ return self._classify(media_type)+ return None++ def _decode(self, body: bytes, content_type: str) -> str:+ """Decode *body* using the configured, declared, or default encoding.++ Falls back to a replacing UTF-8 decode rather than failing: partially+ readable text is more useful to an agent than an error.+ """+ encoding = self.encoding or _charset_from_content_type(content_type) or "utf-8"+ try:+ return body.decode(encoding)+ except (LookupError, UnicodeDecodeError):+ return body.decode("utf-8", errors="replace")++ @staticmethod+ def _extract_pdf(body: bytes) -> str:+ """Extract text from PDF bytes, page by page."""+ try:+ import pymupdf # type: ignore[import-untyped]+ except ImportError as e:+ raise ImportError(+ "Reading PDF URLs requires pymupdf. Install with: uv add pymupdf"+ ) from e++ # Opened from memory: the bytes are already in hand, and a temp file+ # would need cleaning up on every error path.+ document = pymupdf.open(stream=body, filetype="pdf")+ try:+ pages = [+ f"Page {number}:\n{text}"+ for number, page in enumerate(document, 1)+ if (text := page.get_text().strip())+ ]+ finally:+ document.close()++ if not pages:+ return "[PDF with no extractable text]"+ return "\n\n".join(pages)++ @staticmethod+ def _extract_docx(body: bytes) -> str:+ """Extract paragraph text from DOCX bytes."""+ try:+ from docx import Document+ except ImportError as e:+ raise ImportError(+ "Reading DOCX URLs requires python-docx. Install with: "+ "uv add python-docx"+ ) from e++ document = Document(BytesIO(body))+ return "\n".join(+ paragraph.text+ for paragraph in document.paragraphs+ if paragraph.text.strip()+ )++ def _extract_html(self, body: bytes, content_type: str) -> str:+ """Strip HTML bytes down to visible text."""+ try:+ from bs4 import BeautifulSoup+ except ImportError as e:+ raise ImportError(+ "Reading HTML URLs requires beautifulsoup4. Install with: "+ "uv add beautifulsoup4"+ ) from e++ soup = BeautifulSoup(self._decode(body, content_type), "html.parser")+ for element in soup(["script", "style"]):+ element.decompose()++ text = _SPACES_PATTERN.sub(" ", soup.get_text(" "))+ return _NEWLINE_PATTERN.sub("\n", text).strip()++ def _extract(self, body: bytes, kind: str, content_type: str) -> str:+ """Dispatch to the extractor named by *kind*."""+ if kind == "pdf":+ return self._extract_pdf(body)+ if kind == "docx":+ return self._extract_docx(body)+ if kind == "html":+ return self._extract_html(body, content_type)+ return self._decode(body, content_type)++ @staticmethod+ def _window(text: str, start_line: int, line_count: int | None) -> str:+ """Return the requested line window of *text*.++ The whole body has already been fetched by this point, so unlike the+ filesystem equivalent this only trims output -- it saves no transfer.++ The bounds are clamped rather than trusted: the args schema rejects+ anything below 1 before it gets here, but islice raises on a negative+ stop index, and this runs outside the caller's error handling.+ """+ if start_line == 1 and line_count is None:+ return text++ start_index = max(start_line - 1, 0)+ stop_index = None if line_count is None else start_index + max(line_count, 0)+ selected = list(islice(text.splitlines(keepends=True), start_index, stop_index))++ if not selected and start_index > 0:+ return (+ f"Error: Start line {start_line} exceeds the number of lines in "+ f"the content."+ )+ return "".join(selected)++ def _run(+ self,+ url: str,+ start_line: int | None = 1,+ line_count: int | None = None,+ ) -> str:+ """Fetch a URL and return its content, or a window of it, as text."""+ start_line = start_line or 1+ line_count = line_count or None++ try:+ body, content_type, final_url = safe_get_bounded(+ url,+ max_bytes=self.max_bytes,+ timeout=self.timeout,+ headers=self._request_headers(),+ )+ except ValueError as e:+ return f"Error: {e}"+ except requests.RequestException as e:+ return f"Error: Failed to fetch '{url}'. {format_error_for_display(e)}"++ kind = self._resolve_kind(content_type, final_url, url)+ if kind is None:+ return (+ f"Error: Unsupported content type "+ f"'{content_type.split(';', 1)[0].strip() or 'unknown'}' at "+ f"'{url}'. This tool reads text, HTML, JSON, XML, CSV, PDF and "+ f"DOCX responses."+ )++ try:+ text = self._extract(body, kind, content_type)+ except ImportError as e:+ return f"Error: {e}"+ except Exception as e:+ return (+ f"Error: Failed to read {kind.upper()} content from '{url}'. "+ f"{format_error_for_display(e)}"+ )++ return self._window(text, start_line, line_count)diff --git a/lib/crewai-tools/tests/rag/test_pdf_loader.py b/lib/crewai-tools/tests/rag/test_pdf_loader.pynew file mode 100644index 000000000..9bae12151--- /dev/null+++ b/lib/crewai-tools/tests/rag/test_pdf_loader.py@@ -0,0 +1,156 @@+import tempfile+from unittest.mock import patch++from crewai_tools.rag.base_loader import LoaderResult+from crewai_tools.rag.loaders.pdf_loader import PDFLoader+from crewai_tools.rag.source_content import SourceContent+import pytest+++pymupdf = pytest.importorskip("pymupdf")++# Patched at the loader's seam rather than at requests.get: safe_get_bounded+# resolves the hostname before issuing a request, which would make these tests+# depend on DNS for example.com.+FETCH = "crewai_tools.rag.loaders.pdf_loader.safe_get_bounded"+++def build_pdf(text: str = "Quarterly revenue was 42") -> bytes:+ """Return the bytes of a one-page PDF containing *text*."""+ document = pymupdf.open()+ document.new_page().insert_text((72, 72), text)+ try:+ return document.tobytes()+ finally:+ document.close()+++def fetch_result(body: bytes, url: str = "https://example.com/report.pdf"):+ """Build the (body, content_type, final_url) tuple safe_get_bounded returns."""+ return body, "application/pdf", url+++class TestPDFLoader:+ def test_load_pdf_from_file(self):+ """A PDF on disk has its text extracted with page markers."""+ with tempfile.NamedTemporaryFile(suffix=".pdf") as f:+ f.write(build_pdf())+ f.flush()++ result = PDFLoader().load(SourceContent(f.name))++ assert isinstance(result, LoaderResult)+ assert "Page 1:" in result.content+ assert "Quarterly revenue was 42" in result.content+ assert result.metadata["num_pages"] == 1+ assert result.metadata["file_type"] == "pdf"++ def test_load_pdf_from_url(self):+ """A PDF fetched from a URL is extracted and attributed to that URL."""+ with patch(FETCH) as fetch:+ fetch.return_value = fetch_result(build_pdf("Content from URL"))+ result = PDFLoader().load(SourceContent("https://example.com/report.pdf"))++ assert "Content from URL" in result.content+ assert result.source == "https://example.com/report.pdf"+ assert result.metadata["file_name"] == "report.pdf"++ headers = fetch.call_args.kwargs["headers"]+ assert headers["Accept"] == "application/pdf"+ assert "crewai-tools PDFLoader" in headers["User-Agent"]++ def test_load_pdf_from_url_leaves_no_temp_file(self):+ """The URL path must not write a temp file it never cleans up.++ It previously used NamedTemporaryFile(delete=False) without unlinking,+ so every PDF ingested from a URL left a file behind.+ """+ with (+ patch(FETCH) as fetch,+ patch("tempfile.NamedTemporaryFile") as mock_tempfile,+ ):+ fetch.return_value = fetch_result(build_pdf())+ PDFLoader().load(SourceContent("https://example.com/report.pdf"))++ mock_tempfile.assert_not_called()++ def test_load_pdf_from_url_is_size_bounded(self):+ """The download is capped, since the body is held in memory."""+ with patch(FETCH) as fetch:+ fetch.return_value = fetch_result(build_pdf())+ PDFLoader().load(SourceContent("https://example.com/report.pdf"))++ assert fetch.call_args.kwargs["max_bytes"] == 50 * 1024 * 1024++ def test_load_pdf_from_url_accepts_a_custom_size_limit(self):+ """Callers can lower or raise the ceiling per load."""+ with patch(FETCH) as fetch:+ fetch.return_value = fetch_result(build_pdf())+ PDFLoader().load(+ SourceContent("https://example.com/report.pdf"), max_bytes=1024+ )++ assert fetch.call_args.kwargs["max_bytes"] == 1024++ def test_load_pdf_from_url_with_custom_headers(self):+ """Caller-supplied headers replace the loader's defaults."""+ custom_headers = {"Authorization": "Bearer token"}++ with patch(FETCH) as fetch:+ fetch.return_value = fetch_result(build_pdf())+ PDFLoader().load(+ SourceContent("https://example.com/report.pdf"), headers=custom_headers+ )++ assert fetch.call_args.kwargs["headers"] == custom_headers++ def test_load_pdf_url_download_error(self):+ """A failed download surfaces as a ValueError naming the URL."""+ with patch(FETCH, side_effect=Exception("Network error")):+ with pytest.raises(ValueError, match="Failed to download PDF"):+ PDFLoader().load(SourceContent("https://example.com/report.pdf"))++ def test_load_pdf_url_over_size_limit(self):+ """An oversized body is reported rather than partially parsed."""+ with patch(FETCH, side_effect=ValueError("exceeds the 1024 byte limit")):+ with pytest.raises(ValueError, match="Failed to download PDF"):+ PDFLoader().load(SourceContent("https://example.com/huge.pdf"))++ def test_load_pdf_missing_file(self):+ """A missing local path raises FileNotFoundError, not ValueError."""+ with pytest.raises(FileNotFoundError, match="PDF file not found"):+ PDFLoader().load(SourceContent("/nonexistent/report.pdf"))++ def test_load_corrupt_pdf_raises_value_error(self):+ """Bytes that are not a parseable PDF produce a read error."""+ with tempfile.NamedTemporaryFile(suffix=".pdf") as f:+ f.write(b"%PDF-1.4 not really a pdf")+ f.flush()++ with pytest.raises(ValueError, match="Error reading PDF"):+ PDFLoader().load(SourceContent(f.name))++ def test_pdf_with_no_extractable_text(self):+ """A PDF whose pages hold no text says so instead of returning empty."""+ document = pymupdf.open()+ document.new_page()+ blank = document.tobytes()+ document.close()++ with tempfile.NamedTemporaryFile(suffix=".pdf") as f:+ f.write(blank)+ f.flush()++ result = PDFLoader().load(SourceContent(f.name))++ assert "no extractable text" in result.content++ def test_pdf_doc_id_is_stable(self):+ """The same source yields the same doc_id across loads."""+ with tempfile.NamedTemporaryFile(suffix=".pdf") as f:+ f.write(build_pdf())+ f.flush()++ loader = PDFLoader()+ source = SourceContent(f.name)+ assert loader.load(source).doc_id == loader.load(source).doc_iddiff --git a/lib/crewai-tools/tests/url_read_tool_test.py b/lib/crewai-tools/tests/url_read_tool_test.pynew file mode 100644index 000000000..9d48beb80--- /dev/null+++ b/lib/crewai-tools/tests/url_read_tool_test.py@@ -0,0 +1,415 @@+from unittest.mock import patch++import pytest+import requests++from crewai_tools import URLReadTool+from crewai_tools.security.safe_requests import safe_get_bounded+++TOOL_MODULE = "crewai_tools.tools.url_read_tool.url_read_tool"+++class FakeResponse:+ """Minimal stand-in for a streamed requests.Response."""++ def __init__(+ self,+ body: bytes = b"",+ content_type: str = "text/plain",+ url: str = "https://example.com/file.txt",+ status_code: int = 200,+ chunk_size: int | None = None,+ ):+ self._body = body+ self._chunk_size = chunk_size+ self.headers = {"Content-Type": content_type} if content_type else {}+ self.url = url+ self.status_code = status_code+ self.history: list["FakeResponse"] = []+ self.closed = False++ def raise_for_status(self) -> None:+ """Mimic requests' error-status behavior."""+ if self.status_code >= 400:+ raise requests.HTTPError(f"{self.status_code} error")++ def iter_content(self, chunk_size: int = 65536):+ """Yield the body in chunks, like a streamed response."""+ size = self._chunk_size or chunk_size+ for index in range(0, len(self._body), size):+ yield self._body[index : index + size]++ def close(self) -> None:+ """Record that the response was closed."""+ self.closed = True+++def build_pdf(text: str = "Quarterly revenue was 42") -> bytes:+ """Return the bytes of a one-page PDF containing *text*."""+ pymupdf = pytest.importorskip("pymupdf")+ document = pymupdf.open()+ document.new_page().insert_text((72, 72), text)+ try:+ return document.tobytes()+ finally:+ document.close()+++def fetch_result(+ body: bytes,+ content_type: str = "text/plain",+ url: str = "https://example.com/f.txt",+):+ """Build the (body, content_type, final_url) tuple safe_get_bounded returns."""+ return body, content_type, url+++def test_reads_plain_text():+ """A text response is returned as-is, with the configured limits applied."""+ tool = URLReadTool()+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ fetch.return_value = fetch_result(b"hello world")+ assert tool.run(url="https://example.com/f.txt") == "hello world"++ assert fetch.call_args.kwargs["max_bytes"] == 5 * 1024 * 1024+ assert fetch.call_args.kwargs["timeout"] == 30+++def test_honors_declared_charset():+ """The charset in the Content-Type header drives decoding."""+ tool = URLReadTool()+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ fetch.return_value = fetch_result(+ "café".encode("latin-1"), "text/plain; charset=iso-8859-1"+ )+ assert tool.run(url="https://example.com/f.txt") == "café"+++def test_encoding_override_wins_over_server_charset():+ """An explicit encoding beats whatever the server declares."""+ tool = URLReadTool(encoding="latin-1")+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ fetch.return_value = fetch_result(+ "café".encode("latin-1"), "text/plain; charset=utf-8"+ )+ assert tool.run(url="https://example.com/f.txt") == "café"+++def test_undecodable_bytes_fall_back_instead_of_failing():+ """Partially readable text beats an error for the agent."""+ tool = URLReadTool()+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ fetch.return_value = fetch_result(b"\xff\xfe bad bytes", "text/plain")+ result = tool.run(url="https://example.com/f.txt")++ assert "bad bytes" in result+ assert not result.startswith("Error:")+++def test_line_window():+ """start_line and line_count select a window of the extracted text."""+ tool = URLReadTool()+ body = b"one\ntwo\nthree\nfour\nfive\n"+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ fetch.return_value = fetch_result(body)+ result = tool.run(url="https://example.com/f.txt", start_line=2, line_count=2)++ assert result == "two\nthree\n"+++def test_start_line_past_end_reports_error():+ """Asking past the end of the content is reported, not silently empty."""+ tool = URLReadTool()+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ fetch.return_value = fetch_result(b"one\ntwo\n")+ result = tool.run(url="https://example.com/f.txt", start_line=99)++ assert "exceeds the number of lines" in result+++ "line_args",+ [{"line_count": -5}, {"line_count": 0}, {"start_line": 0}, {"start_line": -5}],+)+def test_line_arguments_below_one_are_refused(line_args):+ """Out-of-range line arguments are rejected before any request is made.++ islice raises on a negative stop index, and the windowing runs outside the+ tool's error handling, so these have to be refused at validation time.+ """+ tool = URLReadTool()+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ with pytest.raises(ValueError, match="greater than or equal to 1"):+ tool.run(url="https://example.com/f.txt", **line_args)++ fetch.assert_not_called()+++def test_json_is_returned_verbatim():+ """JSON is passed through undecorated so callers can parse it."""+ tool = URLReadTool()+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ fetch.return_value = fetch_result(b'{"a": 1}', "application/json")+ assert tool.run(url="https://example.com/data.json") == '{"a": 1}'+++def test_structured_suffix_type_is_treated_as_text():+ """A +json vendor type is text, not an unsupported binary type."""+ tool = URLReadTool()+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ fetch.return_value = fetch_result(b'{"a": 1}', "application/vnd.api+json")+ assert tool.run(url="https://example.com/data") == '{"a": 1}'+++def test_html_is_stripped_to_visible_text():+ """HTML returns visible text with script and style content removed."""+ tool = URLReadTool()+ body = b"<html><head><style>p{color:red}</style></head><body><p>Hi</p><script>x=1</script></body></html>"+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ fetch.return_value = fetch_result(body, "text/html; charset=utf-8")+ result = tool.run(url="https://example.com/page")++ assert "Hi" in result+ assert "x=1" not in result+ assert "color:red" not in result+++def test_binary_content_type_is_rejected():+ """An unsupported type is refused rather than returned as base64."""+ tool = URLReadTool()+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ fetch.return_value = fetch_result(b"\x89PNG\r\n", "image/png")+ result = tool.run(url="https://example.com/logo.png")++ assert "Unsupported content type 'image/png'" in result+++def test_octet_stream_pdf_falls_back_to_url_extension():+ """A PDF served as octet-stream is still extracted, via its extension."""+ tool = URLReadTool()+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ fetch.return_value = fetch_result(+ build_pdf("Fallback worked"),+ "application/octet-stream",+ "https://example.com/a/b.pdf",+ )+ result = tool.run(url="https://example.com/a/b.pdf")++ assert "Fallback worked" in result+++def test_missing_content_type_falls_back_to_url_extension():+ """No Content-Type at all still reads as text when the path says .csv."""+ tool = URLReadTool()+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ fetch.return_value = fetch_result(b"a,b\n1,2\n", "", "https://example.com/b.csv")+ assert tool.run(url="https://example.com/b.csv") == "a,b\n1,2\n"+++def test_query_string_does_not_break_extension_fallback():+ """A presigned-style query string does not hide the path's extension."""+ tool = URLReadTool()+ url = "https://example.com/b.csv?X-Amz-Signature=abc"+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ fetch.return_value = fetch_result(b"a,b\n", "application/octet-stream", url)+ assert tool.run(url=url) == "a,b\n"+++def test_extension_from_requested_url_survives_a_redirect():+ """A .pdf link that redirects to an extensionless path is still extracted.++ Presigned CDN targets routinely drop the extension and serve octet-stream,+ so the requested URL is the only place the type survives.+ """+ tool = URLReadTool()+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ fetch.return_value = fetch_result(+ build_pdf("Survived the redirect"),+ "application/octet-stream",+ "https://cdn.example.com/objects/9f8a7b6c5d",+ )+ result = tool.run(url="https://example.com/report.pdf")++ assert "Survived the redirect" in result+++def test_octet_stream_with_unknown_extension_is_rejected():+ """With neither a usable type nor a known extension, the read is refused."""+ tool = URLReadTool()+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ fetch.return_value = fetch_result(+ b"\x00\x01", "application/octet-stream", "https://example.com/a/b.bin"+ )+ result = tool.run(url="https://example.com/a/b.bin")++ assert "Unsupported content type" in result+++def test_validation_failure_is_returned_as_error():+ """An SSRF rejection reaches the agent as an error string, not an exception."""+ tool = URLReadTool()+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ fetch.side_effect = ValueError(+ "URL 'http://169.254.169.254/' resolves to private/reserved IP 169.254.169.254."+ )+ result = tool.run(url="http://169.254.169.254/")++ assert result.startswith("Error:")+ assert "private/reserved IP" in result+++def test_request_failure_is_returned_as_error():+ """A transport failure is reported without raising out of the tool."""+ tool = URLReadTool()+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ fetch.side_effect = requests.ConnectionError("connection refused")+ result = tool.run(url="https://example.com/f.txt")++ assert result.startswith("Error: Failed to fetch")+++def test_custom_headers_are_merged_over_defaults():+ """Caller headers win, but the default User-Agent survives."""+ tool = URLReadTool(headers={"Authorization": "Bearer x", "Accept": "text/plain"})+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ fetch.return_value = fetch_result(b"ok")+ tool.run(url="https://example.com/f.txt")++ headers = fetch.call_args.kwargs["headers"]+ assert headers["Authorization"] == "Bearer x"+ assert headers["Accept"] == "text/plain"+ assert "crewai-tools URLReadTool" in headers["User-Agent"]+++def test_reads_a_real_pdf_end_to_end():+ """Real PDF bytes are extracted page by page."""+ pdf_bytes = build_pdf()++ tool = URLReadTool()+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ fetch.return_value = fetch_result(+ pdf_bytes, "application/pdf", "https://example.com/report.pdf"+ )+ result = tool.run(url="https://example.com/report.pdf")++ assert "Page 1:" in result+ assert "Quarterly revenue was 42" in result+++def test_corrupt_pdf_reports_error_without_raising():+ """A malformed PDF becomes an error string, not a traceback."""+ pytest.importorskip("pymupdf")++ tool = URLReadTool()+ with patch(f"{TOOL_MODULE}.safe_get_bounded") as fetch:+ fetch.return_value = fetch_result(+ b"%PDF-1.4 not really a pdf", "application/pdf"+ )+ result = tool.run(url="https://example.com/report.pdf")++ assert result.startswith("Error: Failed to read PDF content")+++class TestSafeGetBounded:+ """Tests for the bounded-fetch helper itself."""++ def test_returns_body_content_type_and_final_url(self):+ """The helper reports the body alongside where it ended up."""+ response = FakeResponse(b"payload", "text/plain", "https://example.com/final")+ with patch(+ "crewai_tools.security.safe_requests.safe_get", return_value=response+ ):+ body, content_type, final_url = safe_get_bounded(+ "https://example.com/start", max_bytes=1024+ )++ assert body == b"payload"+ assert content_type == "text/plain"+ assert final_url == "https://example.com/final"+ assert response.closed++ def test_rejects_body_over_the_limit(self):+ """Crossing max_bytes raises rather than truncating silently."""+ response = FakeResponse(b"x" * 100, chunk_size=10)+ with patch(+ "crewai_tools.security.safe_requests.safe_get", return_value=response+ ):+ with pytest.raises(ValueError, match="exceeds the 25 byte limit"):+ safe_get_bounded("https://example.com/big", max_bytes=25)++ assert response.closed++ def test_oversized_error_names_the_url_that_served_the_body(self):+ """After a redirect the requested URL is not the one that sent it."""+ response = FakeResponse(+ b"x" * 100, url="https://cdn.example.com/final", chunk_size=10+ )+ with patch(+ "crewai_tools.security.safe_requests.safe_get", return_value=response+ ):+ with pytest.raises(ValueError, match="https://cdn.example.com/final"):+ safe_get_bounded("https://example.com/start", max_bytes=25)++ @pytest.mark.parametrize("max_bytes", [0, -1])+ def test_non_positive_max_bytes_fails_before_requesting(self, max_bytes):+ """A misconfigured cap is caught without issuing a request."""+ with patch("crewai_tools.security.safe_requests.safe_get") as safe_get:+ with pytest.raises(ValueError, match="max_bytes must be positive"):+ safe_get_bounded("https://example.com/f", max_bytes=max_bytes)++ safe_get.assert_not_called()++ def test_stops_reading_once_the_limit_is_crossed(self):+ """The cap must abandon the stream, not buffer the whole body first."""+ chunks_yielded = 0++ class CountingResponse(FakeResponse):+ def iter_content(self, chunk_size: int = 65536):+ nonlocal chunks_yielded+ for _ in range(1000):+ chunks_yielded += 1+ yield b"x" * 10++ response = CountingResponse()+ with patch(+ "crewai_tools.security.safe_requests.safe_get", return_value=response+ ):+ with pytest.raises(ValueError):+ safe_get_bounded("https://example.com/huge", max_bytes=25)++ assert chunks_yielded == 3++ def test_error_status_raises(self):+ """An error status propagates as an HTTPError."""+ response = FakeResponse(b"nope", status_code=404)+ with patch(+ "crewai_tools.security.safe_requests.safe_get", return_value=response+ ):+ with pytest.raises(requests.HTTPError):+ safe_get_bounded("https://example.com/missing", max_bytes=1024)++ assert response.closed++ def test_closes_redirect_hops(self):+ """Streamed redirect hops hold connections until closed."""+ hop = FakeResponse(b"", status_code=302)+ response = FakeResponse(b"done")+ response.history = [hop]+ with patch(+ "crewai_tools.security.safe_requests.safe_get", return_value=response+ ):+ safe_get_bounded("https://example.com/start", max_bytes=1024)++ assert hop.closed+ assert response.closed++ def test_requests_are_streamed(self):+ """Streaming is what lets an oversized body be abandoned early."""+ response = FakeResponse(b"ok")+ with patch(+ "crewai_tools.security.safe_requests.safe_get", return_value=response+ ) as safe_get:+ safe_get_bounded("https://example.com/f", max_bytes=1024)++ assert safe_get.call_args.kwargs["stream"] is Truediff --git a/lib/crewai-tools/tests/utilities/test_safe_requests.py b/lib/crewai-tools/tests/utilities/test_safe_requests.pyindex f45dd86c6..895c0044c 100644--- a/lib/crewai-tools/tests/utilities/test_safe_requests.py+++ b/lib/crewai-tools/tests/utilities/test_safe_requests.py@@ -112,6 +112,80 @@ def test_safe_get_fails_closed_after_too_many_redirects(safe_get("http://public.example/start", max_redirects=1, timeout=15)+def _closable_response(+ url: str, status_code: int, *, location: str | None = None, closed: list[str]+) -> requests.Response:+ """Build a response that records its own URL when closed."""+ response = _response(url, status_code, location=location)+ response.close = lambda: closed.append(url) # type: ignore[method-assign]+ return response+++def test_safe_get_closes_earlier_hops_after_too_many_redirects(+ monkeypatch: pytest.MonkeyPatch, public_dns: None+) -> None:+ """Hops accumulated before the failure must not be left open.++ Under stream=True each hop holds its connection until its body is read or+ closed, and a caller handed an exception has no handle on them.+ """+ closed: list[str] = []++ def fake_get(url: str, **kwargs: Any) -> requests.Response:+ return _closable_response(+ url, 302, location="http://safe.example/again", closed=closed+ )++ _mock_get(monkeypatch, fake_get)++ with pytest.raises(ValueError, match="Too many redirects"):+ safe_get("http://public.example/start", max_redirects=2, timeout=15, stream=True)++ assert len(closed) == 3+++def test_safe_get_closes_earlier_hops_when_a_redirect_is_rejected(+ monkeypatch: pytest.MonkeyPatch, public_dns: None+) -> None:+ """A hop rejected mid-chain still releases the connections already open."""+ closed: list[str] = []++ def fake_get(url: str, **kwargs: Any) -> requests.Response:+ if url == "http://public.example/start":+ return _closable_response(+ url, 302, location="http://safe.example/next", closed=closed+ )+ return _closable_response(+ url, 302, location="http://169.254.169.254/latest", closed=closed+ )++ _mock_get(monkeypatch, fake_get)++ with pytest.raises(ValueError, match="private/reserved IP"):+ safe_get("http://public.example/start", timeout=15, stream=True)++ assert closed == ["http://safe.example/next", "http://public.example/start"]+++def test_safe_get_leaves_hops_open_on_success(+ monkeypatch: pytest.MonkeyPatch, public_dns: None+) -> None:+ """On success the hops belong to the caller, via response.history."""+ closed: list[str] = []++ def fake_get(url: str, **kwargs: Any) -> requests.Response:+ if url == "http://public.example/start":+ return _closable_response(url, 302, location="/final", closed=closed)+ return _closable_response(url, 200, closed=closed)++ _mock_get(monkeypatch, fake_get)++ response = safe_get("http://public.example/start", timeout=15, stream=True)++ assert closed == []+ assert len(response.history) == 1++def test_safe_get_strips_credentials_on_cross_origin_redirect(monkeypatch: pytest.MonkeyPatch, public_dns: None) -> None:diff --git a/lib/crewai-tools/tool.specs.json b/lib/crewai-tools/tool.specs.jsonindex a03513308..6380dc7c6 100644--- a/lib/crewai-tools/tool.specs.json+++ b/lib/crewai-tools/tool.specs.json@@ -26885,6 +26885,148 @@"type": "object"}},+ {+ "description": "A tool that reads the content at a URL and returns it as text. To use this tool, provide a 'url' parameter with an http:// or https:// address. PDF, DOCX, HTML, JSON, XML, CSV and plain-text responses are converted to text; other binary types are rejected. URLs that resolve to private or internal network addresses are refused, as are responses over the tool's size limit. Optionally provide 'start_line' and 'line_count' to read only part of the content.",+ "env_vars": [],+ "humanized_name": "Read content from a URL",+ "init_params_schema": {+ "$defs": {+ "EnvVar": {+ "properties": {+ "default": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "default": null,+ "title": "Default"+ },+ "description": {+ "title": "Description",+ "type": "string"+ },+ "name": {+ "title": "Name",+ "type": "string"+ },+ "required": {+ "default": true,+ "title": "Required",+ "type": "boolean"+ }+ },+ "required": [+ "name",+ "description"+ ],+ "title": "EnvVar",+ "type": "object"+ },+ "ToolFailurePolicy": {+ "description": "How an agent reacts when one of its tools reports a failure.",+ "enum": [+ "ignore",+ "warn",+ "raise"+ ],+ "title": "ToolFailurePolicy",+ "type": "string"+ }+ },+ "description": "Read the content at an arbitrary URL and return it as text.\n\nUnlike :class:`~crewai_tools.tools.file_read_tool.file_read_tool.FileReadTool`,\nwhich is confined to the local filesystem, this tool performs network\nrequests to addresses the caller -- often an LLM -- chooses at runtime. It\nis a separate tool for exactly that reason: granting it is granting network\negress, and that should be a deliberate choice rather than a flag on a\nfilesystem tool.\n\nResponses are decoded to text according to their content type. PDF and DOCX\nbodies have their text extracted, HTML is stripped to visible text, and\ntext-shaped types (plain text, Markdown, JSON, XML, YAML, CSV) are decoded\nas-is. Any other type is refused rather than returned as base64, keeping\nthis tool's output text-only.\n\nSecurity:\n Requests go through :func:`~crewai_tools.security.safe_requests.safe_get_bounded`,\n which resolves each hostname and rejects it when any resolved address is\n private, loopback, link-local, or otherwise reserved -- covering cloud\n metadata endpoints and internal services. Redirects are never followed\n automatically: every hop is revalidated, and credentials are dropped on\n cross-origin hops. Bodies over ``max_bytes`` are abandoned mid-stream.\n\n Two risks are not closed here. Validation resolves the hostname and\n requests resolves it again when connecting, so a DNS entry that changes\n between those lookups can still redirect the connection (DNS\n rebinding); closing that requires pinning the connection to the\n validated address. And the returned text is untrusted remote content\n flowing into an agent's context -- a fetched page can attempt to\n instruct the agent. Neither is addressable by input validation alone;\n network egress policy and prompt-level handling cover them.\n\nArgs:\n max_bytes (int): Largest response body to accept, in decoded bytes.\n Defaults to 5 MiB.\n timeout (float): Per-request timeout in seconds. Defaults to 30.\n headers (Optional[dict[str, str]]): Extra request headers. Developer\n supplied, not chosen by the model.\n encoding (Optional[str]): Force a text encoding instead of honoring the\n charset the server declares.\n **kwargs: Additional keyword arguments passed to BaseTool.\n\nExample:\n >>> tool = URLReadTool()\n >>> content = tool.run(url=\"https://example.com/report.pdf\")\n >>> head = tool.run(url=\"https://example.com/data.csv\", line_count=20)",+ "properties": {+ "encoding": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "default": null,+ "title": "Encoding"+ },+ "headers": {+ "anyOf": [+ {+ "additionalProperties": {+ "type": "string"+ },+ "type": "object"+ },+ {+ "type": "null"+ }+ ],+ "default": null,+ "title": "Headers"+ },+ "max_bytes": {+ "default": 5242880,+ "title": "Max Bytes",+ "type": "integer"+ },+ "timeout": {+ "default": 30,+ "title": "Timeout",+ "type": "number"+ }+ },+ "required": [],+ "title": "URLReadTool",+ "type": "object"+ },+ "name": "URLReadTool",+ "package_dependencies": [],+ "run_params_schema": {+ "description": "Input for URLReadTool.",+ "properties": {+ "line_count": {+ "anyOf": [+ {+ "minimum": 1,+ "type": "integer"+ },+ {+ "type": "null"+ }+ ],+ "default": null,+ "description": "Number of lines to read. If None, reads the entire content",+ "title": "Line Count"+ },+ "start_line": {+ "anyOf": [+ {+ "minimum": 1,+ "type": "integer"+ },+ {+ "type": "null"+ }+ ],+ "default": 1,+ "description": "Line number to start reading from (1-indexed)",+ "title": "Start Line"+ },+ "url": {+ "description": "The http:// or https:// URL to read. Addresses that resolve to private or internal networks are refused.",+ "title": "Url",+ "type": "string"+ }+ },+ "required": [+ "url"+ ],+ "title": "URLReadToolSchema",+ "type": "object"+ }+ },{"description": "This tool uses OpenAI's Vision API to describe the contents of an image.","env_vars": [