Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

4.7 KB
Raw Blame
"""PDF loader for extracting text from PDF files."""

import os
from pathlib import Path
from typing import Any
from urllib.parse import urlparse

from crewai_tools.rag.base_loader import BaseLoader, LoaderResult
from crewai_tools.rag.source_content import SourceContent
from crewai_tools.security.safe_requests import safe_get_bounded


# Remote PDFs are held in memory for the whole extraction, so the download needs
# a ceiling. Override per call with the ``max_bytes`` kwarg to ``load``.
DEFAULT_MAX_PDF_BYTES = 50 * 1024 * 1024


class PDFLoader(BaseLoader):
    """Loader for PDF files and URLs."""

    @staticmethod
    def _is_url(path: str) -> bool:
        """Check if the path is a URL."""
        try:
            parsed = urlparse(path)
            return parsed.scheme in ("http", "https")
        except Exception:
            return False

    @staticmethod
    def _fetch_from_url(url: str, kwargs: dict[str, Any]) -> bytes:
        """Download a PDF from a URL and return its bytes.

        The content stays in memory rather than going to a temporary file: the
        whole body has to be buffered either way, and a temp file would need
        unlinking on every error path to avoid leaving files behind. Because it
        is held in memory, the download is capped.

        Args:
            url: The URL to download from.
            kwargs: Optional dict that may contain custom ``headers`` and a
                ``max_bytes`` ceiling for the download.

        Returns:
            The raw PDF content.

        Raises:
            ValueError: If the download fails or exceeds the size ceiling.
        """
        headers = kwargs.get(
            "headers",
            {
                "Accept": "application/pdf",
                "User-Agent": "Mozilla/5.0 (compatible; crewai-tools PDFLoader)",
            },
        )

        try:
            body, _content_type, _final_url = safe_get_bounded(
                url,
                max_bytes=kwargs.get("max_bytes", DEFAULT_MAX_PDF_BYTES),
                headers=headers,
                timeout=30,
            )
            return body
        except Exception as e:
            raise ValueError(f"Failed to download PDF from {url}: {e!s}") from e

    def load(self, source: SourceContent, **kwargs: Any) -> LoaderResult:  # type: ignore[override]
        """Load and extract text from a PDF file or URL.

        Args:
            source: The source content containing the PDF file path or URL.

        Returns:
            LoaderResult with extracted text content.

        Raises:
            FileNotFoundError: If the PDF file doesn't exist.
            ImportError: If required PDF libraries aren't installed.
            ValueError: If the PDF cannot be read or downloaded.
        """
        try:
            import pymupdf  # type: ignore[import-untyped]
        except ImportError as e:
            raise ImportError(
                "PDF support requires pymupdf. Install with: uv add pymupdf"
            ) from e

        file_path = source.source
        is_url = self._is_url(file_path)

        if is_url:
            source_name = Path(urlparse(file_path).path).name or "downloaded.pdf"
        else:
            source_name = Path(file_path).name

        text_content: list[str] = []
        metadata: dict[str, Any] = {
            "source": file_path,
            "file_name": source_name,
            "file_type": "pdf",
        }

        try:
            if is_url:
                doc = pymupdf.open(
                    stream=self._fetch_from_url(file_path, kwargs), filetype="pdf"
                )
            else:
                if not os.path.isfile(file_path):
                    raise FileNotFoundError(f"PDF file not found: {file_path}")
                doc = pymupdf.open(file_path)

            # Closed in a finally so a failure mid-extraction still releases the
            # document handle.
            try:
                metadata["num_pages"] = len(doc)

                for page_num, page in enumerate(doc, 1):
                    page_text = page.get_text()
                    if page_text.strip():
                        text_content.append(f"Page {page_num}:\n{page_text}")
            finally:
                doc.close()
        except FileNotFoundError:
            raise
        except Exception as e:
            raise ValueError(f"Error reading PDF from {file_path}: {e!s}") from e

        if not text_content:
            content = f"[PDF file with no extractable text: {source_name}]"
        else:
            content = "\n\n".join(text_content)

        return LoaderResult(
            content=content,
            source=file_path,
            metadata=metadata,
            doc_id=self.generate_doc_id(source_ref=file_path, content=content),
        )