Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

1.1 KB
Raw Blame
"""Utility functions for RAG loaders."""

from typing import Any


def load_from_url(
    url: str,
    kwargs: dict[str, Any],
    accept_header: str = "*/*",
    loader_name: str = "Loader",
) -> str:
    """Load content from a URL.

    Args:
        url: The URL to fetch content from
        kwargs: Additional keyword arguments (can include 'headers' override)
        accept_header: The Accept header value for the request
        loader_name: The name of the loader for the User-Agent header

    Returns:
        The text content from the URL

    Raises:
        ValueError: If there's an error fetching the URL
    """
    from crewai_tools.security.safe_requests import safe_get

    headers = kwargs.get(
        "headers",
        {
            "Accept": accept_header,
            "User-Agent": f"Mozilla/5.0 (compatible; crewai-tools {loader_name})",
        },
    )

    try:
        response = safe_get(url, headers=headers, timeout=30)
        response.raise_for_status()
        return response.text
    except Exception as e:
        raise ValueError(f"Error fetching content from URL {url}: {e!s}") from e