Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

13.4 KB
Raw Blame
from abc import ABC, abstractmethod
import os
from typing import Annotated, Any, Literal, cast

from crewai.rag.core.base_embeddings_callable import EmbeddingFunction
from crewai.rag.embeddings.factory import build_embedder
from crewai.rag.embeddings.types import ProviderSpec
from crewai.tools import BaseTool
from pydantic import (
    BaseModel,
    BeforeValidator,
    ConfigDict,
    Field,
    PlainSerializer,
    TypeAdapter,
    ValidationError,
    WithJsonSchema,
    field_validator,
    model_validator,
)
from typing_extensions import Self, Unpack

from crewai_tools.tools.rag.types import (
    AddDocumentParams,
    ContentItem,
    RagToolConfig,
    VectorDbConfig,
)


def _validate_embedding_config(
    value: dict[str, Any] | ProviderSpec,
) -> dict[str, Any] | ProviderSpec:
    """Validate embedding config and provide clearer error messages for union validation.

    This pre-validator catches Pydantic ValidationErrors from the ProviderSpec union
    and provides a cleaner, more focused error message that only shows the relevant
    provider's validation errors instead of all 18 union members.

    Args:
        value: The embedding configuration dictionary or validated ProviderSpec.

    Returns:
        A validated ProviderSpec instance, or the original value if already validated
        or missing required fields.

    Raises:
        ValueError: If the configuration is invalid for the specified provider.
    """
    if not isinstance(value, dict):
        return value

    provider = value.get("provider")
    if not provider:
        return value

    try:
        type_adapter: TypeAdapter[ProviderSpec] = TypeAdapter(ProviderSpec)
        return type_adapter.validate_python(value)
    except ValidationError as e:
        provider_key = f"{provider.lower()}providerspec"
        provider_errors = [
            err for err in e.errors() if provider_key in str(err.get("loc", "")).lower()
        ]

        if provider_errors:
            error_msgs = []
            for err in provider_errors:
                loc_parts = err["loc"]
                if str(loc_parts[0]).lower() == provider_key:
                    loc_parts = loc_parts[1:]
                loc = ".".join(str(x) for x in loc_parts)
                error_msgs.append(f"  - {loc}: {err['msg']}")

            raise ValueError(
                f"Invalid configuration for embedding provider '{provider}':\n"
                + "\n".join(error_msgs)
            ) from e

        raise


class Adapter(BaseModel, ABC):
    """Abstract base class for RAG adapters."""

    model_config = ConfigDict(arbitrary_types_allowed=True)

    @abstractmethod
    def query(
        self,
        question: str,
        similarity_threshold: float | None = None,
        limit: int | None = None,
    ) -> str:
        """Query the knowledge base with a question and return the answer."""

    @abstractmethod
    def add(
        self,
        *args: ContentItem,
        **kwargs: Unpack[AddDocumentParams],
    ) -> None:
        """Add content to the knowledge base."""


def _resolve_adapter(value: Any) -> Any:
    """Validate the ``adapter`` field, returning a placeholder for dict/None input.

    Adapter state is not round-tripped; the ``_ensure_adapter`` post-validator
    rebuilds a fresh adapter from the tool's ``config``.
    """
    if isinstance(value, Adapter):
        return value
    if value is None or isinstance(value, dict):
        return RagTool._AdapterPlaceholder()
    return value


def _serialize_adapter(adapter: Any, info: Any) -> Any:
    """Serialize the ``adapter`` field, dropping runtime state from the payload."""
    if not isinstance(adapter, Adapter):
        return adapter
    return None


class RagTool(BaseTool):
    class _AdapterPlaceholder(Adapter):
        def query(
            self,
            question: str,
            similarity_threshold: float | None = None,
            limit: int | None = None,
        ) -> str:
            raise NotImplementedError

        def add(
            self,
            *args: ContentItem,
            **kwargs: Unpack[AddDocumentParams],
        ) -> None:
            raise NotImplementedError

    name: str = "Knowledge base"
    description: str = "A knowledge base that can be used to answer questions."
    summarize: bool = False
    similarity_threshold: float = 0.6
    limit: int = 5
    collection_name: str = "rag_tool_collection"
    adapter: Annotated[
        Adapter,
        BeforeValidator(_resolve_adapter),
        PlainSerializer(_serialize_adapter, when_used="json"),
        WithJsonSchema({"type": ["object", "null"]}),
    ] = Field(default_factory=_AdapterPlaceholder)
    config: RagToolConfig = Field(
        default_factory=RagToolConfig,
        description="Configuration format accepted by RagTool.",
    )

    @field_validator("config", mode="before")
    @classmethod
    def _validate_config(cls, value: Any) -> Any:
        """Validate config with improved error messages for embedding providers."""
        if not isinstance(value, dict):
            return value

        embedding_model = value.get("embedding_model")
        if embedding_model:
            try:
                value["embedding_model"] = _validate_embedding_config(embedding_model)
            except ValueError:
                raise

        return value

    @model_validator(mode="after")
    def _ensure_adapter(self) -> Self:
        if isinstance(self.adapter, RagTool._AdapterPlaceholder):
            from crewai_tools.adapters.crewai_rag_adapter import CrewAIRagAdapter

            provider_cfg = self._parse_config(self.config)
            self.adapter = CrewAIRagAdapter(
                collection_name=self.collection_name,
                summarize=self.summarize,
                similarity_threshold=self.similarity_threshold,
                limit=self.limit,
                config=provider_cfg,
            )
        return self

    def _parse_config(self, config: RagToolConfig) -> Any:
        """Normalize the RagToolConfig into a provider-specific config object.

        Defaults to 'chromadb' with no extra provider config if none is supplied.
        """
        if not config:
            return self._create_provider_config("chromadb", {}, None)

        vectordb_cfg = cast(VectorDbConfig, config.get("vectordb", {}))
        provider: Literal["chromadb", "qdrant"] = vectordb_cfg.get(
            "provider", "chromadb"
        )
        provider_config: dict[str, Any] = vectordb_cfg.get("config", {})

        supported = ("chromadb", "qdrant")
        if provider not in supported:
            raise ValueError(
                f"Unsupported vector database provider: '{provider}'. "
                f"CrewAI RAG currently supports: {', '.join(supported)}."
            )

        embedding_spec: ProviderSpec | None = config.get("embedding_model")
        if embedding_spec:
            embedding_spec = cast(
                ProviderSpec, _validate_embedding_config(embedding_spec)
            )

        embedding_function = build_embedder(embedding_spec) if embedding_spec else None
        return self._create_provider_config(
            provider, provider_config, embedding_function
        )

    @staticmethod
    def _create_provider_config(
        provider: Literal["chromadb", "qdrant"],
        provider_config: dict[str, Any],
        embedding_function: EmbeddingFunction[Any] | None,
    ) -> Any:
        """Instantiate provider config with optional embedding_function injected."""
        if provider == "chromadb":
            from crewai.rag.chromadb.config import ChromaDBConfig

            kwargs = dict(provider_config)
            if embedding_function is not None:
                kwargs["embedding_function"] = embedding_function
            return ChromaDBConfig(**kwargs)

        if provider == "qdrant":
            from crewai.rag.qdrant.config import QdrantConfig

            kwargs = dict(provider_config)
            if embedding_function is not None:
                kwargs["embedding_function"] = embedding_function
            return QdrantConfig(**kwargs)

        raise ValueError(f"Unhandled provider: {provider}")

    def add(
        self,
        *args: ContentItem,
        **kwargs: Unpack[AddDocumentParams],
    ) -> None:
        """Add content to the knowledge base.


        Args:
            *args: Content items to add (strings, paths, or document dicts)
            data_type: DataType enum or string (e.g., "file", "pdf_file", "text")
            path: Path to file or directory, alias to positional arg
            file_path: Alias for path
            metadata: Additional metadata to attach to documents
            url: URL to fetch content from
            website: Website URL to scrape
            github_url: GitHub repository URL
            youtube_url: YouTube video URL
            directory_path: Path to directory

        Examples:
            rag_tool.add("path/to/document.pdf", data_type=DataType.PDF_FILE)

            # Keyword argument (documented API)
            rag_tool.add(path="path/to/document.pdf", data_type="file")
            rag_tool.add(file_path="path/to/document.pdf", data_type="pdf_file")

            # Auto-detect type from extension
            rag_tool.add("path/to/document.pdf")  # auto-detects PDF
        """
        # Validate file paths and URLs before adding to prevent
        # unauthorized file reads and SSRF.
        from urllib.parse import urlparse

        from crewai_tools.security.safe_path import validate_file_path, validate_url

        def _check_url(value: str, label: str) -> None:
            try:
                validate_url(value)
            except ValueError as e:
                raise ValueError(f"Blocked unsafe {label}: {e}") from e

        def _check_path(value: str, label: str) -> str:
            try:
                return validate_file_path(value)
            except ValueError as e:
                raise ValueError(f"Blocked unsafe {label}: {e}") from e

        validated_args: list[ContentItem] = []
        for arg in args:
            source_ref = (
                str(arg.get("source", arg.get("content", "")))
                if isinstance(arg, dict)
                else str(arg)
            )

            # Check if it's a URL — only catch urlparse-specific errors here;
            # validate_url's ValueError must propagate so it is never silently bypassed.
            try:
                parsed = urlparse(source_ref)
            except (ValueError, AttributeError):
                parsed = None

            if parsed is not None and parsed.scheme in ("http", "https", "file"):
                try:
                    validate_url(source_ref)
                except ValueError as e:
                    raise ValueError(f"Blocked unsafe URL: {e}") from e
                validated_args.append(arg)
                continue

            # Check if it looks like a file path (not a plain text string).
            # Check both os.sep (backslash on Windows) and "/" so that
            # forward-slash paths like "sub/file.txt" are caught on all platforms.
            if (
                os.path.sep in source_ref
                or "/" in source_ref
                or source_ref.startswith(".")
                or os.path.isabs(source_ref)
            ):
                try:
                    resolved_ref = validate_file_path(source_ref)
                except ValueError as e:
                    raise ValueError(f"Blocked unsafe file path: {e}") from e
                # Use the resolved path to prevent symlink TOCTOU
                if isinstance(arg, dict):
                    arg = {**arg}
                    if "source" in arg:
                        arg["source"] = resolved_ref
                    elif "content" in arg:
                        arg["content"] = resolved_ref
                else:
                    arg = resolved_ref

            validated_args.append(arg)

        # Validate keyword path/URL arguments — these are equally user-controlled
        # and must not bypass the checks applied to positional args.
        if "path" in kwargs and kwargs.get("path") is not None:
            kwargs["path"] = _check_path(str(kwargs["path"]), "path")
        if "file_path" in kwargs and kwargs.get("file_path") is not None:
            kwargs["file_path"] = _check_path(str(kwargs["file_path"]), "file_path")

        if "directory_path" in kwargs and kwargs.get("directory_path") is not None:
            kwargs["directory_path"] = _check_path(
                str(kwargs["directory_path"]), "directory_path"
            )

        if "url" in kwargs and kwargs.get("url") is not None:
            _check_url(str(kwargs["url"]), "url")
        if "website" in kwargs and kwargs.get("website") is not None:
            _check_url(str(kwargs["website"]), "website")
        if "github_url" in kwargs and kwargs.get("github_url") is not None:
            _check_url(str(kwargs["github_url"]), "github_url")
        if "youtube_url" in kwargs and kwargs.get("youtube_url") is not None:
            _check_url(str(kwargs["youtube_url"]), "youtube_url")

        self.adapter.add(*validated_args, **kwargs)

    def _run(
        self,
        query: str,
        similarity_threshold: float | None = None,
        limit: int | None = None,
    ) -> str:
        threshold = (
            similarity_threshold
            if similarity_threshold is not None
            else self.similarity_threshold
        )
        result_limit = limit if limit is not None else self.limit
        return f"Relevant Content:\n{self.adapter.query(query, similarity_threshold=threshold, limit=result_limit)}"