Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

1.4 KB
Raw Blame
from typing import Any

from crewai_tools.rag.base_loader import BaseLoader, LoaderResult
from crewai_tools.rag.loaders.utils import load_from_url
from crewai_tools.rag.source_content import SourceContent


class TextFileLoader(BaseLoader):
    def load(self, source_content: SourceContent, **kwargs: Any) -> LoaderResult:  # type: ignore[override]
        source_ref = source_content.source_ref
        if source_content.is_url():
            content = load_from_url(
                source_ref,
                kwargs,
                accept_header="text/plain",
                loader_name="TextFileLoader",
            )
        else:
            if not source_content.path_exists():
                raise FileNotFoundError(
                    f"The following file does not exist: {source_content.source}"
                )

            with open(source_content.source, encoding="utf-8") as file:
                content = file.read()

        return LoaderResult(
            content=content,
            source=source_ref,
            doc_id=self.generate_doc_id(source_ref=source_ref, content=content),
        )


class TextLoader(BaseLoader):
    def load(self, source_content: SourceContent, **kwargs: Any) -> LoaderResult:  # type: ignore[override]
        return LoaderResult(
            content=source_content.source,
            source=source_content.source_ref,
            doc_id=self.generate_doc_id(content=source_content.source),
        )