Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
fix(rag): preserve documentation site source URLs (#7951)
Jialiang Liangcbb500d3b827
diff --git a/lib/crewai-tools/src/crewai_tools/rag/loaders/docs_site_loader.py b/lib/crewai-tools/src/crewai_tools/rag/loaders/docs_site_loader.pyindex c3d3007f7..f083712d7 100644--- a/lib/crewai-tools/src/crewai_tools/rag/loaders/docs_site_loader.py+++ b/lib/crewai-tools/src/crewai_tools/rag/loaders/docs_site_loader.py@@ -108,6 +108,7 @@ class DocsSiteLoader(BaseLoader):return LoaderResult(content=content,+ source=docs_url,metadata={"source": docs_url,"title": title_text,diff --git a/lib/crewai-tools/tests/rag/test_docs_site_loader.py b/lib/crewai-tools/tests/rag/test_docs_site_loader.pynew file mode 100644index 000000000..c6c544d0e--- /dev/null+++ b/lib/crewai-tools/tests/rag/test_docs_site_loader.py@@ -0,0 +1,34 @@+from unittest.mock import patch++from crewai_tools.rag.base_loader import BaseLoader+from crewai_tools.rag.loaders.docs_site_loader import DocsSiteLoader+from crewai_tools.rag.source_content import SourceContent+import pytest+from requests import Response+++ "docs_url",+ ["https://docs.example.com/guide", "https://example.com/docs/guide?version=2"],+)+def test_load_preserves_document_source(docs_url: str) -> None:+ """Keep the original URL available for document attribution and replacement."""+ response = Response()+ response.status_code = 200+ response.url = docs_url+ response.encoding = "utf-8"+ response._content = (+ b"<html><title>Guide</title><main><h1>Getting started</h1>"+ b"<p>Install the package.</p></main></html>"+ )++ with patch(+ "crewai_tools.rag.loaders.docs_site_loader.safe_get", return_value=response+ ) as mock_get:+ result = DocsSiteLoader().load(SourceContent(docs_url))++ mock_get.assert_called_once_with(docs_url, timeout=30)+ assert "Install the package." in result.content+ assert result.source == docs_url+ assert result.metadata["source"] == docs_url+ assert result.doc_id == BaseLoader.generate_doc_id(docs_url, result.content)