Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

fix(rag): preserve documentation site source URLs (#7951)

Jialiang Liangcbb500d3b827
diff --git a/lib/crewai-tools/src/crewai_tools/rag/loaders/docs_site_loader.py b/lib/crewai-tools/src/crewai_tools/rag/loaders/docs_site_loader.py
index c3d3007f7..f083712d7 100644
--- a/lib/crewai-tools/src/crewai_tools/rag/loaders/docs_site_loader.py
+++ b/lib/crewai-tools/src/crewai_tools/rag/loaders/docs_site_loader.py
@@ -108,6 +108,7 @@ class DocsSiteLoader(BaseLoader):
return LoaderResult(
content=content,
+ source=docs_url,
metadata={
"source": docs_url,
"title": title_text,
diff --git a/lib/crewai-tools/tests/rag/test_docs_site_loader.py b/lib/crewai-tools/tests/rag/test_docs_site_loader.py
new file mode 100644
index 000000000..c6c544d0e
--- /dev/null
+++ b/lib/crewai-tools/tests/rag/test_docs_site_loader.py
@@ -0,0 +1,34 @@
+from unittest.mock import patch
+
+from crewai_tools.rag.base_loader import BaseLoader
+from crewai_tools.rag.loaders.docs_site_loader import DocsSiteLoader
+from crewai_tools.rag.source_content import SourceContent
+import pytest
+from requests import Response
+
+
+ "docs_url",
+ ["https://docs.example.com/guide", "https://example.com/docs/guide?version=2"],
+)
+def test_load_preserves_document_source(docs_url: str) -> None:
+ """Keep the original URL available for document attribution and replacement."""
+ response = Response()
+ response.status_code = 200
+ response.url = docs_url
+ response.encoding = "utf-8"
+ response._content = (
+ b"<html><title>Guide</title><main><h1>Getting started</h1>"
+ b"<p>Install the package.</p></main></html>"
+ )
+
+ with patch(
+ "crewai_tools.rag.loaders.docs_site_loader.safe_get", return_value=response
+ ) as mock_get:
+ result = DocsSiteLoader().load(SourceContent(docs_url))
+
+ mock_get.assert_called_once_with(docs_url, timeout=30)
+ assert "Install the package." in result.content
+ assert result.source == docs_url
+ assert result.metadata["source"] == docs_url
+ assert result.doc_id == BaseLoader.generate_doc_id(docs_url, result.content)