Skip to content

The agent engineering platform.

community[patch]: add web loader tests (#26728)

ccurmef2285376a599
diff --git a/libs/community/tests/unit_tests/document_loaders/test_web_base.py b/libs/community/tests/unit_tests/document_loaders/test_web_base.py
index ea78900980..529c19b4c1 100644
--- a/libs/community/tests/unit_tests/document_loaders/test_web_base.py
+++ b/libs/community/tests/unit_tests/document_loaders/test_web_base.py
@@ -1,3 +1,7 @@
+from textwrap import dedent
+from typing import Any
+from unittest.mock import MagicMock, patch
+
import pytest as pytest
from langchain_community.document_loaders.web_base import WebBaseLoader
@@ -19,3 +23,62 @@ class TestWebBaseLoader:
assert web_base_loader.web_paths == ["https://www.example.com"]
web_base_loader = WebBaseLoader(web_path="https://www.example.com")
assert web_base_loader.web_paths == ["https://www.example.com"]
+
+
+@patch("langchain_community.document_loaders.web_base.requests.Session.get")
+def test_lazy_load(mock_get: Any) -> None:
+ import bs4
+
+ mock_response = MagicMock()
+ mock_response.text = "<html><body><p>Test content</p></body></html>"
+ mock_get.return_value = mock_response
+
+ loader = WebBaseLoader(web_paths=["https://www.example.com"])
+ results = list(loader.lazy_load())
+ mock_get.assert_called_with("https://www.example.com")
+ assert len(results) == 1
+ assert results[0].page_content == "Test content"
+
+ # Test bs4 kwargs
+ mock_html = dedent("""
+ <html>
+ <body>
+ <p>Test content</p>
+ <div class="special-class">This is a div with a special class</div>
+ </body>
+ </html>
+ """)
+ mock_response = MagicMock()
+ mock_response.text = mock_html
+ mock_get.return_value = mock_response
+
+ loader = WebBaseLoader(
+ web_paths=["https://www.example.com"],
+ bs_kwargs={"parse_only": bs4.SoupStrainer(class_="special-class")},
+ )
+ results = list(loader.lazy_load())
+ assert len(results) == 1
+ assert results[0].page_content == "This is a div with a special class"
+
+
+@patch("aiohttp.ClientSession.get")
+def test_aload(mock_get: Any) -> None:
+ async def mock_text() -> str:
+ return "<html><body><p>Test content</p></body></html>"
+
+ mock_response = MagicMock()
+ mock_response.text = mock_text
+ mock_get.return_value.__aenter__.return_value = mock_response
+
+ loader = WebBaseLoader(
+ web_paths=["https://www.example.com"],
+ header_template={"User-Agent": "test-user-agent"},
+ )
+ results = loader.aload()
+ assert len(results) == 1
+ assert results[0].page_content == "Test content"
+ mock_get.assert_called_with(
+ "https://www.example.com", headers={"User-Agent": "test-user-agent"}, cookies={}
+ )