The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
community[patch]: add web loader tests (#26728)
ccurmef2285376a599
diff --git a/libs/community/tests/unit_tests/document_loaders/test_web_base.py b/libs/community/tests/unit_tests/document_loaders/test_web_base.pyindex ea78900980..529c19b4c1 100644--- a/libs/community/tests/unit_tests/document_loaders/test_web_base.py+++ b/libs/community/tests/unit_tests/document_loaders/test_web_base.py@@ -1,3 +1,7 @@+from textwrap import dedent+from typing import Any+from unittest.mock import MagicMock, patch+import pytest as pytestfrom langchain_community.document_loaders.web_base import WebBaseLoader@@ -19,3 +23,62 @@ class TestWebBaseLoader:assert web_base_loader.web_paths == ["https://www.example.com"]web_base_loader = WebBaseLoader(web_path="https://www.example.com")assert web_base_loader.web_paths == ["https://www.example.com"]++[email protected]("bs4")+@patch("langchain_community.document_loaders.web_base.requests.Session.get")+def test_lazy_load(mock_get: Any) -> None:+ import bs4++ mock_response = MagicMock()+ mock_response.text = "<html><body><p>Test content</p></body></html>"+ mock_get.return_value = mock_response++ loader = WebBaseLoader(web_paths=["https://www.example.com"])+ results = list(loader.lazy_load())+ mock_get.assert_called_with("https://www.example.com")+ assert len(results) == 1+ assert results[0].page_content == "Test content"++ # Test bs4 kwargs+ mock_html = dedent("""+ <html>+ <body>+ <p>Test content</p>+ <div class="special-class">This is a div with a special class</div>+ </body>+ </html>+ """)+ mock_response = MagicMock()+ mock_response.text = mock_html+ mock_get.return_value = mock_response++ loader = WebBaseLoader(+ web_paths=["https://www.example.com"],+ bs_kwargs={"parse_only": bs4.SoupStrainer(class_="special-class")},+ )+ results = list(loader.lazy_load())+ assert len(results) == 1+ assert results[0].page_content == "This is a div with a special class"++[email protected]("bs4")+@patch("aiohttp.ClientSession.get")+def test_aload(mock_get: Any) -> None:+ async def mock_text() -> str:+ return "<html><body><p>Test content</p></body></html>"++ mock_response = MagicMock()+ mock_response.text = mock_text+ mock_get.return_value.__aenter__.return_value = mock_response++ loader = WebBaseLoader(+ web_paths=["https://www.example.com"],+ header_template={"User-Agent": "test-user-agent"},+ )+ results = loader.aload()+ assert len(results) == 1+ assert results[0].page_content == "Test content"+ mock_get.assert_called_with(+ "https://www.example.com", headers={"User-Agent": "test-user-agent"}, cookies={}+ )