🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
test: adding new tests and updating existing ones
Karim shoaire6e5b3cc80d5
diff --git a/tests/ai/__init__.py b/tests/ai/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.pynew file mode 100644index 0000000..92f5886--- /dev/null+++ b/tests/ai/test_ai_mcp.py@@ -0,0 +1,89 @@+import pytest+from unittest.mock import Mock, patch++from scrapling.core.ai import ScraplingMCPServer, ResponseModel+++class TestMCPServer:+ """Test MCP server functionality"""++ @pytest.fixture+ def server(self):+ return ScraplingMCPServer()++ def test_server_creation(self, server):+ """Test server instance creation"""+ assert server._server is not None+ assert server._server.name == "Scrapling"++ def test_get_tool(self):+ """Test the get tool method"""+ with patch('scrapling.fetchers.Fetcher.get') as mock_get:+ mock_response = Mock()+ mock_response.status = 200+ mock_response.url = "https://example.com"+ mock_get.return_value = mock_response++ with patch('scrapling.core.ai.Convertor._extract_content') as mock_extract:+ mock_extract.return_value = iter(["Content"])++ result = ScraplingMCPServer.get(+ url="https://example.com",+ extraction_type="markdown"+ )++ assert isinstance(result, ResponseModel)+ assert result.status == 200+ assert result.url == "https://example.com"++ @pytest.mark.asyncio+ async def test_bulk_get_tool(self):+ """Test the bulk_get tool method"""+ with patch('scrapling.engines.FetcherSession') as mock_session:+ mock_instance = Mock()+ mock_session.return_value.__aenter__.return_value = mock_instance++ # Mock async get method+ async def mock_async_get(*args, **kwargs):+ mock_resp = Mock()+ mock_resp.status = 200+ mock_resp.url = args[0]+ return mock_resp++ mock_instance.get = mock_async_get++ with patch('scrapling.core.ai.Convertor._extract_content') as mock_extract:+ mock_extract.return_value = iter(["Content"])++ results = await ScraplingMCPServer.bulk_get(+ urls=("https://example1.com", "https://example2.com"),+ extraction_type="html"+ )++ assert len(results) == 2+ assert all(isinstance(r, ResponseModel) for r in results)++ @pytest.mark.asyncio+ async def test_fetch_tool(self):+ """Test the fetch tool method"""+ with patch('scrapling.fetchers.DynamicFetcher.async_fetch') as mock_fetch:+ mock_response = Mock()+ mock_response.status = 200+ mock_response.url = "https://example.com"+ mock_fetch.return_value = mock_response++ with patch('scrapling.core.ai.Convertor._extract_content') as mock_extract:+ mock_extract.return_value = iter(["Content"])++ result = await ScraplingMCPServer.fetch(+ url="https://example.com",+ headless=True+ )++ assert isinstance(result, ResponseModel)++ def test_serve_method(self, server):+ """Test the serve method"""+ with patch.object(server._server, 'run') as mock_run:+ server.serve()+ mock_run.assert_called_once_with(transport="stdio")diff --git a/tests/fetchers/test_response_handling.py b/tests/fetchers/test_response_handling.pynew file mode 100644index 0000000..7ff0f18--- /dev/null+++ b/tests/fetchers/test_response_handling.py@@ -0,0 +1,109 @@+from unittest.mock import Mock++from scrapling.parser import Selector+from scrapling.engines.toolbelt import ResponseFactory, Response+from scrapling.engines.toolbelt.custom import ResponseEncoding+++class TestResponseFactory:+ """Test ResponseFactory functionality"""++ def test_response_from_curl(self):+ """Test creating response from curl_cffi response"""+ # Mock curl response+ mock_curl_response = Mock()+ mock_curl_response.url = "https://example.com"+ mock_curl_response.content = b"<html><body>Test</body></html>"+ mock_curl_response.status_code = 200+ mock_curl_response.reason = "OK"+ mock_curl_response.encoding = "utf-8"+ mock_curl_response.cookies = {"session": "abc"}+ mock_curl_response.headers = {"Content-Type": "text/html"}+ mock_curl_response.request.headers = {"User-Agent": "Test"}+ mock_curl_response.request.method = "GET"+ mock_curl_response.history = []++ response = ResponseFactory.from_http_request(+ mock_curl_response,+ {"adaptive": False}+ )++ assert response.status == 200+ assert response.url == "https://example.com"+ assert isinstance(response, Response)++ def test_response_encoding_edge_cases(self):+ """Test response encoding handling"""+ # Test various content types+ test_cases = [+ (None, "utf-8"),+ ("", "utf-8"),+ ("text/html; charset=invalid", "utf-8"),+ ("application/octet-stream", "utf-8"),+ ]++ for content_type, expected in test_cases:+ encoding = ResponseEncoding.get_value(content_type)+ assert encoding == expected++ def test_response_history_processing(self):+ """Test processing response history"""+ # Mock responses with redirects+ mock_final = Mock()+ mock_final.status = 200+ mock_final.status_text = "OK"+ mock_final.all_headers = Mock(return_value={})++ mock_redirect = Mock()+ mock_redirect.url = "https://example.com/redirect"+ mock_redirect.response = Mock(return_value=mock_final)+ mock_redirect.all_headers = Mock(return_value={})+ mock_redirect.redirected_from = None++ mock_first = Mock()+ mock_first.request.redirected_from = mock_redirect++ # Process history+ history = ResponseFactory._process_response_history(+ mock_first,+ {}+ )++ assert len(history) >= 0 # Should process redirects+++class TestErrorScenarios:+ """Test various error scenarios"""++ def test_invalid_html_handling(self):+ """Test handling of malformed HTML"""+ malformed_html = """+ <html>+ <body>+ <div>Unclosed div+ <p>Paragraph without closing tag+ <span>Nested unclosed+ </body>+ """++ # Should handle gracefully+ page = Selector(malformed_html)+ assert page is not None++ # Should still be able to select elements+ divs = page.css("div")+ assert len(divs) > 0++ def test_empty_responses(self):+ """Test handling of empty responses"""+ # Empty HTML+ page = Selector("")+ assert page is not None++ # Whitespace only+ page = Selector(" \n\t ")+ assert page is not None++ # Null bytes+ page = Selector("Hello\x00World")+ assert "Hello" in page.get_all_text()diff --git a/tests/parser/test_attributes_handler.py b/tests/parser/test_attributes_handler.pynew file mode 100644index 0000000..6827a95--- /dev/null+++ b/tests/parser/test_attributes_handler.py@@ -0,0 +1,336 @@+import pytest+import json++from scrapling import Selector+from scrapling.core.custom_types import AttributesHandler+++class TestAttributesHandler:+ """Test AttributesHandler functionality"""++ @pytest.fixture+ def sample_html(self):+ return """+ <html>+ <body>+ <div id="main"+ class="container active"+ data-config='{"theme": "dark", "version": 2.5}'+ data-items='[1, 2, 3, 4, 5]'+ data-invalid-json='{"broken: json}'+ title="Main Container"+ style="color: red; background: blue;"+ data-empty=""+ data-number="42"+ data-bool="true"+ data-url="https://example.com/page?param=value"+ custom-attr="custom-value"+ data-nested='{"user": {"name": "John", "age": 30}}'+ data-encoded="<div>HTML</div>"+ onclick="handleClick()"+ data-null="null"+ data-undefined="undefined">+ Content+ </div>+ <input type="text"+ name="username"+ value="[email protected]"+ placeholder="Enter email"+ required+ disabled>+ <img src="/images/photo.jpg"+ alt="Photo"+ width="100"+ height="100"+ loading="lazy">+ </body>+ </html>+ """++ @pytest.fixture+ def attributes(self, sample_html):+ page = Selector(sample_html)+ element = page.css("#main")[0]+ return element.attrib++ def test_basic_attribute_access(self, attributes):+ """Test basic attribute access"""+ # Dict-like access+ assert attributes["id"] == "main"+ assert attributes["class"] == "container active"+ assert attributes["title"] == "Main Container"++ # Key existence+ assert "id" in attributes+ assert "nonexistent" not in attributes++ # Get with default+ assert attributes.get("id") == "main"+ assert attributes.get("nonexistent") is None+ assert attributes.get("nonexistent", "default") == "default"++ def test_iteration_methods(self, attributes):+ """Test iteration over attributes"""+ # Keys+ keys = list(attributes.keys())+ assert "id" in keys+ assert "class" in keys+ assert "data-config" in keys++ # Values+ values = list(attributes.values())+ assert "main" in values+ assert "container active" in values++ # Items+ items = dict(attributes.items())+ assert items["id"] == "main"+ assert items["class"] == "container active"++ # Length+ assert len(attributes) > 0++ def test_json_parsing(self, attributes):+ """Test JSON parsing from attributes"""+ # Valid JSON object+ config = attributes["data-config"].json()+ assert config["theme"] == "dark"+ assert config["version"] == 2.5++ # Valid JSON array+ items = attributes["data-items"].json()+ assert items == [1, 2, 3, 4, 5]++ # Nested JSON+ nested = attributes["data-nested"].json()+ assert nested["user"]["name"] == "John"+ assert nested["user"]["age"] == 30++ # JSON null+ assert attributes["data-null"].json() is None++ def test_json_error_handling(self, attributes):+ """Test JSON parsing error handling"""+ # Invalid JSON should raise error or return None+ with pytest.raises((json.JSONDecodeError, AttributeError)):+ attributes["data-invalid-json"].json()++ # Non-existent attribute+ with pytest.raises(KeyError):+ attributes["nonexistent"].json()++ def test_json_string_property(self, attributes):+ """Test json_string property"""+ # Should return JSON representation of all attributes+ json_string = attributes.json_string+ assert isinstance(json_string, bytes)++ # Parse it back+ parsed = json.loads(json_string)+ assert parsed["id"] == "main"+ assert parsed["class"] == "container active"++ def test_search_values(self, attributes):+ """Test search_values method"""+ # Exact match+ results = list(attributes.search_values("main", partial=False))+ assert len(results) == 1+ assert "id" in results[0]++ # Partial match+ results = list(attributes.search_values("container", partial=True))+ assert len(results) >= 1+ found_keys = []+ for result in results:+ found_keys.extend(result.keys())+ assert "class" in found_keys or "title" in found_keys++ # Case sensitivity+ results = list(attributes.search_values("MAIN", partial=False))+ assert len(results) == 0 # Should be case-sensitive by default++ # Multiple matches+ results = list(attributes.search_values("2", partial=True))+ assert len(results) > 1 # Should find multiple attributes++ # No matches+ results = list(attributes.search_values("nonexistent", partial=False))+ assert len(results) == 0++ def test_special_attribute_types(self, sample_html):+ """Test handling of special attribute types"""+ page = Selector(sample_html)++ # Boolean attributes+ input_elem = page.css("input")[0]+ assert "required" in input_elem.attrib+ assert "disabled" in input_elem.attrib++ # Empty attributes+ main_elem = page.css("#main")[0]+ assert main_elem.attrib["data-empty"] == ""++ # Numeric string attributes+ assert main_elem.attrib["data-number"] == "42"+ assert main_elem.attrib["data-bool"] == "true"++ def test_attribute_modification(self, sample_html):+ """Test that AttributesHandler is read-only (if applicable)"""+ page = Selector(sample_html)+ element = page.css("#main")[0]+ attrs = element.attrib++ # Test if attributes can be modified+ # This behavior depends on implementation+ original_id = attrs["id"]+ try:+ attrs["id"] = "new-id"+ # If modification is allowed+ assert attrs["id"] == "new-id"+ # Reset+ attrs["id"] = original_id+ except (TypeError, AttributeError):+ # If modification is not allowed (read-only)+ assert attrs["id"] == original_id++ def test_string_representation(self, attributes):+ """Test string representations"""+ # __str__+ str_repr = str(attributes)+ assert isinstance(str_repr, str)+ assert "id" in str_repr or "main" in str_repr++ # __repr__+ repr_str = repr(attributes)+ assert isinstance(repr_str, str)++ def test_edge_cases(self, sample_html):+ """Test edge cases and special scenarios"""+ page = Selector(sample_html)++ # Element with no attributes+ page_with_no_attrs = Selector("<div>Content</div>")+ elem = page_with_no_attrs.css("div")[0]+ assert len(elem.attrib) == 0+ assert list(elem.attrib.keys()) == []+ assert elem.attrib.get("any") is None++ # Element with encoded content+ main_elem = page.css("#main")[0]+ encoded = main_elem.attrib["data-encoded"]+ assert "<" in encoded # Should decode it++ # Style attribute parsing+ style = main_elem.attrib["style"]+ assert "color: red" in style+ assert "background: blue" in style++ def test_url_attribute(self, attributes):+ """Test URL attributes"""+ url = attributes["data-url"]+ assert url == "https://example.com/page?param=value"++ # Could test URL joining if AttributesHandler supports it+ # based on the parent element's base URL++ def test_comparison_operations(self, sample_html):+ """Test comparison operations if supported"""+ page = Selector(sample_html)+ elem1 = page.css("#main")[0]+ elem2 = page.css("input")[0]++ # Different elements should have different attributes+ assert elem1.attrib != elem2.attrib++ # The same element should have equal attributes+ elem1_again = page.css("#main")[0]+ assert elem1.attrib == elem1_again.attrib++ def test_complex_search_patterns(self, attributes):+ """Test complex search patterns"""+ # Search for JSON-containing attributes+ json_attrs = []+ for key, value in attributes.items():+ try:+ if isinstance(value, str) and (value.startswith('{') or value.startswith('[')):+ json.loads(value)+ json_attrs.append(key)+ except:+ pass++ assert "data-config" in json_attrs+ assert "data-items" in json_attrs+ assert "data-nested" in json_attrs++ def test_attribute_filtering(self, attributes):+ """Test filtering attributes by patterns"""+ # Get all data-* attributes+ data_attrs = {k: v for k, v in attributes.items() if k.startswith("data-")}+ assert len(data_attrs) > 5+ assert "data-config" in data_attrs+ assert "data-items" in data_attrs++ # Get all event handler attributes+ event_attrs = {k: v for k, v in attributes.items() if k.startswith("on")}+ assert "onclick" in event_attrs++ def test_performance_with_many_attributes(self):+ """Test performance with elements having many attributes"""+ # Create an element with many attributes+ attrs_list = [f'data-attr{i}="value{i}"' for i in range(100)]+ html = f'<div id="test" {" ".join(attrs_list)}>Content</div>'++ page = Selector(html)+ element = page.css("#test")[0]+ attribs = element.attrib++ # Should handle many attributes efficiently+ assert len(attribs) == 101 # id + 100 data attributes++ # Search should still work efficiently+ results = list(attribs.search_values("value50", partial=False))+ assert len(results) == 1++ def test_unicode_attributes(self):+ """Test handling of Unicode in attributes"""+ html = """+ <div id="unicode-test"+ data-emoji="😀🎉"+ data-chinese="你好世界"+ data-arabic="مرحبا بالعالم"+ data-special="café naïve">+ </div>+ """++ page = Selector(html)+ attrs = page.css("#unicode-test")[0].attrib++ assert attrs["data-emoji"] == "😀🎉"+ assert attrs["data-chinese"] == "你好世界"+ assert attrs["data-arabic"] == "مرحبا بالعالم"+ assert attrs["data-special"] == "café naïve"++ # Search with Unicode+ results = list(attrs.search_values("你好", partial=True))+ assert len(results) == 1++ def test_malformed_attributes(self):+ """Test handling of malformed attributes"""+ # Various malformed HTML scenarios+ test_cases = [+ '<div id="test" class=>Content</div>', # Empty attribute value+ '<div id="test" class>Content</div>', # No attribute value+ '<div id="test" data-"invalid"="value">Content</div>', # Invalid attribute name+ '<div id=test class=no-quotes>Content</div>', # Unquoted values+ ]++ for html in test_cases:+ try:+ page = Selector(html)+ if page.css("div"):+ attrs = page.css("div")[0].attrib+ # Should handle gracefully without crashing+ assert isinstance(attrs, AttributesHandler)+ except:+ # Some malformed HTML might not parse at all+ passdiff --git a/tests/parser/test_parser_advanced.py b/tests/parser/test_parser_advanced.pynew file mode 100644index 0000000..aa9ea7a--- /dev/null+++ b/tests/parser/test_parser_advanced.py@@ -0,0 +1,224 @@+import re+import pytest++from scrapling import Selector, Selectors+from scrapling.core.custom_types import TextHandler, TextHandlers+++class TestAdvancedSelectors:+ """Test advanced selector functionality"""++ @pytest.fixture+ def complex_html(self):+ return """+ <html>+ <body>+ <div class="container" data-test='{"key": "value"}'>+ <p>First paragraph</p>+ <!-- Comment -->+ <p>Second paragraph</p>+ <![CDATA[Some CDATA content]]>+ <div class="nested">+ <span id="special">Special content</span>+ <span>Regular content</span>+ </div>+ <table>+ <tr><td>Cell 1</td><td>Cell 2</td></tr>+ <tr><td>Cell 3</td><td>Cell 4</td></tr>+ </table>+ </div>+ </body>+ </html>+ """++ def test_comment_and_cdata_handling(self, complex_html):+ """Test handling of comments and CDATA"""+ # With comments/CDATA kept+ page = Selector(+ complex_html,+ keep_comments=True,+ keep_cdata=True+ )+ content = page.body+ assert "Comment" in content+ assert "CDATA" in content++ # Without comments/CDATA+ page = Selector(+ complex_html,+ keep_comments=False,+ keep_cdata=False+ )+ content = page.body+ assert "Comment" not in content++ def test_advanced_xpath_variables(self, complex_html):+ """Test XPath with variables"""+ page = Selector(complex_html)++ # Using XPath variables+ cells = page.xpath(+ "//td[text()=$cell_text]",+ cell_text="Cell 1"+ )+ assert len(cells) == 1+ assert cells[0].text == "Cell 1"++ def test_pseudo_elements(self, complex_html):+ """Test CSS pseudo-elements"""+ page = Selector(complex_html)++ # ::text pseudo-element+ texts = page.css("p::text")+ assert len(texts) == 2+ assert isinstance(texts[0], TextHandler)++ # ::attr() pseudo-element+ attrs = page.css("div::attr(class)")+ assert "container" in attrs++ def test_complex_attribute_operations(self, complex_html):+ """Test complex attribute handling"""+ page = Selector(complex_html)+ container = page.css(".container")[0]++ # JSON in attributes+ data = container.attrib["data-test"].json()+ assert data["key"] == "value"++ # Attribute searching+ matches = list(container.attrib.search_values("container"))+ assert len(matches) == 1++ def test_url_joining(self):+ """Test URL joining functionality"""+ page = Selector("<html></html>", url="https://example.com/page")++ # Relative URL+ assert page.urljoin("../other") == "https://example.com/other"+ assert page.urljoin("/absolute") == "https://example.com/absolute"+ assert page.urljoin("relative") == "https://example.com/relative"++ def test_find_operations_edge_cases(self, complex_html):+ """Test edge cases in find operations"""+ page = Selector(complex_html)++ # Multiple argument types+ _ = page.find_all(+ "span",+ ["div"],+ {"class": "nested"},+ lambda e: e.text != ""+ )++ # Regex pattern matching+ pattern = re.compile(r"Cell \d+")+ cells = page.find_all(pattern)+ assert len(cells) == 4++ def test_text_operations_edge_cases(self, complex_html):+ """Test text operation edge cases"""+ page = Selector(complex_html)++ # get_all_text with a custom separator+ text = page.get_all_text(separator=" | ", strip=True)+ assert " | " in text++ # Ignore specific tags+ text = page.get_all_text(ignore_tags=("table",))+ assert "Cell" not in text++ # With empty values+ text = page.get_all_text(valid_values=False)+ assert text != ""+++class TestTextHandlerAdvanced:+ """Test advanced TextHandler functionality"""++ def test_text_handler_operations(self):+ """Test various TextHandler operations"""+ text = TextHandler(" Hello World ")++ # All string methods should return TextHandler+ assert isinstance(text.strip(), TextHandler)+ assert isinstance(text.upper(), TextHandler)+ assert isinstance(text.lower(), TextHandler)+ assert isinstance(text.replace("World", "Python"), TextHandler)++ # Custom methods+ assert text.clean() == "Hello World"++ # Sorting+ text2 = TextHandler("dcba")+ assert text2.sort() == "abcd"++ def test_text_handler_regex(self):+ """Test regex operations on TextHandler"""+ text = TextHandler("Price: $10.99, Sale: $8.99")++ # Basic regex+ prices = text.re(r"\$[\d.]+")+ assert len(prices) == 2+ assert prices[0] == "$10.99"++ # Case insensitive+ text2 = TextHandler("HELLO hello HeLLo")+ matches = text2.re(r"hello", case_sensitive=False)+ assert len(matches) == 3++ # Clean match+ text3 = TextHandler(" He l lo ")+ matches = text3.re(r"He l lo", clean_match=True, case_sensitive=False)+ assert len(matches) == 1++ def test_text_handlers_operations(self):+ """Test TextHandlers list operations"""+ handlers = TextHandlers([+ TextHandler("First"),+ TextHandler("Second"),+ TextHandler("Third")+ ])++ # Slicing should return TextHandlers+ assert isinstance(handlers[0:2], TextHandlers)++ # Get methods+ assert handlers.get() == "First"+ assert handlers.get("default") == "First"+ assert TextHandlers([]).get("default") == "default"+++class TestSelectorsAdvanced:+ """Test advanced Selectors functionality"""++ def test_selectors_filtering(self):+ """Test filtering operations on Selectors"""+ html = """+ <div>+ <p class="highlight">Important</p>+ <p>Regular</p>+ <p class="highlight">Also important</p>+ </div>+ """+ page = Selector(html)+ paragraphs = page.css("p")++ # Filter by class+ highlighted = paragraphs.filter(lambda p: p.has_class("highlight"))+ assert len(highlighted) == 2++ # Search for a specific element+ found = paragraphs.search(lambda p: p.text == "Regular")+ assert found is not None+ assert found.text == "Regular"++ def test_selectors_properties(self):+ """Test Selectors properties"""+ html = "<div><p>1</p><p>2</p><p>3</p></div>"+ page = Selector(html)+ paragraphs = page.css("p")++ assert paragraphs.first.text == "1"+ assert paragraphs.last.text == "3"+ assert paragraphs.length == 3