Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

test: adding new tests and updating existing ones

Karim shoaire6e5b3cc80d5
diff --git a/tests/ai/__init__.py b/tests/ai/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.py
new file mode 100644
index 0000000..92f5886
--- /dev/null
+++ b/tests/ai/test_ai_mcp.py
@@ -0,0 +1,89 @@
+import pytest
+from unittest.mock import Mock, patch
+
+from scrapling.core.ai import ScraplingMCPServer, ResponseModel
+
+
+class TestMCPServer:
+ """Test MCP server functionality"""
+
+ @pytest.fixture
+ def server(self):
+ return ScraplingMCPServer()
+
+ def test_server_creation(self, server):
+ """Test server instance creation"""
+ assert server._server is not None
+ assert server._server.name == "Scrapling"
+
+ def test_get_tool(self):
+ """Test the get tool method"""
+ with patch('scrapling.fetchers.Fetcher.get') as mock_get:
+ mock_response = Mock()
+ mock_response.status = 200
+ mock_response.url = "https://example.com"
+ mock_get.return_value = mock_response
+
+ with patch('scrapling.core.ai.Convertor._extract_content') as mock_extract:
+ mock_extract.return_value = iter(["Content"])
+
+ result = ScraplingMCPServer.get(
+ url="https://example.com",
+ extraction_type="markdown"
+ )
+
+ assert isinstance(result, ResponseModel)
+ assert result.status == 200
+ assert result.url == "https://example.com"
+
+ @pytest.mark.asyncio
+ async def test_bulk_get_tool(self):
+ """Test the bulk_get tool method"""
+ with patch('scrapling.engines.FetcherSession') as mock_session:
+ mock_instance = Mock()
+ mock_session.return_value.__aenter__.return_value = mock_instance
+
+ # Mock async get method
+ async def mock_async_get(*args, **kwargs):
+ mock_resp = Mock()
+ mock_resp.status = 200
+ mock_resp.url = args[0]
+ return mock_resp
+
+ mock_instance.get = mock_async_get
+
+ with patch('scrapling.core.ai.Convertor._extract_content') as mock_extract:
+ mock_extract.return_value = iter(["Content"])
+
+ results = await ScraplingMCPServer.bulk_get(
+ urls=("https://example1.com", "https://example2.com"),
+ extraction_type="html"
+ )
+
+ assert len(results) == 2
+ assert all(isinstance(r, ResponseModel) for r in results)
+
+ @pytest.mark.asyncio
+ async def test_fetch_tool(self):
+ """Test the fetch tool method"""
+ with patch('scrapling.fetchers.DynamicFetcher.async_fetch') as mock_fetch:
+ mock_response = Mock()
+ mock_response.status = 200
+ mock_response.url = "https://example.com"
+ mock_fetch.return_value = mock_response
+
+ with patch('scrapling.core.ai.Convertor._extract_content') as mock_extract:
+ mock_extract.return_value = iter(["Content"])
+
+ result = await ScraplingMCPServer.fetch(
+ url="https://example.com",
+ headless=True
+ )
+
+ assert isinstance(result, ResponseModel)
+
+ def test_serve_method(self, server):
+ """Test the serve method"""
+ with patch.object(server._server, 'run') as mock_run:
+ server.serve()
+ mock_run.assert_called_once_with(transport="stdio")
diff --git a/tests/fetchers/test_response_handling.py b/tests/fetchers/test_response_handling.py
new file mode 100644
index 0000000..7ff0f18
--- /dev/null
+++ b/tests/fetchers/test_response_handling.py
@@ -0,0 +1,109 @@
+from unittest.mock import Mock
+
+from scrapling.parser import Selector
+from scrapling.engines.toolbelt import ResponseFactory, Response
+from scrapling.engines.toolbelt.custom import ResponseEncoding
+
+
+class TestResponseFactory:
+ """Test ResponseFactory functionality"""
+
+ def test_response_from_curl(self):
+ """Test creating response from curl_cffi response"""
+ # Mock curl response
+ mock_curl_response = Mock()
+ mock_curl_response.url = "https://example.com"
+ mock_curl_response.content = b"<html><body>Test</body></html>"
+ mock_curl_response.status_code = 200
+ mock_curl_response.reason = "OK"
+ mock_curl_response.encoding = "utf-8"
+ mock_curl_response.cookies = {"session": "abc"}
+ mock_curl_response.headers = {"Content-Type": "text/html"}
+ mock_curl_response.request.headers = {"User-Agent": "Test"}
+ mock_curl_response.request.method = "GET"
+ mock_curl_response.history = []
+
+ response = ResponseFactory.from_http_request(
+ mock_curl_response,
+ {"adaptive": False}
+ )
+
+ assert response.status == 200
+ assert response.url == "https://example.com"
+ assert isinstance(response, Response)
+
+ def test_response_encoding_edge_cases(self):
+ """Test response encoding handling"""
+ # Test various content types
+ test_cases = [
+ (None, "utf-8"),
+ ("", "utf-8"),
+ ("text/html; charset=invalid", "utf-8"),
+ ("application/octet-stream", "utf-8"),
+ ]
+
+ for content_type, expected in test_cases:
+ encoding = ResponseEncoding.get_value(content_type)
+ assert encoding == expected
+
+ def test_response_history_processing(self):
+ """Test processing response history"""
+ # Mock responses with redirects
+ mock_final = Mock()
+ mock_final.status = 200
+ mock_final.status_text = "OK"
+ mock_final.all_headers = Mock(return_value={})
+
+ mock_redirect = Mock()
+ mock_redirect.url = "https://example.com/redirect"
+ mock_redirect.response = Mock(return_value=mock_final)
+ mock_redirect.all_headers = Mock(return_value={})
+ mock_redirect.redirected_from = None
+
+ mock_first = Mock()
+ mock_first.request.redirected_from = mock_redirect
+
+ # Process history
+ history = ResponseFactory._process_response_history(
+ mock_first,
+ {}
+ )
+
+ assert len(history) >= 0 # Should process redirects
+
+
+class TestErrorScenarios:
+ """Test various error scenarios"""
+
+ def test_invalid_html_handling(self):
+ """Test handling of malformed HTML"""
+ malformed_html = """
+ <html>
+ <body>
+ <div>Unclosed div
+ <p>Paragraph without closing tag
+ <span>Nested unclosed
+ </body>
+ """
+
+ # Should handle gracefully
+ page = Selector(malformed_html)
+ assert page is not None
+
+ # Should still be able to select elements
+ divs = page.css("div")
+ assert len(divs) > 0
+
+ def test_empty_responses(self):
+ """Test handling of empty responses"""
+ # Empty HTML
+ page = Selector("")
+ assert page is not None
+
+ # Whitespace only
+ page = Selector(" \n\t ")
+ assert page is not None
+
+ # Null bytes
+ page = Selector("Hello\x00World")
+ assert "Hello" in page.get_all_text()
diff --git a/tests/parser/test_attributes_handler.py b/tests/parser/test_attributes_handler.py
new file mode 100644
index 0000000..6827a95
--- /dev/null
+++ b/tests/parser/test_attributes_handler.py
@@ -0,0 +1,336 @@
+import pytest
+import json
+
+from scrapling import Selector
+from scrapling.core.custom_types import AttributesHandler
+
+
+class TestAttributesHandler:
+ """Test AttributesHandler functionality"""
+
+ @pytest.fixture
+ def sample_html(self):
+ return """
+ <html>
+ <body>
+ <div id="main"
+ class="container active"
+ data-config='{"theme": "dark", "version": 2.5}'
+ data-items='[1, 2, 3, 4, 5]'
+ data-invalid-json='{"broken: json}'
+ title="Main Container"
+ style="color: red; background: blue;"
+ data-empty=""
+ data-number="42"
+ data-bool="true"
+ data-url="https://example.com/page?param=value"
+ custom-attr="custom-value"
+ data-nested='{"user": {"name": "John", "age": 30}}'
+ data-encoded="&lt;div&gt;HTML&lt;/div&gt;"
+ onclick="handleClick()"
+ data-null="null"
+ data-undefined="undefined">
+ Content
+ </div>
+ <input type="text"
+ name="username"
+ placeholder="Enter email"
+ required
+ disabled>
+ <img src="/images/photo.jpg"
+ alt="Photo"
+ width="100"
+ height="100"
+ loading="lazy">
+ </body>
+ </html>
+ """
+
+ @pytest.fixture
+ def attributes(self, sample_html):
+ page = Selector(sample_html)
+ element = page.css("#main")[0]
+ return element.attrib
+
+ def test_basic_attribute_access(self, attributes):
+ """Test basic attribute access"""
+ # Dict-like access
+ assert attributes["id"] == "main"
+ assert attributes["class"] == "container active"
+ assert attributes["title"] == "Main Container"
+
+ # Key existence
+ assert "id" in attributes
+ assert "nonexistent" not in attributes
+
+ # Get with default
+ assert attributes.get("id") == "main"
+ assert attributes.get("nonexistent") is None
+ assert attributes.get("nonexistent", "default") == "default"
+
+ def test_iteration_methods(self, attributes):
+ """Test iteration over attributes"""
+ # Keys
+ keys = list(attributes.keys())
+ assert "id" in keys
+ assert "class" in keys
+ assert "data-config" in keys
+
+ # Values
+ values = list(attributes.values())
+ assert "main" in values
+ assert "container active" in values
+
+ # Items
+ items = dict(attributes.items())
+ assert items["id"] == "main"
+ assert items["class"] == "container active"
+
+ # Length
+ assert len(attributes) > 0
+
+ def test_json_parsing(self, attributes):
+ """Test JSON parsing from attributes"""
+ # Valid JSON object
+ config = attributes["data-config"].json()
+ assert config["theme"] == "dark"
+ assert config["version"] == 2.5
+
+ # Valid JSON array
+ items = attributes["data-items"].json()
+ assert items == [1, 2, 3, 4, 5]
+
+ # Nested JSON
+ nested = attributes["data-nested"].json()
+ assert nested["user"]["name"] == "John"
+ assert nested["user"]["age"] == 30
+
+ # JSON null
+ assert attributes["data-null"].json() is None
+
+ def test_json_error_handling(self, attributes):
+ """Test JSON parsing error handling"""
+ # Invalid JSON should raise error or return None
+ with pytest.raises((json.JSONDecodeError, AttributeError)):
+ attributes["data-invalid-json"].json()
+
+ # Non-existent attribute
+ with pytest.raises(KeyError):
+ attributes["nonexistent"].json()
+
+ def test_json_string_property(self, attributes):
+ """Test json_string property"""
+ # Should return JSON representation of all attributes
+ json_string = attributes.json_string
+ assert isinstance(json_string, bytes)
+
+ # Parse it back
+ parsed = json.loads(json_string)
+ assert parsed["id"] == "main"
+ assert parsed["class"] == "container active"
+
+ def test_search_values(self, attributes):
+ """Test search_values method"""
+ # Exact match
+ results = list(attributes.search_values("main", partial=False))
+ assert len(results) == 1
+ assert "id" in results[0]
+
+ # Partial match
+ results = list(attributes.search_values("container", partial=True))
+ assert len(results) >= 1
+ found_keys = []
+ for result in results:
+ found_keys.extend(result.keys())
+ assert "class" in found_keys or "title" in found_keys
+
+ # Case sensitivity
+ results = list(attributes.search_values("MAIN", partial=False))
+ assert len(results) == 0 # Should be case-sensitive by default
+
+ # Multiple matches
+ results = list(attributes.search_values("2", partial=True))
+ assert len(results) > 1 # Should find multiple attributes
+
+ # No matches
+ results = list(attributes.search_values("nonexistent", partial=False))
+ assert len(results) == 0
+
+ def test_special_attribute_types(self, sample_html):
+ """Test handling of special attribute types"""
+ page = Selector(sample_html)
+
+ # Boolean attributes
+ input_elem = page.css("input")[0]
+ assert "required" in input_elem.attrib
+ assert "disabled" in input_elem.attrib
+
+ # Empty attributes
+ main_elem = page.css("#main")[0]
+ assert main_elem.attrib["data-empty"] == ""
+
+ # Numeric string attributes
+ assert main_elem.attrib["data-number"] == "42"
+ assert main_elem.attrib["data-bool"] == "true"
+
+ def test_attribute_modification(self, sample_html):
+ """Test that AttributesHandler is read-only (if applicable)"""
+ page = Selector(sample_html)
+ element = page.css("#main")[0]
+ attrs = element.attrib
+
+ # Test if attributes can be modified
+ # This behavior depends on implementation
+ original_id = attrs["id"]
+ try:
+ attrs["id"] = "new-id"
+ # If modification is allowed
+ assert attrs["id"] == "new-id"
+ # Reset
+ attrs["id"] = original_id
+ except (TypeError, AttributeError):
+ # If modification is not allowed (read-only)
+ assert attrs["id"] == original_id
+
+ def test_string_representation(self, attributes):
+ """Test string representations"""
+ # __str__
+ str_repr = str(attributes)
+ assert isinstance(str_repr, str)
+ assert "id" in str_repr or "main" in str_repr
+
+ # __repr__
+ repr_str = repr(attributes)
+ assert isinstance(repr_str, str)
+
+ def test_edge_cases(self, sample_html):
+ """Test edge cases and special scenarios"""
+ page = Selector(sample_html)
+
+ # Element with no attributes
+ page_with_no_attrs = Selector("<div>Content</div>")
+ elem = page_with_no_attrs.css("div")[0]
+ assert len(elem.attrib) == 0
+ assert list(elem.attrib.keys()) == []
+ assert elem.attrib.get("any") is None
+
+ # Element with encoded content
+ main_elem = page.css("#main")[0]
+ encoded = main_elem.attrib["data-encoded"]
+ assert "<" in encoded # Should decode it
+
+ # Style attribute parsing
+ style = main_elem.attrib["style"]
+ assert "color: red" in style
+ assert "background: blue" in style
+
+ def test_url_attribute(self, attributes):
+ """Test URL attributes"""
+ url = attributes["data-url"]
+ assert url == "https://example.com/page?param=value"
+
+ # Could test URL joining if AttributesHandler supports it
+ # based on the parent element's base URL
+
+ def test_comparison_operations(self, sample_html):
+ """Test comparison operations if supported"""
+ page = Selector(sample_html)
+ elem1 = page.css("#main")[0]
+ elem2 = page.css("input")[0]
+
+ # Different elements should have different attributes
+ assert elem1.attrib != elem2.attrib
+
+ # The same element should have equal attributes
+ elem1_again = page.css("#main")[0]
+ assert elem1.attrib == elem1_again.attrib
+
+ def test_complex_search_patterns(self, attributes):
+ """Test complex search patterns"""
+ # Search for JSON-containing attributes
+ json_attrs = []
+ for key, value in attributes.items():
+ try:
+ if isinstance(value, str) and (value.startswith('{') or value.startswith('[')):
+ json.loads(value)
+ json_attrs.append(key)
+ except:
+ pass
+
+ assert "data-config" in json_attrs
+ assert "data-items" in json_attrs
+ assert "data-nested" in json_attrs
+
+ def test_attribute_filtering(self, attributes):
+ """Test filtering attributes by patterns"""
+ # Get all data-* attributes
+ data_attrs = {k: v for k, v in attributes.items() if k.startswith("data-")}
+ assert len(data_attrs) > 5
+ assert "data-config" in data_attrs
+ assert "data-items" in data_attrs
+
+ # Get all event handler attributes
+ event_attrs = {k: v for k, v in attributes.items() if k.startswith("on")}
+ assert "onclick" in event_attrs
+
+ def test_performance_with_many_attributes(self):
+ """Test performance with elements having many attributes"""
+ # Create an element with many attributes
+ attrs_list = [f'data-attr{i}="value{i}"' for i in range(100)]
+ html = f'<div id="test" {" ".join(attrs_list)}>Content</div>'
+
+ page = Selector(html)
+ element = page.css("#test")[0]
+ attribs = element.attrib
+
+ # Should handle many attributes efficiently
+ assert len(attribs) == 101 # id + 100 data attributes
+
+ # Search should still work efficiently
+ results = list(attribs.search_values("value50", partial=False))
+ assert len(results) == 1
+
+ def test_unicode_attributes(self):
+ """Test handling of Unicode in attributes"""
+ html = """
+ <div id="unicode-test"
+ data-emoji="😀🎉"
+ data-chinese="你好世界"
+ data-arabic="مرحبا بالعالم"
+ data-special="café naïve">
+ </div>
+ """
+
+ page = Selector(html)
+ attrs = page.css("#unicode-test")[0].attrib
+
+ assert attrs["data-emoji"] == "😀🎉"
+ assert attrs["data-chinese"] == "你好世界"
+ assert attrs["data-arabic"] == "مرحبا بالعالم"
+ assert attrs["data-special"] == "café naïve"
+
+ # Search with Unicode
+ results = list(attrs.search_values("你好", partial=True))
+ assert len(results) == 1
+
+ def test_malformed_attributes(self):
+ """Test handling of malformed attributes"""
+ # Various malformed HTML scenarios
+ test_cases = [
+ '<div id="test" class=>Content</div>', # Empty attribute value
+ '<div id="test" class>Content</div>', # No attribute value
+ '<div id="test" data-"invalid"="value">Content</div>', # Invalid attribute name
+ '<div id=test class=no-quotes>Content</div>', # Unquoted values
+ ]
+
+ for html in test_cases:
+ try:
+ page = Selector(html)
+ if page.css("div"):
+ attrs = page.css("div")[0].attrib
+ # Should handle gracefully without crashing
+ assert isinstance(attrs, AttributesHandler)
+ except:
+ # Some malformed HTML might not parse at all
+ pass
diff --git a/tests/parser/test_parser_advanced.py b/tests/parser/test_parser_advanced.py
new file mode 100644
index 0000000..aa9ea7a
--- /dev/null
+++ b/tests/parser/test_parser_advanced.py
@@ -0,0 +1,224 @@
+import re
+import pytest
+
+from scrapling import Selector, Selectors
+from scrapling.core.custom_types import TextHandler, TextHandlers
+
+
+class TestAdvancedSelectors:
+ """Test advanced selector functionality"""
+
+ @pytest.fixture
+ def complex_html(self):
+ return """
+ <html>
+ <body>
+ <div class="container" data-test='{"key": "value"}'>
+ <p>First paragraph</p>
+ <!-- Comment -->
+ <p>Second paragraph</p>
+ <![CDATA[Some CDATA content]]>
+ <div class="nested">
+ <span id="special">Special content</span>
+ <span>Regular content</span>
+ </div>
+ <table>
+ <tr><td>Cell 1</td><td>Cell 2</td></tr>
+ <tr><td>Cell 3</td><td>Cell 4</td></tr>
+ </table>
+ </div>
+ </body>
+ </html>
+ """
+
+ def test_comment_and_cdata_handling(self, complex_html):
+ """Test handling of comments and CDATA"""
+ # With comments/CDATA kept
+ page = Selector(
+ complex_html,
+ keep_comments=True,
+ keep_cdata=True
+ )
+ content = page.body
+ assert "Comment" in content
+ assert "CDATA" in content
+
+ # Without comments/CDATA
+ page = Selector(
+ complex_html,
+ keep_comments=False,
+ keep_cdata=False
+ )
+ content = page.body
+ assert "Comment" not in content
+
+ def test_advanced_xpath_variables(self, complex_html):
+ """Test XPath with variables"""
+ page = Selector(complex_html)
+
+ # Using XPath variables
+ cells = page.xpath(
+ "//td[text()=$cell_text]",
+ cell_text="Cell 1"
+ )
+ assert len(cells) == 1
+ assert cells[0].text == "Cell 1"
+
+ def test_pseudo_elements(self, complex_html):
+ """Test CSS pseudo-elements"""
+ page = Selector(complex_html)
+
+ # ::text pseudo-element
+ texts = page.css("p::text")
+ assert len(texts) == 2
+ assert isinstance(texts[0], TextHandler)
+
+ # ::attr() pseudo-element
+ attrs = page.css("div::attr(class)")
+ assert "container" in attrs
+
+ def test_complex_attribute_operations(self, complex_html):
+ """Test complex attribute handling"""
+ page = Selector(complex_html)
+ container = page.css(".container")[0]
+
+ # JSON in attributes
+ data = container.attrib["data-test"].json()
+ assert data["key"] == "value"
+
+ # Attribute searching
+ matches = list(container.attrib.search_values("container"))
+ assert len(matches) == 1
+
+ def test_url_joining(self):
+ """Test URL joining functionality"""
+ page = Selector("<html></html>", url="https://example.com/page")
+
+ # Relative URL
+ assert page.urljoin("../other") == "https://example.com/other"
+ assert page.urljoin("/absolute") == "https://example.com/absolute"
+ assert page.urljoin("relative") == "https://example.com/relative"
+
+ def test_find_operations_edge_cases(self, complex_html):
+ """Test edge cases in find operations"""
+ page = Selector(complex_html)
+
+ # Multiple argument types
+ _ = page.find_all(
+ "span",
+ ["div"],
+ {"class": "nested"},
+ lambda e: e.text != ""
+ )
+
+ # Regex pattern matching
+ pattern = re.compile(r"Cell \d+")
+ cells = page.find_all(pattern)
+ assert len(cells) == 4
+
+ def test_text_operations_edge_cases(self, complex_html):
+ """Test text operation edge cases"""
+ page = Selector(complex_html)
+
+ # get_all_text with a custom separator
+ text = page.get_all_text(separator=" | ", strip=True)
+ assert " | " in text
+
+ # Ignore specific tags
+ text = page.get_all_text(ignore_tags=("table",))
+ assert "Cell" not in text
+
+ # With empty values
+ text = page.get_all_text(valid_values=False)
+ assert text != ""
+
+
+class TestTextHandlerAdvanced:
+ """Test advanced TextHandler functionality"""
+
+ def test_text_handler_operations(self):
+ """Test various TextHandler operations"""
+ text = TextHandler(" Hello World ")
+
+ # All string methods should return TextHandler
+ assert isinstance(text.strip(), TextHandler)
+ assert isinstance(text.upper(), TextHandler)
+ assert isinstance(text.lower(), TextHandler)
+ assert isinstance(text.replace("World", "Python"), TextHandler)
+
+ # Custom methods
+ assert text.clean() == "Hello World"
+
+ # Sorting
+ text2 = TextHandler("dcba")
+ assert text2.sort() == "abcd"
+
+ def test_text_handler_regex(self):
+ """Test regex operations on TextHandler"""
+ text = TextHandler("Price: $10.99, Sale: $8.99")
+
+ # Basic regex
+ prices = text.re(r"\$[\d.]+")
+ assert len(prices) == 2
+ assert prices[0] == "$10.99"
+
+ # Case insensitive
+ text2 = TextHandler("HELLO hello HeLLo")
+ matches = text2.re(r"hello", case_sensitive=False)
+ assert len(matches) == 3
+
+ # Clean match
+ text3 = TextHandler(" He l lo ")
+ matches = text3.re(r"He l lo", clean_match=True, case_sensitive=False)
+ assert len(matches) == 1
+
+ def test_text_handlers_operations(self):
+ """Test TextHandlers list operations"""
+ handlers = TextHandlers([
+ TextHandler("First"),
+ TextHandler("Second"),
+ TextHandler("Third")
+ ])
+
+ # Slicing should return TextHandlers
+ assert isinstance(handlers[0:2], TextHandlers)
+
+ # Get methods
+ assert handlers.get() == "First"
+ assert handlers.get("default") == "First"
+ assert TextHandlers([]).get("default") == "default"
+
+
+class TestSelectorsAdvanced:
+ """Test advanced Selectors functionality"""
+
+ def test_selectors_filtering(self):
+ """Test filtering operations on Selectors"""
+ html = """
+ <div>
+ <p class="highlight">Important</p>
+ <p>Regular</p>
+ <p class="highlight">Also important</p>
+ </div>
+ """
+ page = Selector(html)
+ paragraphs = page.css("p")
+
+ # Filter by class
+ highlighted = paragraphs.filter(lambda p: p.has_class("highlight"))
+ assert len(highlighted) == 2
+
+ # Search for a specific element
+ found = paragraphs.search(lambda p: p.text == "Regular")
+ assert found is not None
+ assert found.text == "Regular"
+
+ def test_selectors_properties(self):
+ """Test Selectors properties"""
+ html = "<div><p>1</p><p>2</p><p>3</p></div>"
+ page = Selector(html)
+ paragraphs = page.css("p")
+
+ assert paragraphs.first.text == "1"
+ assert paragraphs.last.text == "3"
+ assert paragraphs.length == 3