🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
tests: Multiple changes to tests
Karim shoaireae088a07b7e
diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.pyindex 8acf28f..cae9fa0 100644--- a/tests/cli/test_cli.py+++ b/tests/cli/test_cli.py@@ -14,6 +14,7 @@ def configure_selector_mock():"""Helper function to create a properly configured Selector mock"""mock_response = MagicMock(spec=Selector)mock_response.body = "<html><body>Test content</body></html>"+ mock_response.encoding = "utf-8"mock_response.get_all_text.return_value = "Test content"mock_response.css_first.return_value = mock_responsemock_response.css.return_value = [mock_response]diff --git a/tests/fetchers/async/test_camoufox.py b/tests/fetchers/async/test_camoufox.pyindex 6a0700b..ffe6eac 100644--- a/tests/fetchers/async/test_camoufox.py+++ b/tests/fetchers/async/test_camoufox.py@@ -1,3 +1,4 @@+from playwright._impl._errors import TimeoutErrorimport pytestimport pytest_httpbin@@ -23,14 +24,9 @@ class TestStealthyFetcher:"basic_url": f"{url}/get","html_url": f"{url}/html","delayed_url": f"{url}/delay/10", # 10 Seconds delay response- "cookies_url": f"{url}/cookies/set/test/value",- "cloudflare_url": "https://nopecha.com/demo/cloudflare", # Interactive turnstile page+ "cookies_url": f"{url}/cookies/set/test/value"}- async def test_cloudflare_fetch(self, fetcher, urls):- """Test if Cloudflare bypass is working"""- assert (await fetcher.async_fetch(urls["cloudflare_url"], solve_cloudflare=True)).status == 200-async def test_basic_fetch(self, fetcher, urls):"""Test doing a basic fetch request with multiple statuses"""assert (await fetcher.async_fetch(urls["status_200"])).status == 200@@ -86,9 +82,3 @@ class TestStealthyFetcher:**kwargs)assert response.status == 200-- async def test_infinite_timeout(self, fetcher, urls):- """Test if infinite timeout breaks the code or not"""- assert (- await fetcher.async_fetch(urls["delayed_url"], timeout=0)- ).status == 200diff --git a/tests/fetchers/async/test_dynamic.py b/tests/fetchers/async/test_dynamic.pyindex 0d171ce..04106fa 100644--- a/tests/fetchers/async/test_dynamic.py+++ b/tests/fetchers/async/test_dynamic.py@@ -90,9 +90,3 @@ class TestDynamicFetcherAsync:with pytest.raises(Exception):await fetcher.async_fetch(urls["html_url"], cdp_url="ws://blahblah")-- @pytest.mark.asyncio- async def test_infinite_timeout(self, fetcher, urls):- """Test if infinite timeout breaks the code or not"""- response = await fetcher.async_fetch(urls["delayed_url"], timeout=0)- assert response.status == 200diff --git a/tests/fetchers/sync/test_camoufox.py b/tests/fetchers/sync/test_camoufox.pyindex 5a9209d..83c5f6c 100644--- a/tests/fetchers/sync/test_camoufox.py+++ b/tests/fetchers/sync/test_camoufox.py@@ -77,7 +77,3 @@ class TestStealthyFetcher:**kwargs)assert response.status == 200-- def test_infinite_timeout(self, fetcher):- """Test if infinite timeout breaks the code or not"""- assert fetcher.fetch(self.delayed_url, timeout=0).status == 200diff --git a/tests/fetchers/test_response_handling.py b/tests/fetchers/test_response_handling.pyindex 4db96bf..1327db8 100644--- a/tests/fetchers/test_response_handling.py+++ b/tests/fetchers/test_response_handling.py@@ -1,7 +1,6 @@from unittest.mock import Mockfrom scrapling.parser import Selector-from scrapling.engines.toolbelt.custom import ResponseEncodingfrom scrapling.engines.toolbelt.convertor import ResponseFactory, Response@@ -32,20 +31,6 @@ class TestResponseFactory:assert response.url == "https://example.com"assert isinstance(response, Response)- def test_response_encoding_edge_cases(self):- """Test response encoding handling"""- # Test various content types- test_cases = [- (None, "utf-8"),- ("", "utf-8"),- ("text/html; charset=invalid", "utf-8"),- ("application/octet-stream", "utf-8"),- ]-- for content_type, expected in test_cases:- encoding = ResponseEncoding.get_value(content_type)- assert encoding == expected-def test_response_history_processing(self):"""Test processing response history"""# Mock responses with redirectsdiff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.pyindex 4a1563c..0028902 100644--- a/tests/fetchers/test_utils.py+++ b/tests/fetchers/test_utils.py@@ -1,7 +1,7 @@import pytestfrom pathlib import Path-from scrapling.engines.toolbelt.custom import ResponseEncoding, StatusText, Response+from scrapling.engines.toolbelt.custom import StatusText, Responsefrom scrapling.engines.toolbelt.navigation import (construct_proxy_dict,js_bypass_path@@ -131,12 +131,6 @@ def status_map():}-def test_parsing_content_type(content_type_map):- """Test if parsing different types of 'content-type' returns the expected result"""- for header_value, expected_encoding in content_type_map.items():- assert ResponseEncoding.get_value(header_value) == expected_encoding--def test_parsing_response_status(status_map):"""Test if using different http responses' status codes returns the expected result"""for status_code, expected_status_text in status_map.items():diff --git a/tests/parser/test_parser_advanced.py b/tests/parser/test_parser_advanced.pyindex 71552f9..3ac81cf 100644--- a/tests/parser/test_parser_advanced.py+++ b/tests/parser/test_parser_advanced.py@@ -99,7 +99,7 @@ class TestAdvancedSelectors:keep_comments=False,keep_cdata=False)- content = page.body+ content = page.html_contentassert "Comment" not in contentdef test_advanced_xpath_variables(self, complex_html):