🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
test: adding new tests and updating existing ones
Karim shoairba81890daf94
diff --git a/tests/fetchers/sync/test_requests_session.py b/tests/fetchers/sync/test_requests_session.pynew file mode 100644index 0000000..1009c2f--- /dev/null+++ b/tests/fetchers/sync/test_requests_session.py@@ -0,0 +1,56 @@+import pytest+++from scrapling.engines.static import FetcherSession, FetcherClient, AsyncFetcherClient+++class TestFetcherSession:+ """Test FetcherSession functionality"""++ def test_fetcher_session_creation(self):+ """Test FetcherSession creation"""+ session = FetcherSession(+ timeout=30,+ retries=3,+ stealthy_headers=True+ )++ assert session.default_timeout == 30+ assert session.default_retries == 3+ assert session.stealth is True++ def test_fetcher_session_context_manager(self):+ """Test FetcherSession as a context manager"""+ session = FetcherSession()++ with session as s:+ assert s == session+ assert session._curl_session is not None++ # Session should be cleaned up++ def test_fetcher_session_double_enter(self):+ """Test error on double entering"""+ session = FetcherSession()++ with session:+ with pytest.raises(RuntimeError):+ session.__enter__()++ def test_fetcher_client_creation(self):+ """Test FetcherClient creation"""+ client = FetcherClient()++ # Should not have context manager methods+ assert client.__enter__ is None+ assert client.__exit__ is None+ assert client._curl_session is True # Special marker++ def test_async_fetcher_client_creation(self):+ """Test AsyncFetcherClient creation"""+ client = AsyncFetcherClient()++ # Should not have context manager methods+ assert client.__aenter__ is None+ assert client.__aexit__ is None+ assert client._async_curl_session is True # Special markerdiff --git a/tests/fetchers/test_base.py b/tests/fetchers/test_base.pynew file mode 100644index 0000000..32db09d--- /dev/null+++ b/tests/fetchers/test_base.py@@ -0,0 +1,84 @@+import pytest++from scrapling.engines.toolbelt.custom import BaseFetcher+++class TestBaseFetcher:+ """Test BaseFetcher configuration functionality"""++ def test_default_configuration(self):+ """Test default configuration values"""+ config = BaseFetcher.display_config()++ assert config['huge_tree'] is True+ assert config['adaptive'] is False+ assert config['keep_comments'] is False+ assert config['keep_cdata'] is False++ def test_configure_single_parameter(self):+ """Test configuring single parameter"""+ BaseFetcher.configure(adaptive=True)++ config = BaseFetcher.display_config()+ assert config['adaptive'] is True++ # Reset+ BaseFetcher.configure(adaptive=False)++ def test_configure_multiple_parameters(self):+ """Test configuring multiple parameters"""+ BaseFetcher.configure(+ huge_tree=False,+ keep_comments=True,+ adaptive=True+ )++ config = BaseFetcher.display_config()+ assert config['huge_tree'] is False+ assert config['keep_comments'] is True+ assert config['adaptive'] is True++ # Reset+ BaseFetcher.configure(+ huge_tree=True,+ keep_comments=False,+ adaptive=False+ )++ def test_configure_invalid_parameter(self):+ """Test configuring invalid parameter"""+ with pytest.raises(ValueError):+ BaseFetcher.configure(invalid_param=True)++ def test_configure_no_parameters(self):+ """Test configure with no parameters"""+ with pytest.raises(AttributeError):+ BaseFetcher.configure()++ def test_configure_non_parser_keyword(self):+ """Test configuring non-parser keyword"""+ with pytest.raises(AttributeError):+ # Assuming there's some attribute that's not in parser_keywords+ BaseFetcher.some_other_attr = "test"+ BaseFetcher.configure(some_other_attr="new_value")++ def test_generate_parser_arguments(self):+ """Test parser arguments generation"""+ BaseFetcher.configure(+ huge_tree=False,+ adaptive=True,+ adaptive_domain="example.com"+ )++ args = BaseFetcher._generate_parser_arguments()++ assert args['huge_tree'] is False+ assert args['adaptive'] is True+ assert args['adaptive_domain'] == "example.com"++ # Reset+ BaseFetcher.configure(+ huge_tree=True,+ adaptive=False+ )+ BaseFetcher.adaptive_domain = Nonediff --git a/tests/fetchers/test_constants.py b/tests/fetchers/test_constants.pynew file mode 100644index 0000000..b6aee29--- /dev/null+++ b/tests/fetchers/test_constants.py@@ -0,0 +1,27 @@+from scrapling.engines.constants import (+ DEFAULT_DISABLED_RESOURCES,+ DEFAULT_STEALTH_FLAGS,+ HARMFUL_DEFAULT_ARGS+)+++class TestConstants:+ """Test constant values"""++ def test_default_disabled_resources(self):+ """Test default disabled resources"""+ assert "image" in DEFAULT_DISABLED_RESOURCES+ assert "font" in DEFAULT_DISABLED_RESOURCES+ assert "stylesheet" in DEFAULT_DISABLED_RESOURCES+ assert "media" in DEFAULT_DISABLED_RESOURCES++ def test_harmful_default_args(self):+ """Test harmful default arguments"""+ assert "--enable-automation" in HARMFUL_DEFAULT_ARGS+ assert "--disable-popup-blocking" in HARMFUL_DEFAULT_ARGS++ def test_default_stealth_flags(self):+ """Test default stealth flags"""+ assert "--no-pings" in DEFAULT_STEALTH_FLAGS+ assert "--incognito" in DEFAULT_STEALTH_FLAGS+ assert "--disable-blink-features=AutomationControlled" in DEFAULT_STEALTH_FLAGSdiff --git a/tests/fetchers/test_pages.py b/tests/fetchers/test_pages.pynew file mode 100644index 0000000..fe85bd3--- /dev/null+++ b/tests/fetchers/test_pages.py@@ -0,0 +1,154 @@+import pytest+from unittest.mock import Mock+from scrapling.engines._browsers._page import PageInfo, PagePool+++class TestPageInfo:+ """Test PageInfo functionality"""++ def test_page_info_creation(self):+ """Test PageInfo creation"""+ mock_page = Mock()+ page_info = PageInfo(mock_page, "ready", "https://example.com")++ assert page_info.page == mock_page+ assert page_info.state == "ready"+ assert page_info.url == "https://example.com"++ def test_page_info_marking(self):+ """Test marking page"""+ mock_page = Mock()+ page_info = PageInfo(mock_page, "ready", None)++ page_info.mark_busy("https://example.com")+ assert page_info.state == "busy"+ assert page_info.url == "https://example.com"++ page_info.mark_ready()+ assert page_info.state == "ready"+ assert page_info.url == ""++ page_info.mark_error()+ assert page_info.state == "error"++ def test_page_info_equality(self):+ """Test PageInfo equality comparison"""+ mock_page1 = Mock()+ mock_page2 = Mock()++ page_info1 = PageInfo(mock_page1, "ready", None)+ page_info2 = PageInfo(mock_page1, "busy", None) # Same page, different state+ page_info3 = PageInfo(mock_page2, "ready", None) # Different page++ assert page_info1 == page_info2 # Same page+ assert page_info1 != page_info3 # Different page+ assert page_info1 != "not a page info" # Different type++ def test_page_info_repr(self):+ """Test PageInfo string representation"""+ mock_page = Mock()+ page_info = PageInfo(mock_page, "ready", "https://example.com")++ repr_str = repr(page_info)+ assert "ready" in repr_str+ assert "https://example.com" in repr_str+++class TestPagePool:+ """Test PagePool functionality"""++ def test_page_pool_creation(self):+ """Test PagePool creation"""+ pool = PagePool(max_pages=5)++ assert pool.max_pages == 5+ assert pool.pages_count == 0+ assert pool.ready_count == 0+ assert pool.busy_count == 0++ def test_add_page(self):+ """Test adding page to pool"""+ pool = PagePool(max_pages=2)+ mock_page = Mock()++ page_info = pool.add_page(mock_page)++ assert isinstance(page_info, PageInfo)+ assert page_info.page == mock_page+ assert page_info.state == "ready"+ assert pool.pages_count == 1++ def test_add_page_limit_exceeded(self):+ """Test adding page when limit exceeded"""+ pool = PagePool(max_pages=1)++ # Add first page+ pool.add_page(Mock())++ # Try to add a second page+ with pytest.raises(RuntimeError):+ pool.add_page(Mock())++ def test_get_ready_page(self):+ """Test getting ready page"""+ pool = PagePool(max_pages=3)++ # Add pages+ page1 = pool.add_page(Mock())+ page2 = pool.add_page(Mock())++ # Mark one as busy+ page1.mark_busy("https://example.com")++ # Should get the ready page+ ready_page = pool.get_ready_page()+ assert ready_page == page2++ def test_get_ready_page_none_available(self):+ """Test getting ready page when none available"""+ pool = PagePool(max_pages=2)++ # Add pages and mark all as busy+ page1 = pool.add_page(Mock())+ page2 = pool.add_page(Mock())+ page1.mark_busy("https://example1.com")+ page2.mark_busy("https://example2.com")++ # Should return None+ ready_page = pool.get_ready_page()+ assert ready_page is None++ def test_page_counts(self):+ """Test page count properties"""+ pool = PagePool(max_pages=3)++ # Add pages with different states+ page1 = pool.add_page(Mock())+ page2 = pool.add_page(Mock())+ page3 = pool.add_page(Mock())++ page1.mark_busy("https://example.com")+ page3.mark_error()++ assert pool.pages_count == 3+ assert pool.ready_count == 1+ assert pool.busy_count == 1++ def test_cleanup_error_pages(self):+ """Test cleaning up error pages"""+ pool = PagePool(max_pages=3)++ # Add pages+ page1 = pool.add_page(Mock())+ page2 = pool.add_page(Mock())+ page3 = pool.add_page(Mock())++ # Mark some as error+ page1.mark_error()+ page3.mark_error()++ assert pool.pages_count == 3++ pool.cleanup_error_pages()++ assert pool.pages_count == 1 # Only page2 should remaindiff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.pyindex de4450b..b787d36 100644--- a/tests/fetchers/test_utils.py+++ b/tests/fetchers/test_utils.py@@ -1,6 +1,17 @@import pytest+from pathlib import Path-from scrapling.engines.toolbelt.custom import ResponseEncoding, StatusText+from scrapling.engines.toolbelt.custom import ResponseEncoding, StatusText, Response+from scrapling.engines.toolbelt.navigation import (+ construct_proxy_dict,+ construct_cdp_url,+ js_bypass_path+)+from scrapling.engines.toolbelt.fingerprints import (+ generate_convincing_referer,+ get_os_name,+ generate_headers+)@pytest.fixture@@ -122,7 +133,7 @@ def status_map():def test_parsing_content_type(content_type_map):- """Test if parsing different types of content-type returns the expected result"""+ """Test if parsing different types of 'content-type' returns the expected result"""for header_value, expected_encoding in content_type_map.items():assert ResponseEncoding.get_value(header_value) == expected_encoding@@ -136,3 +147,208 @@ def test_parsing_response_status(status_map):def test_unknown_status_code():"""Test handling of an unknown status code"""assert StatusText.get(1000) == "Unknown Status Code"+++class TestConstructProxyDict:+ """Test proxy dictionary construction"""++ def test_proxy_string_basic(self):+ """Test a basic proxy string"""+ result = construct_proxy_dict("http://proxy.example.com:8080")++ expected = {+ "server": "http://proxy.example.com:8080",+ "username": "",+ "password": ""+ }+ assert result == expected++ def test_proxy_string_with_auth(self):+ """Test proxy string with authentication"""+ result = construct_proxy_dict("http://user:[email protected]:8080")++ expected = {+ "server": "http://proxy.example.com:8080",+ "username": "user",+ "password": "pass"+ }+ assert result == expected++ def test_proxy_dict_input(self):+ """Test proxy dictionary input"""+ input_dict = {+ "server": "http://proxy.example.com:8080",+ "username": "user",+ "password": "pass"+ }+ result = construct_proxy_dict(input_dict)++ assert result == input_dict++ def test_proxy_dict_minimal(self):+ """Test minimal proxy dictionary"""+ input_dict = {"server": "http://proxy.example.com:8080"}+ result = construct_proxy_dict(input_dict)++ expected = {+ "server": "http://proxy.example.com:8080",+ "username": "",+ "password": ""+ }+ assert result == expected++ def test_proxy_as_tuple(self):+ """Test returning proxy as a tuple"""+ result = construct_proxy_dict("http://proxy.example.com:8080", as_tuple=True)++ assert isinstance(result, tuple)+ result_dict = dict(result)+ assert result_dict["server"] == "http://proxy.example.com:8080"++ def test_invalid_proxy_string(self):+ """Test invalid proxy string"""+ with pytest.raises(ValueError):+ construct_proxy_dict("invalid-proxy-format")++ def test_invalid_proxy_dict(self):+ """Test invalid proxy dictionary"""+ with pytest.raises(TypeError):+ construct_proxy_dict({"invalid": "structure"})+++class TestConstructCdpUrl:+ """Test CDP URL construction"""++ def test_basic_cdp_url(self):+ """Test basic CDP URL"""+ result = construct_cdp_url("ws://localhost:9222/devtools/browser")+ assert result == "ws://localhost:9222/devtools/browser"++ def test_cdp_url_with_params(self):+ """Test CDP URL with query parameters"""+ params = {"timeout": "30000", "headless": "true"}+ result = construct_cdp_url("ws://localhost:9222/devtools/browser", params)++ assert "timeout=30000" in result+ assert "headless=true" in result++ def test_cdp_url_without_leading_slash(self):+ """Test CDP URL without a leading slash in the path"""+ with pytest.raises(ValueError):+ construct_cdp_url("ws://localhost:9222devtools/browser")++ def test_invalid_cdp_scheme(self):+ """Test invalid CDP URL scheme"""+ with pytest.raises(ValueError):+ construct_cdp_url("http://localhost:9222/devtools/browser")++ def test_invalid_cdp_netloc(self):+ """Test invalid CDP URL network location"""+ with pytest.raises(ValueError):+ construct_cdp_url("ws:///devtools/browser")++ def test_malformed_cdp_url(self):+ """Test malformed CDP URL"""+ with pytest.raises(ValueError):+ construct_cdp_url("not-a-url")+++class TestJsBypassPath:+ """Test JavaScript bypass path utility"""++ def test_js_bypass_path(self):+ """Test getting JavaScript bypass file path"""+ result = js_bypass_path("webdriver_fully.js")++ assert isinstance(result, str)+ assert result.endswith("webdriver_fully.js")+ assert Path(result).exists()++ def test_js_bypass_path_caching(self):+ """Test that js_bypass_path is cached"""+ result1 = js_bypass_path("webdriver_fully.js")+ result2 = js_bypass_path("webdriver_fully.js")++ assert result1 == result2+++class TestFingerprintFunctions:+ """Test fingerprint generation functions"""++ def test_generate_convincing_referer(self):+ """Test referer generation"""+ url = "https://sub.example.com/page.html"+ result = generate_convincing_referer(url)++ assert result.startswith("https://www.google.com/search?q=")+ assert "example" in result++ def test_generate_convincing_referer_caching(self):+ """Test referer generation caching"""+ url = "https://example.com"+ result1 = generate_convincing_referer(url)+ result2 = generate_convincing_referer(url)++ assert result1 == result2++ def test_get_os_name(self):+ """Test OS name detection"""+ result = get_os_name()++ # Should return one of the known OS names or None+ valid_names = ["linux", "macos", "windows", "ios"]+ assert result is None or result in valid_names++ def test_generate_headers_basic(self):+ """Test basic header generation"""+ headers = generate_headers()++ assert isinstance(headers, dict)+ assert "User-Agent" in headers+ assert len(headers["User-Agent"]) > 0++ def test_generate_headers_browser_mode(self):+ """Test header generation in browser mode"""+ headers = generate_headers(browser_mode=True)++ assert isinstance(headers, dict)+ assert "User-Agent" in headers+++class TestResponse:+ """Test Response class functionality"""++ def test_response_creation(self):+ """Test Response object creation"""+ response = Response(+ url="https://example.com",+ content="<html><body>Test</body></html>",+ status=200,+ reason="OK",+ cookies={"session": "abc123"},+ headers={"Content-Type": "text/html"},+ request_headers={"User-Agent": "Test"},+ encoding="utf-8"+ )++ assert response.url == "https://example.com"+ assert response.status == 200+ assert response.reason == "OK"+ assert response.cookies == {"session": "abc123"}++ def test_response_with_bytes_content(self):+ """Test Response with 'bytes' content"""+ content_bytes = "<html><body>Test</body></html>".encode('utf-8')++ response = Response(+ url="https://example.com",+ content=content_bytes,+ status=200,+ reason="OK",+ cookies={},+ headers={},+ request_headers={}+ )++ # Should handle 'bytes' content properly+ assert response.status == 200diff --git a/tests/fetchers/test_validator.py b/tests/fetchers/test_validator.pynew file mode 100644index 0000000..118554c--- /dev/null+++ b/tests/fetchers/test_validator.py@@ -0,0 +1,79 @@+import pytest+from scrapling.engines._browsers._validators import (+ validate,+ PlaywrightConfig,+ CamoufoxConfig+)+++class TestValidators:+ """Test configuration validators"""++ def test_playwright_config_valid(self):+ """Test valid PlaywrightConfig"""+ params = {+ "max_pages": 2,+ "headless": True,+ "timeout": 30000,+ "proxy": "http://proxy.example.com:8080"+ }++ config = validate(params, PlaywrightConfig)++ assert config.max_pages == 2+ assert config.headless is True+ assert config.timeout == 30000+ assert isinstance(config.proxy, tuple) # Should be converted to tuple++ def test_playwright_config_invalid_max_pages(self):+ """Test PlaywrightConfig with invalid max_pages"""+ params = {"max_pages": 0}++ with pytest.raises(TypeError):+ validate(params, PlaywrightConfig)++ params = {"max_pages": 51}++ with pytest.raises(TypeError):+ validate(params, PlaywrightConfig)++ def test_playwright_config_invalid_timeout(self):+ """Test PlaywrightConfig with an invalid timeout"""+ params = {"timeout": -1}++ with pytest.raises(TypeError):+ validate(params, PlaywrightConfig)++ def test_playwright_config_invalid_cdp_url(self):+ """Test PlaywrightConfig with invalid CDP URL"""+ params = {"cdp_url": "invalid-url"}++ with pytest.raises(TypeError):+ validate(params, PlaywrightConfig)++ def test_camoufox_config_valid(self):+ """Test valid CamoufoxConfig"""+ params = {+ "max_pages": 1,+ "headless": True,+ "solve_cloudflare": False,+ "timeout": 30000+ }++ config = validate(params, CamoufoxConfig)++ assert config.max_pages == 1+ assert config.headless is True+ assert config.solve_cloudflare is False+ assert config.timeout == 30000++ def test_camoufox_config_cloudflare_timeout(self):+ """Test CamoufoxConfig timeout adjustment for Cloudflare"""+ params = {+ "solve_cloudflare": True,+ "timeout": 10000 # Less than the required 60,000+ }++ config = validate(params, CamoufoxConfig)++ assert config.timeout == 60000 # Should be increased