🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
test: add tests for the new feature
Karim shoair5f557a7f9f0e
diff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.pyindex fea3f3f..9c08f7f 100644--- a/tests/fetchers/test_utils.py+++ b/tests/fetchers/test_utils.py@@ -4,8 +4,11 @@ from pathlib import Pathfrom scrapling.engines.toolbelt.custom import StatusText, Responsefrom scrapling.engines.toolbelt.navigation import (construct_proxy_dict,- js_bypass_path+ create_intercept_handler,+ create_async_intercept_handler,+ js_bypass_path,)+from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCESfrom scrapling.engines.toolbelt.fingerprints import (generate_convincing_referer,get_os_name,@@ -300,3 +303,150 @@ class TestResponse:# Should handle 'bytes' content properlyassert response.status == 200+++class _MockRequest:+ """Minimal mock for Playwright's Request object."""+ def __init__(self, url: str, resource_type: str = "document"):+ self.url = url+ self.resource_type = resource_type+++class _MockRoute:+ """Minimal mock for Playwright's sync Route object."""+ def __init__(self, url: str, resource_type: str = "document"):+ self.request = _MockRequest(url, resource_type)+ self.aborted = False+ self.continued = False++ def abort(self):+ self.aborted = True++ def continue_(self):+ self.continued = True+++class _AsyncMockRoute:+ """Minimal mock for Playwright's async Route object."""+ def __init__(self, url: str, resource_type: str = "document"):+ self.request = _MockRequest(url, resource_type)+ self.aborted = False+ self.continued = False++ async def abort(self):+ self.aborted = True++ async def continue_(self):+ self.continued = True+++class TestCreateInterceptHandler:+ """Test the unified sync route handler factory."""++ def test_blocks_disabled_resource_types(self):+ handler = create_intercept_handler(disable_resources=True)+ route = _MockRoute("https://example.com/image.png", resource_type="image")+ handler(route)+ assert route.aborted++ def test_continues_allowed_resource_types(self):+ handler = create_intercept_handler(disable_resources=True)+ route = _MockRoute("https://example.com/page", resource_type="document")+ handler(route)+ assert route.continued++ def test_blocks_exact_domain(self):+ handler = create_intercept_handler(disable_resources=False, blocked_domains={"ads.example.com"})+ route = _MockRoute("https://ads.example.com/tracker.js")+ handler(route)+ assert route.aborted++ def test_blocks_subdomain(self):+ handler = create_intercept_handler(disable_resources=False, blocked_domains={"example.com"})+ route = _MockRoute("https://sub.example.com/page")+ handler(route)+ assert route.aborted++ def test_continues_non_blocked_domain(self):+ handler = create_intercept_handler(disable_resources=False, blocked_domains={"ads.example.com"})+ route = _MockRoute("https://safe.example.com/page")+ handler(route)+ assert route.continued++ def test_resource_blocking_takes_priority_over_domain(self):+ """When both are active, resource type check comes first."""+ handler = create_intercept_handler(disable_resources=True, blocked_domains={"example.com"})+ route = _MockRoute("https://example.com/style.css", resource_type="stylesheet")+ handler(route)+ assert route.aborted++ def test_domain_blocking_with_resources_disabled(self):+ """Non-blocked resource type from a blocked domain should still be aborted."""+ handler = create_intercept_handler(disable_resources=True, blocked_domains={"tracker.io"})+ route = _MockRoute("https://tracker.io/api", resource_type="document")+ handler(route)+ assert route.aborted++ def test_no_blocking_continues(self):+ handler = create_intercept_handler(disable_resources=False)+ route = _MockRoute("https://example.com/page")+ handler(route)+ assert route.continued++ def test_does_not_block_partial_domain_match(self):+ """'example.com' should not block 'notexample.com'."""+ handler = create_intercept_handler(disable_resources=False, blocked_domains={"example.com"})+ route = _MockRoute("https://notexample.com/page")+ handler(route)+ assert route.continued++ def test_multiple_blocked_domains(self):+ handler = create_intercept_handler(disable_resources=False, blocked_domains={"ads.com", "tracker.io"})+ route_ads = _MockRoute("https://ads.com/banner")+ route_tracker = _MockRoute("https://cdn.tracker.io/script.js")+ route_safe = _MockRoute("https://example.com/page")+ handler(route_ads)+ handler(route_tracker)+ handler(route_safe)+ assert route_ads.aborted+ assert route_tracker.aborted+ assert route_safe.continued+++class TestCreateAsyncInterceptHandler:+ """Test the unified async route handler factory."""++ @pytest.mark.asyncio+ async def test_blocks_disabled_resource_types(self):+ handler = create_async_intercept_handler(disable_resources=True)+ route = _AsyncMockRoute("https://example.com/font.woff", resource_type="font")+ await handler(route)+ assert route.aborted++ @pytest.mark.asyncio+ async def test_blocks_domain(self):+ handler = create_async_intercept_handler(disable_resources=False, blocked_domains={"ads.example.com"})+ route = _AsyncMockRoute("https://ads.example.com/track")+ await handler(route)+ assert route.aborted++ @pytest.mark.asyncio+ async def test_continues_non_blocked(self):+ handler = create_async_intercept_handler(disable_resources=False, blocked_domains={"ads.example.com"})+ route = _AsyncMockRoute("https://safe.example.com/page")+ await handler(route)+ assert route.continued++ @pytest.mark.asyncio+ async def test_blocks_subdomain(self):+ handler = create_async_intercept_handler(disable_resources=False, blocked_domains={"tracker.io"})+ route = _AsyncMockRoute("https://cdn.tracker.io/script.js")+ await handler(route)+ assert route.aborted++ @pytest.mark.asyncio+ async def test_does_not_block_partial_domain_match(self):+ handler = create_async_intercept_handler(disable_resources=False, blocked_domains={"example.com"})+ route = _AsyncMockRoute("https://notexample.com/page")+ await handler(route)+ assert route.continueddiff --git a/tests/fetchers/test_validator.py b/tests/fetchers/test_validator.pyindex 209fae0..a682f9e 100644--- a/tests/fetchers/test_validator.py+++ b/tests/fetchers/test_validator.py@@ -77,3 +77,25 @@ class TestValidators:config = validate(params, StealthConfig)assert config.timeout == 60000 # Should be increased++ def test_playwright_config_blocked_domains(self):+ """Test PlaywrightConfig with blocked_domains"""+ params = {"blocked_domains": {"ads.example.com", "tracker.io"}}++ config = validate(params, PlaywrightConfig)++ assert config.blocked_domains == {"ads.example.com", "tracker.io"}++ def test_playwright_config_blocked_domains_default_none(self):+ """Test PlaywrightConfig blocked_domains defaults to None"""+ config = validate({}, PlaywrightConfig)++ assert config.blocked_domains is None++ def test_stealth_config_blocked_domains(self):+ """Test StealthConfig inherits blocked_domains"""+ params = {"blocked_domains": {"ads.example.com"}}++ config = validate(params, StealthConfig)++ assert config.blocked_domains == {"ads.example.com"}