Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(browsers): Add option to block requests to specific domains

Karim shoair66476d42be02
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py
index 01bced5..8bd442d 100644
--- a/scrapling/engines/_browsers/_base.py
+++ b/scrapling/engines/_browsers/_base.py
@@ -24,11 +24,16 @@ from scrapling.parser import Selector
from scrapling.engines._browsers._page import PageInfo, PagePool
from scrapling.engines._browsers._validators import validate, PlaywrightConfig, StealthConfig
from scrapling.engines._browsers._config_tools import __default_chrome_useragent__, __default_useragent__
-from scrapling.engines.toolbelt.navigation import construct_proxy_dict, intercept_route, async_intercept_route
+from scrapling.engines.toolbelt.navigation import (
+ construct_proxy_dict,
+ create_intercept_handler,
+ create_async_intercept_handler,
+)
from scrapling.core._types import (
Any,
Dict,
List,
+ Set,
Optional,
Callable,
TYPE_CHECKING,
@@ -105,6 +110,7 @@ class SyncSession:
timeout: int | float,
extra_headers: Optional[Dict[str, str]],
disable_resources: bool,
+ blocked_domains: Optional[Set[str]] = None,
context: Optional[BrowserContext] = None,
) -> PageInfo[Page]: # pragma: no cover
"""Get a new page to use"""
@@ -117,9 +123,8 @@ class SyncSession:
if extra_headers:
page.set_extra_http_headers(extra_headers)
- if disable_resources:
- page.route("**/*", intercept_route)
-
+ if disable_resources or blocked_domains:
+ page.route("**/*", create_intercept_handler(disable_resources, blocked_domains))
page_info = self.page_pool.add_page(page)
page_info.mark_busy()
return page_info
@@ -173,6 +178,7 @@ class SyncSession:
extra_headers: Optional[Dict[str, str]],
disable_resources: bool,
proxy: Optional[ProxyType] = None,
+ blocked_domains: Optional[Set[str]] = None,
) -> Generator["PageInfo[Page]", None, None]:
"""Acquire a page - either from persistent context or fresh context with proxy."""
if proxy:
@@ -184,13 +190,13 @@ class SyncSession:
try:
context = self._initialize_context(self._config, context)
- page_info = self._get_page(timeout, extra_headers, disable_resources, context=context)
+ page_info = self._get_page(timeout, extra_headers, disable_resources, blocked_domains, context=context)
yield page_info
finally:
context.close()
else:
# Standard mode: use PagePool with persistent context
- page_info = self._get_page(timeout, extra_headers, disable_resources)
+ page_info = self._get_page(timeout, extra_headers, disable_resources, blocked_domains)
try:
yield page_info
finally:
@@ -261,6 +267,7 @@ class AsyncSession:
timeout: int | float,
extra_headers: Optional[Dict[str, str]],
disable_resources: bool,
+ blocked_domains: Optional[Set[str]] = None,
context: Optional[AsyncBrowserContext] = None,
) -> PageInfo[AsyncPage]: # pragma: no cover
"""Get a new page to use"""
@@ -288,8 +295,8 @@ class AsyncSession:
if extra_headers:
await page.set_extra_http_headers(extra_headers)
- if disable_resources:
- await page.route("**/*", async_intercept_route)
+ if disable_resources or blocked_domains:
+ await page.route("**/*", create_async_intercept_handler(disable_resources, blocked_domains))
return self.page_pool.add_page(page)
@@ -342,6 +349,7 @@ class AsyncSession:
extra_headers: Optional[Dict[str, str]],
disable_resources: bool,
proxy: Optional[ProxyType] = None,
+ blocked_domains: Optional[Set[str]] = None,
) -> AsyncGenerator["PageInfo[AsyncPage]", None]:
"""Acquire a page - either from persistent context or fresh context with proxy."""
if proxy:
@@ -353,13 +361,15 @@ class AsyncSession:
try:
context = await self._initialize_context(self._config, context)
- page_info = await self._get_page(timeout, extra_headers, disable_resources, context=context)
+ page_info = await self._get_page(
+ timeout, extra_headers, disable_resources, blocked_domains, context=context
+ )
yield page_info
finally:
await context.close()
else:
# Standard mode: use PagePool with persistent context
- page_info = await self._get_page(timeout, extra_headers, disable_resources)
+ page_info = await self._get_page(timeout, extra_headers, disable_resources, blocked_domains)
try:
yield page_info
finally:
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index b43807c..5f0b247 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -43,6 +43,7 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+ :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
@@ -110,6 +111,7 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+ :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
@@ -138,7 +140,7 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
proxy = static_proxy
with self._page_generator(
- params.timeout, params.extra_headers, params.disable_resources, proxy
+ params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
) as page_info:
final_response = [None]
page = page_info.page
@@ -208,6 +210,7 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+ :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
@@ -277,6 +280,7 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+ :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
@@ -306,7 +310,7 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
proxy = static_proxy
async with self._page_generator(
- params.timeout, params.extra_headers, params.disable_resources, proxy
+ params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
) as page_info:
final_response = [None]
page = page_info.page
diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py
index 5564376..730cd7e 100644
--- a/scrapling/engines/_browsers/_stealth.py
+++ b/scrapling/engines/_browsers/_stealth.py
@@ -47,6 +47,7 @@ class StealthySession(SyncSession, StealthySessionMixin):
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+ :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
@@ -198,6 +199,7 @@ class StealthySession(SyncSession, StealthySessionMixin):
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+ :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
@@ -227,7 +229,7 @@ class StealthySession(SyncSession, StealthySessionMixin):
proxy = static_proxy
with self._page_generator(
- params.timeout, params.extra_headers, params.disable_resources, proxy
+ params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
) as page_info:
final_response = [None]
page = page_info.page
@@ -302,6 +304,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+ :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
@@ -454,6 +457,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+ :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
@@ -484,7 +488,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
proxy = static_proxy
async with self._page_generator(
- params.timeout, params.extra_headers, params.disable_resources, proxy
+ params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
) as page_info:
final_response = [None]
page = page_info.page
diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py
index d298248..cd5ea19 100644
--- a/scrapling/engines/_browsers/_types.py
+++ b/scrapling/engines/_browsers/_types.py
@@ -9,6 +9,7 @@ from curl_cffi.requests import (
from scrapling.core._types import (
Dict,
List,
+ Set,
Tuple,
Mapping,
Optional,
@@ -84,6 +85,7 @@ if TYPE_CHECKING: # pragma: no cover
cdp_url: Optional[str]
useragent: Optional[str]
extra_flags: Optional[List[str]]
+ blocked_domains: Optional[Set[str]]
retries: int
retry_delay: int | float
@@ -99,6 +101,7 @@ if TYPE_CHECKING: # pragma: no cover
selector_config: Optional[Dict]
extra_headers: Optional[Dict[str, str]]
wait_selector_state: SelectorWaitStates
+ blocked_domains: Optional[Set[str]]
proxy: Optional[str | Dict[str, str]]
class StealthSession(PlaywrightSession, total=False):
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index 2bda436..af2baef 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -10,6 +10,7 @@ from scrapling.core._types import (
Any,
Dict,
List,
+ Set,
Tuple,
Optional,
Callable,
@@ -83,6 +84,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
cdp_url: Optional[str] = None
useragent: Optional[str] = None
extra_flags: Optional[List[str]] = None
+ blocked_domains: Optional[Set[str]] = None
retries: RetriesCount = 3
retry_delay: Seconds = 1
@@ -145,6 +147,7 @@ class _fetch_params:
wait_selector_state: SelectorWaitStates
network_idle: bool
load_dom: bool
+ blocked_domains: Optional[Set[str]]
solve_cloudflare: bool
selector_config: Dict
@@ -183,6 +186,7 @@ def validate_fetch(
# solve_cloudflare defaults to False for models that don't have it (PlaywrightConfig)
result.setdefault("solve_cloudflare", False)
+ result.setdefault("blocked_domains", None)
return _fetch_params(**result)
diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py
index 1b44f5a..e9e74ae 100644
--- a/scrapling/engines/toolbelt/navigation.py
+++ b/scrapling/engines/toolbelt/navigation.py
@@ -11,7 +11,7 @@ from msgspec import Struct, structs, convert, ValidationError
from playwright.sync_api import Route
from scrapling.core.utils import log
-from scrapling.core._types import Dict, Tuple
+from scrapling.core._types import Dict, Set, Tuple, Optional, Callable
from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
__BYPASSES_DIR__ = Path(__file__).parent / "bypasses"
@@ -23,30 +23,58 @@ class ProxyDict(Struct):
password: str = ""
-def intercept_route(route: Route):
- """This is just a route handler, but it drops requests that its type falls in `DEFAULT_DISABLED_RESOURCES`
+def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[Set[str]] = None) -> Callable:
+ """Create a route handler that blocks both resource types and specific domains.
- :param route: PlayWright `Route` object of the current page
- :return: PlayWright `Route` object
+ :param disable_resources: Whether to block default resource types.
+ :param blocked_domains: Set of domain names to block requests to.
+ :return: A sync route handler function.
"""
- if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
- log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
- route.abort()
- else:
- route.continue_()
-
-
-async def async_intercept_route(route: async_Route):
- """This is just a route handler, but it drops requests that its type falls in `DEFAULT_DISABLED_RESOURCES`
-
- :param route: PlayWright `Route` object of the current page
- :return: PlayWright `Route` object
+ disabled_resources = DEFAULT_DISABLED_RESOURCES if disable_resources else set()
+ domains = blocked_domains or set()
+
+ def handler(route: Route):
+ if route.request.resource_type in disabled_resources:
+ log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
+ route.abort()
+ elif domains:
+ hostname = urlparse(route.request.url).hostname or ""
+ if any(hostname == d or hostname.endswith("." + d) for d in domains):
+ log.debug(f'Blocking request to blocked domain "{hostname}" ({route.request.url})')
+ route.abort()
+ else:
+ route.continue_()
+ else:
+ route.continue_()
+
+ return handler
+
+
+def create_async_intercept_handler(disable_resources: bool, blocked_domains: Optional[Set[str]] = None) -> Callable:
+ """Create an async route handler that blocks both resource types and specific domains.
+
+ :param disable_resources: Whether to block default resource types.
+ :param blocked_domains: Set of domain names to block requests to.
+ :return: An async route handler function.
"""
- if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
- log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
- await route.abort()
- else:
- await route.continue_()
+ disabled_resources = DEFAULT_DISABLED_RESOURCES if disable_resources else set()
+ domains = blocked_domains or set()
+
+ async def handler(route: async_Route):
+ if route.request.resource_type in disabled_resources:
+ log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
+ await route.abort()
+ elif domains:
+ hostname = urlparse(route.request.url).hostname or ""
+ if any(hostname == d or hostname.endswith("." + d) for d in domains):
+ log.debug(f'Blocking request to blocked domain "{hostname}" ({route.request.url})')
+ await route.abort()
+ else:
+ await route.continue_()
+ else:
+ await route.continue_()
+
+ return handler
def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple) -> Dict:
diff --git a/scrapling/fetchers/chrome.py b/scrapling/fetchers/chrome.py
index 0d50c19..591e5e9 100644
--- a/scrapling/fetchers/chrome.py
+++ b/scrapling/fetchers/chrome.py
@@ -13,7 +13,8 @@ class DynamicFetcher(BaseFetcher):
:param url: Target url.
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
- :param disable_resources: Drop requests of unnecessary resources for a speed boost.
+ :param disable_resources: Drop requests for unnecessary resources for a speed boost.
+ :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
@@ -55,7 +56,8 @@ class DynamicFetcher(BaseFetcher):
:param url: Target url.
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
- :param disable_resources: Drop requests of unnecessary resources for a speed boost.
+ :param disable_resources: Drop requests for unnecessary resources for a speed boost.
+ :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
diff --git a/scrapling/fetchers/stealth_chrome.py b/scrapling/fetchers/stealth_chrome.py
index 58574d3..6a70246 100644
--- a/scrapling/fetchers/stealth_chrome.py
+++ b/scrapling/fetchers/stealth_chrome.py
@@ -19,6 +19,7 @@ class StealthyFetcher(BaseFetcher):
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+ :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
@@ -67,6 +68,7 @@ class StealthyFetcher(BaseFetcher):
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+ :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.