Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

refactor(fetchers)!: Remove dynamic sessions and dynamic fetcher

Karim shoairc54966a8c7a8
diff --git a/scrapling/__init__.py b/scrapling/__init__.py
index f9453ce..3ec50af 100644
--- a/scrapling/__init__.py
+++ b/scrapling/__init__.py
@@ -7,7 +7,7 @@ from typing import Any, TYPE_CHECKING
if TYPE_CHECKING:
from scrapling.parser import Selector, Selectors
from scrapling.core.custom_types import AttributesHandler, TextHandler
- from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher
+ from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher
# Lazy import mapping
@@ -19,9 +19,8 @@ _LAZY_IMPORTS = {
"TextHandler": ("scrapling.core.custom_types", "TextHandler"),
"AsyncFetcher": ("scrapling.fetchers", "AsyncFetcher"),
"StealthyFetcher": ("scrapling.fetchers", "StealthyFetcher"),
- "DynamicFetcher": ("scrapling.fetchers", "DynamicFetcher"),
}
-__all__ = ["Selector", "Fetcher", "AsyncFetcher", "StealthyFetcher", "DynamicFetcher"]
+__all__ = ["Selector", "Fetcher", "AsyncFetcher", "StealthyFetcher"]
def __getattr__(name: str) -> Any:
diff --git a/scrapling/cli.py b/scrapling/cli.py
index 6bbb485..0bc709f 100644
--- a/scrapling/cli.py
+++ b/scrapling/cli.py
@@ -121,7 +121,7 @@ def install(force): # pragma: no cover
if force or not __PACKAGE_DIR__.joinpath(".scrapling_dependencies_installed").exists():
__Execute(
[python_executable, "-m", "playwright", "install", "chromium"],
- "Playwright browsers",
+ "Installing browsers",
)
__Execute(
[
@@ -131,7 +131,7 @@ def install(force): # pragma: no cover
"install-deps",
"chromium",
],
- "Playwright dependencies",
+ "Installing browsers dependencies",
)
from tld.utils import update_tld_names
@@ -291,7 +291,7 @@ def _common_http_options(f):
def _common_browser_options(f):
- """Apply shared Click options for browser-based commands (fetch/stealthy_fetch)."""
+ """Apply Click options for browser fetching."""
decorators = [
option(
"--ai-targeted",
@@ -586,53 +586,6 @@ def __build_browser_kwargs(
return kwargs
[email protected](help=f"Use DynamicFetcher to fetch content with browser automation.\n\n{__OUTPUT_FILE_HELP__}")
-@argument("url", required=True)
-@argument("output_file", required=True)
-@_common_browser_options
-def fetch(
- url,
- output_file,
- headless,
- disable_resources,
- network_idle,
- pierce_shadow,
- timeout,
- wait,
- css_selector,
- wait_selector,
- locale,
- real_chrome,
- proxy,
- extra_headers,
- ai_targeted,
- executable_path,
- dns_over_https,
- block_ads,
-):
- """Opens up a browser and fetch content using DynamicFetcher."""
- parsed_headers, _ = _ParseHeaders(extra_headers, False)
- kwargs = __build_browser_kwargs(
- headless,
- disable_resources,
- network_idle,
- pierce_shadow,
- timeout,
- wait,
- wait_selector,
- locale,
- real_chrome,
- proxy,
- parsed_headers,
- dns_over_https,
- block_ads,
- executable_path,
- )
- from scrapling.fetchers import DynamicFetcher
-
- __Request_and_Save(DynamicFetcher.fetch, url, output_file, css_selector, ai_targeted=ai_targeted, **kwargs)
-
-
@extract.command(help=f"Use StealthyFetcher to fetch content with advanced stealth features.\n\n{__OUTPUT_FILE_HELP__}")
@argument("url", required=True)
@argument("output_file", required=True)
@@ -653,7 +606,7 @@ def fetch(
help="Add noise to canvas operations (default: False)",
)
@_common_browser_options
-def stealthy_fetch(
+def fetch(
url,
output_file,
headless,
diff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.py
index 0a90e64..f5fc77c 100644
--- a/scrapling/core/_shell_signatures.py
+++ b/scrapling/core/_shell_signatures.py
@@ -12,6 +12,8 @@ from scrapling.core._types import (
FollowRedirects,
)
+from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator
+
# Parameter definitions for shell function signatures (defined once at module level)
# Mirrors TypedDict definitions from _types.py but runtime-accessible for IPython introspection
_REQUESTS_PARAMS = {
@@ -36,6 +38,7 @@ _REQUESTS_PARAMS = {
}
_FETCH_PARAMS = {
+ "max_pages": int,
"headless": bool,
"disable_resources": bool,
"network_idle": bool,
@@ -43,13 +46,14 @@ _FETCH_PARAMS = {
"pierce_shadow": bool,
"wait_selector": Optional[str],
"wait_selector_state": SelectorWaitStates,
- "cookies": Sequence[SetCookieParam],
+ "cookies": Sequence[SetCookieParam] | None,
"google_search": bool,
"wait": int | float,
"timezone_id": str | None,
"page_action": Optional[Callable],
"page_setup": Optional[Callable],
"proxy": Optional[str | Dict[str, str] | Tuple],
+ "proxy_rotator": Optional[ProxyRotator],
"extra_headers": Optional[Dict[str, str]],
"timeout": int | float,
"init_script": Optional[str],
@@ -69,10 +73,6 @@ _FETCH_PARAMS = {
"max_recorded_requests": int,
"executable_path": Optional[str],
"dns_over_https": bool,
-}
-
-_STEALTHY_FETCH_PARAMS = {
- **_FETCH_PARAMS,
"allow_webgl": bool,
"hide_canvas": bool,
"block_webrtc": bool,
@@ -86,5 +86,4 @@ Signatures_map = {
"put": {**_REQUESTS_PARAMS, "data": Optional[Dict | str], "json": Optional[Dict | List]},
"delete": _REQUESTS_PARAMS,
"fetch": _FETCH_PARAMS,
- "stealthy_fetch": _STEALTHY_FETCH_PARAMS,
}
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py
index 40e90c5..1c85f72 100644
--- a/scrapling/core/shell.py
+++ b/scrapling/core/shell.py
@@ -432,9 +432,6 @@ class CustomShell:
Fetcher as __Fetcher,
AsyncFetcher as __AsyncFetcher,
FetcherSession as __FetcherSession,
- DynamicFetcher as __DynamicFetcher,
- DynamicSession as __DynamicSession,
- AsyncDynamicSession as __AsyncDynamicSession,
StealthyFetcher as __StealthyFetcher,
StealthySession as __StealthySession,
AsyncStealthySession as __AsyncStealthySession,
@@ -444,9 +441,6 @@ class CustomShell:
self.__Fetcher = __Fetcher
self.__AsyncFetcher = __AsyncFetcher
self.__FetcherSession = __FetcherSession
- self.__DynamicFetcher = __DynamicFetcher
- self.__DynamicSession = __DynamicSession
- self.__AsyncDynamicSession = __AsyncDynamicSession
self.__StealthyFetcher = __StealthyFetcher
self.__StealthySession = __StealthySession
self.__AsyncStealthySession = __AsyncStealthySession
@@ -486,7 +480,6 @@ class CustomShell:
return f"""
-> Available Scrapling objects:
- Fetcher/AsyncFetcher/FetcherSession
- - DynamicFetcher/DynamicSession/AsyncDynamicSession
- StealthyFetcher/StealthySession/AsyncStealthySession
- Selector
@@ -495,8 +488,7 @@ class CustomShell:
- {"post":<30} Shortcut for `Fetcher.post`
- {"put":<30} Shortcut for `Fetcher.put`
- {"delete":<30} Shortcut for `Fetcher.delete`
- - {"fetch":<30} Shortcut for `DynamicFetcher.fetch`
- - {"stealthy_fetch":<30} Shortcut for `StealthyFetcher.fetch`
+ - {"fetch":<30} Shortcut for `StealthyFetcher.fetch`
-> Useful commands
- {"page / response":<30} The response object of the last page you fetched
@@ -551,8 +543,7 @@ Type 'exit' or press Ctrl+D to exit.
post = self.create_wrapper(self.__Fetcher.post)
put = self.create_wrapper(self.__Fetcher.put)
delete = self.create_wrapper(self.__Fetcher.delete)
- dynamic_fetch = self.create_wrapper(self.__DynamicFetcher.fetch)
- stealthy_fetch = self.create_wrapper(self.__StealthyFetcher.fetch, signature_name="stealthy_fetch")
+ fetch = self.create_wrapper(self.__StealthyFetcher.fetch)
curl2fetcher = self.create_wrapper(self._curl_parser.convert2fetcher, get_signature=False)
# Create the namespace dictionary
@@ -564,13 +555,9 @@ Type 'exit' or press Ctrl+D to exit.
"Fetcher": self.__Fetcher,
"AsyncFetcher": self.__AsyncFetcher,
"FetcherSession": self.__FetcherSession,
- "DynamicSession": self.__DynamicSession,
- "AsyncDynamicSession": self.__AsyncDynamicSession,
"StealthySession": self.__StealthySession,
"AsyncStealthySession": self.__AsyncStealthySession,
- "fetch": dynamic_fetch,
- "DynamicFetcher": self.__DynamicFetcher,
- "stealthy_fetch": stealthy_fetch,
+ "fetch": fetch,
"StealthyFetcher": self.__StealthyFetcher,
"Selector": Selector,
"page": self.page,
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py
index 686472b..eebac44 100644
--- a/scrapling/engines/_browsers/_base.py
+++ b/scrapling/engines/_browsers/_base.py
@@ -3,19 +3,18 @@ from pathlib import Path
from asyncio import sleep as asyncio_sleep, Lock
from contextlib import contextmanager, asynccontextmanager, suppress
-from playwright.sync_api._generated import Page
-from playwright.sync_api import (
+from patchright.sync_api import Page
+from patchright.sync_api import (
Frame,
BrowserContext,
Response as SyncPlaywrightResponse,
)
-from playwright.async_api._generated import Page as AsyncPage
-from playwright.async_api import (
+from patchright.async_api import Page as AsyncPage
+from patchright.async_api import (
Frame as AsyncFrame,
Response as AsyncPlaywrightResponse,
BrowserContext as AsyncBrowserContext,
)
-from playwright._impl._errors import Error as PlaywrightError
from scrapling.parser import Selector
from scrapling.engines._browsers._page import PageInfo, PagePool
@@ -178,7 +177,7 @@ class SyncSession:
"""Wait for the page to become idle (no network activity) even if there are never-ending requests."""
try:
page.wait_for_load_state("networkidle", timeout=timeout)
- except (PlaywrightError, Exception):
+ except Exception:
pass
def _wait_for_page_stability(self, page: Page | Frame, load_dom: bool, network_idle: bool):
@@ -404,7 +403,7 @@ class AsyncSession:
"""Wait for the page to become idle (no network activity) even if there are never-ending requests."""
try:
await page.wait_for_load_state("networkidle", timeout=timeout)
- except (PlaywrightError, Exception):
+ except Exception:
pass
async def _wait_for_page_stability(self, page: AsyncPage | AsyncFrame, load_dom: bool, network_idle: bool):
@@ -584,12 +583,6 @@ class BaseSessionMixin:
return context_options
-class DynamicSessionMixin(BaseSessionMixin):
- def __validate__(self, **params):
- self._config = self.__validate_routine__(params, model=PlaywrightConfig)
- self.__generate_options__()
-
-
class StealthySessionMixin(BaseSessionMixin):
def __validate__(self, **params):
self._config = self.__validate_routine__(params, model=StealthConfig)
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
deleted file mode 100644
index e53d27f..0000000
--- a/scrapling/engines/_browsers/_controllers.py
+++ /dev/null
@@ -1,404 +0,0 @@
-from time import sleep as time_sleep
-from asyncio import sleep as asyncio_sleep
-
-from playwright.sync_api import (
- Locator,
- sync_playwright,
-)
-from playwright.async_api import (
- async_playwright,
- Locator as AsyncLocator,
-)
-
-from scrapling.core.utils import log
-from scrapling.core._types import Optional, List, ProxyType, Unpack
-from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error
-from scrapling.engines.toolbelt.convertor import Response, ResponseFactory
-from scrapling.engines._browsers._types import PlaywrightSession, PlaywrightFetchParams
-from scrapling.engines._browsers._base import SyncSession, AsyncSession, DynamicSessionMixin
-from scrapling.engines._browsers._validators import validate_fetch as _validate, PlaywrightConfig
-
-
-class DynamicSession(SyncSession, DynamicSessionMixin):
- """A Browser session manager with page pooling."""
-
- __slots__ = (
- "_config",
- "_context_options",
- "_browser_options",
- "_user_data_dir",
- "_headers_keys",
- "max_pages",
- "page_pool",
- "_max_wait_for_page",
- "playwright",
- "context",
- )
-
- def __init__(self, **kwargs: Unpack[PlaywrightSession]):
- """A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.
-
- :param record_requests: Save completed browser requests and responses. Failed or unfinished requests are omitted. Defaults to False.
- :param max_recorded_requests: Maximum retained requests per session. Defaults to 1000; older entries are removed.
- :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
- :param disable_resources: Drop requests for unnecessary resources for a speed boost.
- Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
- :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
- :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
- :param cookies: Set cookies for the next request.
- :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
- :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
- :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
- :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
- :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
- :param wait_selector: Wait for a specific CSS selector to be in a specific state.
- :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
- :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
- rules. Defaults to the system default locale.
- :param timezone_id: Changes the timezone of the browser. Defaults to the system timezone.
- :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
- :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
- :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
- :param pierce_shadow: Include open Shadow DOM content in the response HTML. Defaults to False.
- :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
- :param google_search: Enabled by default, Scrapling will set a Google referer header.
- :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
- :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
- :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
- :param extra_flags: A list of additional browser flags to pass to the browser on launch.
- :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
- :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
- """
- self.__validate__(**kwargs)
- super().__init__(
- record_requests=self._config.record_requests,
- max_recorded_requests=self._config.max_recorded_requests,
- )
-
- def start(self):
- """Create a browser for this instance and context."""
- if not self.playwright:
- self.playwright = sync_playwright().start()
-
- try:
- if self._config.cdp_url: # pragma: no cover
- self.browser = self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)
- if not self._config.proxy_rotator and self.browser:
- self.context = self.browser.new_context(**self._context_options)
- elif self._config.proxy_rotator:
- self.browser = self.playwright.chromium.launch(**self._browser_options)
- else:
- persistent_options = (
- self._browser_options | self._context_options | {"user_data_dir": self._user_data_dir}
- )
- self.context = self.playwright.chromium.launch_persistent_context(**persistent_options)
-
- if self.context:
- self.context = self._initialize_context(self._config, self.context)
-
- self._is_alive = True
- except Exception:
- # Clean up playwright if browser setup fails
- self.playwright.stop()
- self.playwright = None
- raise
- else:
- raise RuntimeError("Session has been already started")
-
- def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response:
- """Opens up the browser and do your request based on your chosen options.
-
- :param url: The Target url.
- :param google_search: Enabled by default, Scrapling will set a Google referer header.
- :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
- :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
- :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
- :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
- :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
- :param disable_resources: Drop requests for unnecessary resources for a speed boost.
- Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
- :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
- :param wait_selector: Wait for a specific CSS selector to be in a specific state.
- :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
- :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
- :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
- :param pierce_shadow: Include open Shadow DOM content in the response HTML. Defaults to False.
- :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
- :param proxy: Static proxy to override rotator and session proxy. A new browser context will be created and used with it.
- :return: A `Response` object.
- """
- static_proxy = kwargs.pop("proxy", None)
-
- params = _validate(kwargs, self, PlaywrightConfig)
- if not self._is_alive: # pragma: no cover
- raise RuntimeError("Context manager has been closed")
-
- request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set()
- referer = (
- "https://www.google.com/" if (params.google_search and "referer" not in request_headers_keys) else None
- )
-
- for attempt in range(self._config.retries):
- proxy: Optional[ProxyType] = None
- if self._config.proxy_rotator and static_proxy is None:
- proxy = self._config.proxy_rotator.get_proxy()
- else:
- proxy = static_proxy
-
- with self._page_generator(
- params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
- ) as page_info:
- final_response: List = [None]
- page = page_info.page
- handler = self._create_response_handler(page_info, final_response)
- page.on("response", handler)
- try:
- if params.page_setup:
- try:
- params.page_setup(page)
- except Exception as e: # pragma: no cover
- log.error(f"Error executing page_setup: {e}")
-
- try:
- first_response = page.goto(url, referer=referer)
- self._wait_for_page_stability(page, params.load_dom, params.network_idle)
-
- if not first_response:
- raise RuntimeError(f"Failed to get response for {url}")
-
- if params.page_action:
- try:
- _ = params.page_action(page)
- except Exception as e: # pragma: no cover
- log.error(f"Error executing page_action: {e}")
-
- if params.wait_selector:
- try:
- waiter: Locator = page.locator(params.wait_selector)
- waiter.first.wait_for(state=params.wait_selector_state)
- self._wait_for_page_stability(page, params.load_dom, params.network_idle)
- except Exception as e: # pragma: no cover
- log.error(f"Error waiting for selector {params.wait_selector}: {e}")
-
- page.wait_for_timeout(params.wait)
-
- response = ResponseFactory.from_playwright_response(
- page,
- first_response,
- final_response[0],
- params.selector_config,
- meta={"proxy": proxy},
- pierce_shadow=params.pierce_shadow,
- )
- return response
-
- except Exception as e:
- page_info.mark_error()
- if attempt < self._config.retries - 1:
- if is_proxy_error(e):
- log.warning(
- f"Proxy '{proxy}' failed (attempt {attempt + 1}) | Retrying in {self._config.retry_delay}s..."
- )
- else:
- log.warning(
- f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s..."
- )
- time_sleep(self._config.retry_delay)
- else:
- log.error(f"Failed after {self._config.retries} attempts: {e}")
- raise
- finally:
- page.remove_listener("response", handler)
-
- raise RuntimeError("Request failed") # pragma: no cover
-
-
-class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
- """An async Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory."""
-
- __slots__ = (
- "_config",
- "_context_options",
- "_browser_options",
- "_user_data_dir",
- "_headers_keys",
- )
-
- def __init__(self, **kwargs: Unpack[PlaywrightSession]):
- """A Browser session manager with page pooling
-
- :param record_requests: Save completed browser requests and responses. Failed or unfinished requests are omitted. Defaults to False.
- :param max_recorded_requests: Maximum retained requests per session. Defaults to 1000; older entries are removed.
- :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
- :param disable_resources: Drop requests for unnecessary resources for a speed boost.
- Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
- :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
- :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
- :param cookies: Set cookies for the next request.
- :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
- :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
- :param pierce_shadow: Include open Shadow DOM content in the response HTML. Defaults to False.
- :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
- :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
- :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
- :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
- :param wait_selector: Wait for a specific CSS selector to be in a specific state.
- :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
- :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
- rules. Defaults to the system default locale.
- :param timezone_id: Changes the timezone of the browser. Defaults to the system timezone.
- :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
- :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
- :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
- :param google_search: Enabled by default, Scrapling will set a Google referer header.
- :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
- :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
- :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
- :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
- :param extra_flags: A list of additional browser flags to pass to the browser on launch.
- :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
- :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
- """
- self.__validate__(**kwargs)
- super().__init__(
- max_pages=self._config.max_pages,
- record_requests=self._config.record_requests,
- max_recorded_requests=self._config.max_recorded_requests,
- )
-
- async def start(self) -> None:
- """Create a browser for this instance and context."""
- if not self.playwright:
- self.playwright = await async_playwright().start()
- try:
- if self._config.cdp_url:
- self.browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)
- if not self._config.proxy_rotator and self.browser:
- self.context = await self.browser.new_context(**self._context_options)
- elif self._config.proxy_rotator:
- self.browser = await self.playwright.chromium.launch(**self._browser_options)
- else:
- persistent_options = (
- self._browser_options | self._context_options | {"user_data_dir": self._user_data_dir}
- )
- self.context = await self.playwright.chromium.launch_persistent_context(**persistent_options)
-
- if self.context:
- self.context = await self._initialize_context(self._config, self.context)
-
- self._is_alive = True
- except Exception:
- # Clean up playwright if browser setup fails
- await self.playwright.stop()
- self.playwright = None
- raise
- else:
- raise RuntimeError("Session has been already started")
-
- async def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response:
- """Opens up the browser and do your request based on your chosen options.
-
- :param url: The Target url.
- :param google_search: Enabled by default, Scrapling will set a Google referer header.
- :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
- :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
- :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
- :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
- :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
- :param disable_resources: Drop requests for unnecessary resources for a speed boost.
- Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
- :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
- :param wait_selector: Wait for a specific CSS selector to be in a specific state.
- :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
- :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
- :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
- :param pierce_shadow: Include open Shadow DOM content in the response HTML. Defaults to False.
- :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
- :param proxy: Static proxy to override rotator and session proxy. A new browser context will be created and used with it.
- :return: A `Response` object.
- """
- static_proxy = kwargs.pop("proxy", None)
-
- params = _validate(kwargs, self, PlaywrightConfig)
-
- if not self._is_alive: # pragma: no cover
- raise RuntimeError("Context manager has been closed")
-
- request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set()
- referer = (
- "https://www.google.com/" if (params.google_search and "referer" not in request_headers_keys) else None
- )
-
- for attempt in range(self._config.retries):
- proxy: Optional[ProxyType] = None
- if self._config.proxy_rotator and static_proxy is None:
- proxy = self._config.proxy_rotator.get_proxy()
- else:
- proxy = static_proxy
-
- async with self._page_generator(
- params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
- ) as page_info:
- final_response: List = [None]
- page = page_info.page
- handler = self._create_response_handler(page_info, final_response)
- page.on("response", handler)
- try:
- if params.page_setup:
- try:
- await params.page_setup(page)
- except Exception as e: # pragma: no cover
- log.error(f"Error executing page_setup: {e}")
-
- try:
- first_response = await page.goto(url, referer=referer)
- await self._wait_for_page_stability(page, params.load_dom, params.network_idle)
-
- if not first_response:
- raise RuntimeError(f"Failed to get response for {url}")
-
- if params.page_action:
- try:
- _ = await params.page_action(page)
- except Exception as e: # pragma: no cover
- log.error(f"Error executing page_action: {e}")
-
- if params.wait_selector:
- try:
- waiter: AsyncLocator = page.locator(params.wait_selector)
- await waiter.first.wait_for(state=params.wait_selector_state)
- await self._wait_for_page_stability(page, params.load_dom, params.network_idle)
- except Exception as e: # pragma: no cover
- log.error(f"Error waiting for selector {params.wait_selector}: {e}")
-
- await page.wait_for_timeout(params.wait)
-
- response = await ResponseFactory.from_async_playwright_response(
- page,
- first_response,
- final_response[0],
- params.selector_config,
- meta={"proxy": proxy},
- pierce_shadow=params.pierce_shadow,
- )
- return response
-
- except Exception as e:
- page_info.mark_error()
- if attempt < self._config.retries - 1:
- if is_proxy_error(e):
- log.warning(
- f"Proxy '{proxy}' failed (attempt {attempt + 1}) | Retrying in {self._config.retry_delay}s..."
- )
- else:
- log.warning(
- f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s..."
- )
- await asyncio_sleep(self._config.retry_delay)
- else:
- log.error(f"Failed after {self._config.retries} attempts: {e}")
- raise
- finally:
- page.remove_listener("response", handler)
-
- raise RuntimeError("Request failed") # pragma: no cover
diff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.py
index 2b32e35..6bf3b66 100644
--- a/scrapling/engines/_browsers/_page.py
+++ b/scrapling/engines/_browsers/_page.py
@@ -1,8 +1,8 @@
from threading import RLock
from dataclasses import dataclass
-from playwright.sync_api._generated import Page as SyncPage
-from playwright.async_api._generated import Page as AsyncPage
+from patchright.sync_api import Page as SyncPage
+from patchright.async_api import Page as AsyncPage
from scrapling.core._types import Optional, List, Literal, overload, TypeVar, Generic, cast
diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py
index ff86e73..875b5cb 100644
--- a/scrapling/engines/_browsers/_stealth.py
+++ b/scrapling/engines/_browsers/_stealth.py
@@ -3,8 +3,8 @@ from re import compile as re_compile
from time import sleep as time_sleep
from asyncio import sleep as asyncio_sleep
-from playwright.sync_api import Locator, Page
-from playwright.async_api import Page as async_Page, Locator as AsyncLocator
+from patchright.sync_api import Locator, Page
+from patchright.async_api import Page as async_Page, Locator as AsyncLocator
from patchright.sync_api import sync_playwright
from patchright.async_api import async_playwright
diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py
index 72cc089..4030188 100644
--- a/scrapling/engines/toolbelt/convertor.py
+++ b/scrapling/engines/toolbelt/convertor.py
@@ -4,10 +4,9 @@ from functools import lru_cache
from lxml.etree import LxmlError
from curl_cffi.requests import Response as CurlResponse
-from playwright._impl._errors import Error as PlaywrightError
-from patchright._impl._errors import Error as PatchrightError
-from playwright.sync_api import Page as SyncPage, Response as SyncResponse
-from playwright.async_api import Page as AsyncPage, Response as AsyncResponse
+from patchright.sync_api import Error as PatchrightError
+from patchright.sync_api import Page as SyncPage, Response as SyncResponse
+from patchright.async_api import Page as AsyncPage, Response as AsyncResponse
from scrapling.core.utils import log
from .custom import Response, StatusText
@@ -175,7 +174,7 @@ class ResponseFactory:
are also derived from the provided response headers or reasonable defaults.
Additionally, the page content and cookies are extracted for further use.
- :param page: A synchronous Playwright `Page` instance that represents the current browser page. Required to retrieve the page's URL, cookies, and content.
+ :param page: A synchronous Patchright `Page` instance that represents the current browser page. Required to retrieve the page's URL, cookies, and content.
:param final_response: The last response received for the given request from the Playwright instance. Typically used as the main response object to derive status, headers, and other metadata.
:param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one.
:param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into
@@ -310,7 +309,7 @@ class ResponseFactory:
for _ in range(max_retries):
try:
return page.content() or ""
- except (PlaywrightError, PatchrightError):
+ except PatchrightError:
page.wait_for_timeout(500)
raise RuntimeError(f"Failed to retrieve the page content after retrying for {max_retries * 500}ms.")
@@ -325,7 +324,7 @@ class ResponseFactory:
for _ in range(max_retries):
try:
return (await page.content()) or ""
- except (PlaywrightError, PatchrightError):
+ except PatchrightError:
await page.wait_for_timeout(500)
raise RuntimeError(f"Failed to retrieve the page content after retrying for {max_retries * 500}ms.")
@@ -350,7 +349,7 @@ class ResponseFactory:
are also derived from the provided response headers or reasonable defaults.
Additionally, the page content and cookies are extracted for further use.
- :param page: An asynchronous Playwright `Page` instance that represents the current browser page. Required to retrieve the page's URL, cookies, and content.
+ :param page: An asynchronous Patchright `Page` instance that represents the current browser page. Required to retrieve the page's URL, cookies, and content.
:param final_response: The last response received for the given request from the Playwright instance. Typically used as the main response object to derive status, headers, and other metadata.
:param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one.
:param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into
diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py
index 80b004d..16e1920 100644
--- a/scrapling/engines/toolbelt/fingerprints.py
+++ b/scrapling/engines/toolbelt/fingerprints.py
@@ -43,10 +43,10 @@ def get_os_name() -> OSName | Tuple:
@lru_cache(2, typed=True)
-def driven_browser_version(package: str = "playwright") -> int | None:
+def driven_browser_version(package: str = "patchright") -> int | None:
"""Get the Chromium major version the installed automation package drives, read from its bundled `browsers.json`.
- :param package: The automation package to inspect, e.g. `"playwright"` or `"patchright"`
+ :param package: The automation package to inspect. Defaults to `"patchright"`
:return: The Chromium major version, or `None` if it couldn't be determined
"""
spec = find_spec(package)
@@ -66,7 +66,7 @@ def driven_browser_version(package: str = "playwright") -> int | None:
def generate_headers(browser_mode: bool | str = False) -> Dict:
"""Generate real browser-like headers using browserforge's generator
- :param browser_mode: If enabled, the headers created are used for playwright, so it has to match everything
+ :param browser_mode: If enabled, the headers created are used for the browser, so it has to match everything
:return: A dictionary of the generated headers
"""
os_name = get_os_name()
@@ -84,7 +84,7 @@ def generate_headers(browser_mode: bool | str = False) -> Dict:
headers = HeaderGenerator(
browser=[Browser(name=b.name) for b in browsers], os=os_name, device=device
).generate()
- if browser_mode and (driven := driven_browser_version() or driven_browser_version("patchright")):
+ if browser_mode and (driven := driven_browser_version()):
if user_agent := headers.get("User-Agent"):
headers["User-Agent"] = re.sub(r"Chrome/[\d.]+", f"Chrome/{driven}.0.0.0", user_agent)
return headers
diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py
index 01bee07..c23532a 100644
--- a/scrapling/engines/toolbelt/navigation.py
+++ b/scrapling/engines/toolbelt/navigation.py
@@ -4,9 +4,9 @@ Functions related to files and URLs
from urllib.parse import urlparse
-from playwright.async_api import Route as async_Route
+from patchright.async_api import Route as async_Route
from msgspec import Struct, structs, convert, ValidationError
-from playwright.sync_api import Route
+from patchright.sync_api import Route
from scrapling.core.utils import log
from scrapling.core._types import Dict, Set, Tuple, Optional, Callable
diff --git a/scrapling/fetchers/__init__.py b/scrapling/fetchers/__init__.py
index 7db355d..1ef500e 100644
--- a/scrapling/fetchers/__init__.py
+++ b/scrapling/fetchers/__init__.py
@@ -3,7 +3,6 @@ from scrapling.engines.toolbelt import ProxyRotator
if TYPE_CHECKING:
from scrapling.fetchers.requests import Fetcher, AsyncFetcher, FetcherSession
- from scrapling.fetchers.chrome import DynamicFetcher, DynamicSession, AsyncDynamicSession
from scrapling.fetchers.stealth_chrome import StealthyFetcher, StealthySession, AsyncStealthySession
@@ -12,9 +11,6 @@ _LAZY_IMPORTS = {
"Fetcher": ("scrapling.fetchers.requests", "Fetcher"),
"AsyncFetcher": ("scrapling.fetchers.requests", "AsyncFetcher"),
"FetcherSession": ("scrapling.fetchers.requests", "FetcherSession"),
- "DynamicFetcher": ("scrapling.fetchers.chrome", "DynamicFetcher"),
- "DynamicSession": ("scrapling.fetchers.chrome", "DynamicSession"),
- "AsyncDynamicSession": ("scrapling.fetchers.chrome", "AsyncDynamicSession"),
"StealthyFetcher": ("scrapling.fetchers.stealth_chrome", "StealthyFetcher"),
"StealthySession": ("scrapling.fetchers.stealth_chrome", "StealthySession"),
"AsyncStealthySession": ("scrapling.fetchers.stealth_chrome", "AsyncStealthySession"),
@@ -25,9 +21,6 @@ __all__ = [
"AsyncFetcher",
"ProxyRotator",
"FetcherSession",
- "DynamicFetcher",
- "DynamicSession",
- "AsyncDynamicSession",
"StealthyFetcher",
"StealthySession",
"AsyncStealthySession",
diff --git a/scrapling/fetchers/chrome.py b/scrapling/fetchers/chrome.py
deleted file mode 100644
index b03285a..0000000
--- a/scrapling/fetchers/chrome.py
+++ /dev/null
@@ -1,99 +0,0 @@
-from scrapling.core._types import Unpack
-from scrapling.engines._browsers._types import PlaywrightSession
-from scrapling.engines.toolbelt.custom import BaseFetcher, Response
-from scrapling.engines._browsers._controllers import DynamicSession, AsyncDynamicSession
-
-
-class DynamicFetcher(BaseFetcher):
- """A `Fetcher` that provide many options to fetch/load websites' pages through chromium-based browsers."""
-
- @classmethod
- def fetch(cls, url: str, **kwargs: Unpack[PlaywrightSession]) -> Response:
- """Opens up a browser and do your request based on your chosen options below.
-
- :param url: Target url.
- :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
- :param disable_resources: Drop requests for unnecessary resources for a speed boost.
- :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
- :param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
- :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
- :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
- :param cookies: Set cookies for the next request.
- :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
- :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
- :param pierce_shadow: Include open Shadow DOM content in the response HTML. Defaults to False.
- :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
- :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object.
- :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
- :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
- :param wait_selector: Wait for a specific CSS selector to be in a specific state.
- :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.
- :param locale: Set the locale for the browser if wanted. Defaults to the system default locale.
- :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
- :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
- :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
- :param google_search: Enabled by default, Scrapling will set a Google referer header.
- :param extra_headers: A dictionary of extra headers to add to the request.
- :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
- :param extra_flags: A list of additional browser flags to pass to the browser on launch.
- :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
- :param additional_args: Additional arguments to be passed to Playwright's context as additional settings.
- :return: A `Response` object.
- """
- selector_config = kwargs.get("selector_config", {}) or kwargs.get(
- "custom_config", {}
- ) # Checking `custom_config` for backward compatibility
- if not isinstance(selector_config, dict):
- raise TypeError("Argument `selector_config` must be a dictionary.")
-
- kwargs["selector_config"] = {**cls._generate_parser_arguments(), **selector_config}
-
- with DynamicSession(**kwargs) as session:
- return session.fetch(url)
-
- @classmethod
- async def async_fetch(cls, url: str, **kwargs: Unpack[PlaywrightSession]) -> Response:
- """Opens up a browser and do your request based on your chosen options below.
-
- :param url: Target url.
- :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
- :param disable_resources: Drop requests for unnecessary resources for a speed boost.
- :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).
- :param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.
- :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.
- :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
- :param cookies: Set cookies for the next request.
- :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
- :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
- :param pierce_shadow: Include open Shadow DOM content in the response HTML. Defaults to False.
- :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
- :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object.
- :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
- :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
- :param wait_selector: Wait for a specific CSS selector to be in a specific state.
- :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.
- :param locale: Set the locale for the browser if wanted. Defaults to the system default locale.
- :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
- :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
- :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
- :param google_search: Enabled by default, Scrapling will set a Google referer header.
- :param extra_headers: A dictionary of extra headers to add to the request.
- :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
- :param extra_flags: A list of additional browser flags to pass to the browser on launch.
- :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
- :param additional_args: Additional arguments to be passed to Playwright's context as additional settings.
- :return: A `Response` object.
- """
- selector_config = kwargs.get("selector_config", {}) or kwargs.get(
- "custom_config", {}
- ) # Checking `custom_config` for backward compatibility
- if not isinstance(selector_config, dict):
- raise TypeError("Argument `selector_config` must be a dictionary.")
-
- kwargs["selector_config"] = {**cls._generate_parser_arguments(), **selector_config}
-
- async with AsyncDynamicSession(**kwargs) as session:
- return await session.fetch(url)
-
-
-PlayWrightFetcher = DynamicFetcher # For backward-compatibility
diff --git a/scrapling/spiders/session.py b/scrapling/spiders/session.py
index 513b423..89e008b 100644
--- a/scrapling/spiders/session.py
+++ b/scrapling/spiders/session.py
@@ -4,9 +4,9 @@ from scrapling.spiders.request import Request
from scrapling.engines.static import _ASyncSessionLogic
from scrapling.engines.toolbelt.convertor import Response
from scrapling.core._types import Set, cast, SUPPORTED_HTTP_METHODS
-from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, FetcherSession
+from scrapling.fetchers import AsyncStealthySession, FetcherSession
-Session = FetcherSession | AsyncDynamicSession | AsyncStealthySession
+Session = FetcherSession | AsyncStealthySession
class SessionManager:
diff --git a/zensical.toml b/zensical.toml
index 980bcb7..03c8b40 100644
--- a/zensical.toml
+++ b/zensical.toml
@@ -27,8 +27,7 @@ nav = [
{Fetching = [
{"Fetchers basics" = "fetching/choosing.md"},
{"HTTP requests" = "fetching/static.md"},
- {"Dynamic websites" = "fetching/dynamic.md"},
- {"Dynamic websites with hard protections" = "fetching/stealthy.md"}
+ {"Browser fetching" = "fetching/stealthy.md"}
]},
{Spiders = [
{"Architecture" = "spiders/architecture.md"},