🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor(fetchers)!: Remove dynamic sessions and dynamic fetcher
Karim shoairc54966a8c7a8
diff --git a/scrapling/__init__.py b/scrapling/__init__.pyindex f9453ce..3ec50af 100644--- a/scrapling/__init__.py+++ b/scrapling/__init__.py@@ -7,7 +7,7 @@ from typing import Any, TYPE_CHECKINGif TYPE_CHECKING:from scrapling.parser import Selector, Selectorsfrom scrapling.core.custom_types import AttributesHandler, TextHandler- from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher, DynamicFetcher+ from scrapling.fetchers import Fetcher, AsyncFetcher, StealthyFetcher# Lazy import mapping@@ -19,9 +19,8 @@ _LAZY_IMPORTS = {"TextHandler": ("scrapling.core.custom_types", "TextHandler"),"AsyncFetcher": ("scrapling.fetchers", "AsyncFetcher"),"StealthyFetcher": ("scrapling.fetchers", "StealthyFetcher"),- "DynamicFetcher": ("scrapling.fetchers", "DynamicFetcher"),}-__all__ = ["Selector", "Fetcher", "AsyncFetcher", "StealthyFetcher", "DynamicFetcher"]+__all__ = ["Selector", "Fetcher", "AsyncFetcher", "StealthyFetcher"]def __getattr__(name: str) -> Any:diff --git a/scrapling/cli.py b/scrapling/cli.pyindex 6bbb485..0bc709f 100644--- a/scrapling/cli.py+++ b/scrapling/cli.py@@ -121,7 +121,7 @@ def install(force): # pragma: no coverif force or not __PACKAGE_DIR__.joinpath(".scrapling_dependencies_installed").exists():__Execute([python_executable, "-m", "playwright", "install", "chromium"],- "Playwright browsers",+ "Installing browsers",)__Execute([@@ -131,7 +131,7 @@ def install(force): # pragma: no cover"install-deps","chromium",],- "Playwright dependencies",+ "Installing browsers dependencies",)from tld.utils import update_tld_names@@ -291,7 +291,7 @@ def _common_http_options(f):def _common_browser_options(f):- """Apply shared Click options for browser-based commands (fetch/stealthy_fetch)."""+ """Apply Click options for browser fetching."""decorators = [option("--ai-targeted",@@ -586,53 +586,6 @@ def __build_browser_kwargs(return kwargs[email protected](help=f"Use DynamicFetcher to fetch content with browser automation.\n\n{__OUTPUT_FILE_HELP__}")-@argument("url", required=True)-@argument("output_file", required=True)-@_common_browser_options-def fetch(- url,- output_file,- headless,- disable_resources,- network_idle,- pierce_shadow,- timeout,- wait,- css_selector,- wait_selector,- locale,- real_chrome,- proxy,- extra_headers,- ai_targeted,- executable_path,- dns_over_https,- block_ads,-):- """Opens up a browser and fetch content using DynamicFetcher."""- parsed_headers, _ = _ParseHeaders(extra_headers, False)- kwargs = __build_browser_kwargs(- headless,- disable_resources,- network_idle,- pierce_shadow,- timeout,- wait,- wait_selector,- locale,- real_chrome,- proxy,- parsed_headers,- dns_over_https,- block_ads,- executable_path,- )- from scrapling.fetchers import DynamicFetcher-- __Request_and_Save(DynamicFetcher.fetch, url, output_file, css_selector, ai_targeted=ai_targeted, **kwargs)--@extract.command(help=f"Use StealthyFetcher to fetch content with advanced stealth features.\n\n{__OUTPUT_FILE_HELP__}")@argument("url", required=True)@argument("output_file", required=True)@@ -653,7 +606,7 @@ def fetch(help="Add noise to canvas operations (default: False)",)@_common_browser_options-def stealthy_fetch(+def fetch(url,output_file,headless,diff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.pyindex 0a90e64..f5fc77c 100644--- a/scrapling/core/_shell_signatures.py+++ b/scrapling/core/_shell_signatures.py@@ -12,6 +12,8 @@ from scrapling.core._types import (FollowRedirects,)+from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator+# Parameter definitions for shell function signatures (defined once at module level)# Mirrors TypedDict definitions from _types.py but runtime-accessible for IPython introspection_REQUESTS_PARAMS = {@@ -36,6 +38,7 @@ _REQUESTS_PARAMS = {}_FETCH_PARAMS = {+ "max_pages": int,"headless": bool,"disable_resources": bool,"network_idle": bool,@@ -43,13 +46,14 @@ _FETCH_PARAMS = {"pierce_shadow": bool,"wait_selector": Optional[str],"wait_selector_state": SelectorWaitStates,- "cookies": Sequence[SetCookieParam],+ "cookies": Sequence[SetCookieParam] | None,"google_search": bool,"wait": int | float,"timezone_id": str | None,"page_action": Optional[Callable],"page_setup": Optional[Callable],"proxy": Optional[str | Dict[str, str] | Tuple],+ "proxy_rotator": Optional[ProxyRotator],"extra_headers": Optional[Dict[str, str]],"timeout": int | float,"init_script": Optional[str],@@ -69,10 +73,6 @@ _FETCH_PARAMS = {"max_recorded_requests": int,"executable_path": Optional[str],"dns_over_https": bool,-}--_STEALTHY_FETCH_PARAMS = {- **_FETCH_PARAMS,"allow_webgl": bool,"hide_canvas": bool,"block_webrtc": bool,@@ -86,5 +86,4 @@ Signatures_map = {"put": {**_REQUESTS_PARAMS, "data": Optional[Dict | str], "json": Optional[Dict | List]},"delete": _REQUESTS_PARAMS,"fetch": _FETCH_PARAMS,- "stealthy_fetch": _STEALTHY_FETCH_PARAMS,}diff --git a/scrapling/core/shell.py b/scrapling/core/shell.pyindex 40e90c5..1c85f72 100644--- a/scrapling/core/shell.py+++ b/scrapling/core/shell.py@@ -432,9 +432,6 @@ class CustomShell:Fetcher as __Fetcher,AsyncFetcher as __AsyncFetcher,FetcherSession as __FetcherSession,- DynamicFetcher as __DynamicFetcher,- DynamicSession as __DynamicSession,- AsyncDynamicSession as __AsyncDynamicSession,StealthyFetcher as __StealthyFetcher,StealthySession as __StealthySession,AsyncStealthySession as __AsyncStealthySession,@@ -444,9 +441,6 @@ class CustomShell:self.__Fetcher = __Fetcherself.__AsyncFetcher = __AsyncFetcherself.__FetcherSession = __FetcherSession- self.__DynamicFetcher = __DynamicFetcher- self.__DynamicSession = __DynamicSession- self.__AsyncDynamicSession = __AsyncDynamicSessionself.__StealthyFetcher = __StealthyFetcherself.__StealthySession = __StealthySessionself.__AsyncStealthySession = __AsyncStealthySession@@ -486,7 +480,6 @@ class CustomShell:return f"""-> Available Scrapling objects:- Fetcher/AsyncFetcher/FetcherSession- - DynamicFetcher/DynamicSession/AsyncDynamicSession- StealthyFetcher/StealthySession/AsyncStealthySession- Selector@@ -495,8 +488,7 @@ class CustomShell:- {"post":<30} Shortcut for `Fetcher.post`- {"put":<30} Shortcut for `Fetcher.put`- {"delete":<30} Shortcut for `Fetcher.delete`- - {"fetch":<30} Shortcut for `DynamicFetcher.fetch`- - {"stealthy_fetch":<30} Shortcut for `StealthyFetcher.fetch`+ - {"fetch":<30} Shortcut for `StealthyFetcher.fetch`-> Useful commands- {"page / response":<30} The response object of the last page you fetched@@ -551,8 +543,7 @@ Type 'exit' or press Ctrl+D to exit.post = self.create_wrapper(self.__Fetcher.post)put = self.create_wrapper(self.__Fetcher.put)delete = self.create_wrapper(self.__Fetcher.delete)- dynamic_fetch = self.create_wrapper(self.__DynamicFetcher.fetch)- stealthy_fetch = self.create_wrapper(self.__StealthyFetcher.fetch, signature_name="stealthy_fetch")+ fetch = self.create_wrapper(self.__StealthyFetcher.fetch)curl2fetcher = self.create_wrapper(self._curl_parser.convert2fetcher, get_signature=False)# Create the namespace dictionary@@ -564,13 +555,9 @@ Type 'exit' or press Ctrl+D to exit."Fetcher": self.__Fetcher,"AsyncFetcher": self.__AsyncFetcher,"FetcherSession": self.__FetcherSession,- "DynamicSession": self.__DynamicSession,- "AsyncDynamicSession": self.__AsyncDynamicSession,"StealthySession": self.__StealthySession,"AsyncStealthySession": self.__AsyncStealthySession,- "fetch": dynamic_fetch,- "DynamicFetcher": self.__DynamicFetcher,- "stealthy_fetch": stealthy_fetch,+ "fetch": fetch,"StealthyFetcher": self.__StealthyFetcher,"Selector": Selector,"page": self.page,diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex 686472b..eebac44 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -3,19 +3,18 @@ from pathlib import Pathfrom asyncio import sleep as asyncio_sleep, Lockfrom contextlib import contextmanager, asynccontextmanager, suppress-from playwright.sync_api._generated import Page-from playwright.sync_api import (+from patchright.sync_api import Page+from patchright.sync_api import (Frame,BrowserContext,Response as SyncPlaywrightResponse,)-from playwright.async_api._generated import Page as AsyncPage-from playwright.async_api import (+from patchright.async_api import Page as AsyncPage+from patchright.async_api import (Frame as AsyncFrame,Response as AsyncPlaywrightResponse,BrowserContext as AsyncBrowserContext,)-from playwright._impl._errors import Error as PlaywrightErrorfrom scrapling.parser import Selectorfrom scrapling.engines._browsers._page import PageInfo, PagePool@@ -178,7 +177,7 @@ class SyncSession:"""Wait for the page to become idle (no network activity) even if there are never-ending requests."""try:page.wait_for_load_state("networkidle", timeout=timeout)- except (PlaywrightError, Exception):+ except Exception:passdef _wait_for_page_stability(self, page: Page | Frame, load_dom: bool, network_idle: bool):@@ -404,7 +403,7 @@ class AsyncSession:"""Wait for the page to become idle (no network activity) even if there are never-ending requests."""try:await page.wait_for_load_state("networkidle", timeout=timeout)- except (PlaywrightError, Exception):+ except Exception:passasync def _wait_for_page_stability(self, page: AsyncPage | AsyncFrame, load_dom: bool, network_idle: bool):@@ -584,12 +583,6 @@ class BaseSessionMixin:return context_options-class DynamicSessionMixin(BaseSessionMixin):- def __validate__(self, **params):- self._config = self.__validate_routine__(params, model=PlaywrightConfig)- self.__generate_options__()--class StealthySessionMixin(BaseSessionMixin):def __validate__(self, **params):self._config = self.__validate_routine__(params, model=StealthConfig)diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pydeleted file mode 100644index e53d27f..0000000--- a/scrapling/engines/_browsers/_controllers.py+++ /dev/null@@ -1,404 +0,0 @@-from time import sleep as time_sleep-from asyncio import sleep as asyncio_sleep--from playwright.sync_api import (- Locator,- sync_playwright,-)-from playwright.async_api import (- async_playwright,- Locator as AsyncLocator,-)--from scrapling.core.utils import log-from scrapling.core._types import Optional, List, ProxyType, Unpack-from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error-from scrapling.engines.toolbelt.convertor import Response, ResponseFactory-from scrapling.engines._browsers._types import PlaywrightSession, PlaywrightFetchParams-from scrapling.engines._browsers._base import SyncSession, AsyncSession, DynamicSessionMixin-from scrapling.engines._browsers._validators import validate_fetch as _validate, PlaywrightConfig---class DynamicSession(SyncSession, DynamicSessionMixin):- """A Browser session manager with page pooling."""-- __slots__ = (- "_config",- "_context_options",- "_browser_options",- "_user_data_dir",- "_headers_keys",- "max_pages",- "page_pool",- "_max_wait_for_page",- "playwright",- "context",- )-- def __init__(self, **kwargs: Unpack[PlaywrightSession]):- """A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.-- :param record_requests: Save completed browser requests and responses. Failed or unfinished requests are omitted. Defaults to False.- :param max_recorded_requests: Maximum retained requests per session. Defaults to 1000; older entries are removed.- :param headless: Run the browser in headless/hidden (default), or headful/visible mode.- :param disable_resources: Drop requests for unnecessary resources for a speed boost.- Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.- :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).- :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.- :param cookies: Set cookies for the next request.- :param network_idle: Wait for the page until there are no network connections for at least 500 ms.- :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000- :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.- :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.- :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.- :param wait_selector: Wait for a specific CSS selector to be in a specific state.- :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.- :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting- rules. Defaults to the system default locale.- :param timezone_id: Changes the timezone of the browser. Defaults to the system timezone.- :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.- :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.- :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.- :param pierce_shadow: Include open Shadow DOM content in the response HTML. Defaults to False.- :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.- :param google_search: Enabled by default, Scrapling will set a Google referer header.- :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._- :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.- :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.- :param extra_flags: A list of additional browser flags to pass to the browser on launch.- :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.- :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.- """- self.__validate__(**kwargs)- super().__init__(- record_requests=self._config.record_requests,- max_recorded_requests=self._config.max_recorded_requests,- )-- def start(self):- """Create a browser for this instance and context."""- if not self.playwright:- self.playwright = sync_playwright().start()-- try:- if self._config.cdp_url: # pragma: no cover- self.browser = self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)- if not self._config.proxy_rotator and self.browser:- self.context = self.browser.new_context(**self._context_options)- elif self._config.proxy_rotator:- self.browser = self.playwright.chromium.launch(**self._browser_options)- else:- persistent_options = (- self._browser_options | self._context_options | {"user_data_dir": self._user_data_dir}- )- self.context = self.playwright.chromium.launch_persistent_context(**persistent_options)-- if self.context:- self.context = self._initialize_context(self._config, self.context)-- self._is_alive = True- except Exception:- # Clean up playwright if browser setup fails- self.playwright.stop()- self.playwright = None- raise- else:- raise RuntimeError("Session has been already started")-- def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response:- """Opens up the browser and do your request based on your chosen options.-- :param url: The Target url.- :param google_search: Enabled by default, Scrapling will set a Google referer header.- :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000- :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.- :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.- :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.- :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._- :param disable_resources: Drop requests for unnecessary resources for a speed boost.- Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.- :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).- :param wait_selector: Wait for a specific CSS selector to be in a specific state.- :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.- :param network_idle: Wait for the page until there are no network connections for at least 500 ms.- :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.- :param pierce_shadow: Include open Shadow DOM content in the response HTML. Defaults to False.- :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.- :param proxy: Static proxy to override rotator and session proxy. A new browser context will be created and used with it.- :return: A `Response` object.- """- static_proxy = kwargs.pop("proxy", None)-- params = _validate(kwargs, self, PlaywrightConfig)- if not self._is_alive: # pragma: no cover- raise RuntimeError("Context manager has been closed")-- request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set()- referer = (- "https://www.google.com/" if (params.google_search and "referer" not in request_headers_keys) else None- )-- for attempt in range(self._config.retries):- proxy: Optional[ProxyType] = None- if self._config.proxy_rotator and static_proxy is None:- proxy = self._config.proxy_rotator.get_proxy()- else:- proxy = static_proxy-- with self._page_generator(- params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains- ) as page_info:- final_response: List = [None]- page = page_info.page- handler = self._create_response_handler(page_info, final_response)- page.on("response", handler)- try:- if params.page_setup:- try:- params.page_setup(page)- except Exception as e: # pragma: no cover- log.error(f"Error executing page_setup: {e}")-- try:- first_response = page.goto(url, referer=referer)- self._wait_for_page_stability(page, params.load_dom, params.network_idle)-- if not first_response:- raise RuntimeError(f"Failed to get response for {url}")-- if params.page_action:- try:- _ = params.page_action(page)- except Exception as e: # pragma: no cover- log.error(f"Error executing page_action: {e}")-- if params.wait_selector:- try:- waiter: Locator = page.locator(params.wait_selector)- waiter.first.wait_for(state=params.wait_selector_state)- self._wait_for_page_stability(page, params.load_dom, params.network_idle)- except Exception as e: # pragma: no cover- log.error(f"Error waiting for selector {params.wait_selector}: {e}")-- page.wait_for_timeout(params.wait)-- response = ResponseFactory.from_playwright_response(- page,- first_response,- final_response[0],- params.selector_config,- meta={"proxy": proxy},- pierce_shadow=params.pierce_shadow,- )- return response-- except Exception as e:- page_info.mark_error()- if attempt < self._config.retries - 1:- if is_proxy_error(e):- log.warning(- f"Proxy '{proxy}' failed (attempt {attempt + 1}) | Retrying in {self._config.retry_delay}s..."- )- else:- log.warning(- f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s..."- )- time_sleep(self._config.retry_delay)- else:- log.error(f"Failed after {self._config.retries} attempts: {e}")- raise- finally:- page.remove_listener("response", handler)-- raise RuntimeError("Request failed") # pragma: no cover---class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):- """An async Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory."""-- __slots__ = (- "_config",- "_context_options",- "_browser_options",- "_user_data_dir",- "_headers_keys",- )-- def __init__(self, **kwargs: Unpack[PlaywrightSession]):- """A Browser session manager with page pooling-- :param record_requests: Save completed browser requests and responses. Failed or unfinished requests are omitted. Defaults to False.- :param max_recorded_requests: Maximum retained requests per session. Defaults to 1000; older entries are removed.- :param headless: Run the browser in headless/hidden (default), or headful/visible mode.- :param disable_resources: Drop requests for unnecessary resources for a speed boost.- Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.- :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).- :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.- :param cookies: Set cookies for the next request.- :param network_idle: Wait for the page until there are no network connections for at least 500 ms.- :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.- :param pierce_shadow: Include open Shadow DOM content in the response HTML. Defaults to False.- :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000- :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.- :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.- :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.- :param wait_selector: Wait for a specific CSS selector to be in a specific state.- :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.- :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting- rules. Defaults to the system default locale.- :param timezone_id: Changes the timezone of the browser. Defaults to the system timezone.- :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.- :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.- :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.- :param google_search: Enabled by default, Scrapling will set a Google referer header.- :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._- :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.- :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.- :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.- :param extra_flags: A list of additional browser flags to pass to the browser on launch.- :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.- :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.- """- self.__validate__(**kwargs)- super().__init__(- max_pages=self._config.max_pages,- record_requests=self._config.record_requests,- max_recorded_requests=self._config.max_recorded_requests,- )-- async def start(self) -> None:- """Create a browser for this instance and context."""- if not self.playwright:- self.playwright = await async_playwright().start()- try:- if self._config.cdp_url:- self.browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)- if not self._config.proxy_rotator and self.browser:- self.context = await self.browser.new_context(**self._context_options)- elif self._config.proxy_rotator:- self.browser = await self.playwright.chromium.launch(**self._browser_options)- else:- persistent_options = (- self._browser_options | self._context_options | {"user_data_dir": self._user_data_dir}- )- self.context = await self.playwright.chromium.launch_persistent_context(**persistent_options)-- if self.context:- self.context = await self._initialize_context(self._config, self.context)-- self._is_alive = True- except Exception:- # Clean up playwright if browser setup fails- await self.playwright.stop()- self.playwright = None- raise- else:- raise RuntimeError("Session has been already started")-- async def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response:- """Opens up the browser and do your request based on your chosen options.-- :param url: The Target url.- :param google_search: Enabled by default, Scrapling will set a Google referer header.- :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000- :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.- :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.- :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.- :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._- :param disable_resources: Drop requests for unnecessary resources for a speed boost.- Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.- :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).- :param wait_selector: Wait for a specific CSS selector to be in a specific state.- :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.- :param network_idle: Wait for the page until there are no network connections for at least 500 ms.- :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.- :param pierce_shadow: Include open Shadow DOM content in the response HTML. Defaults to False.- :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.- :param proxy: Static proxy to override rotator and session proxy. A new browser context will be created and used with it.- :return: A `Response` object.- """- static_proxy = kwargs.pop("proxy", None)-- params = _validate(kwargs, self, PlaywrightConfig)-- if not self._is_alive: # pragma: no cover- raise RuntimeError("Context manager has been closed")-- request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set()- referer = (- "https://www.google.com/" if (params.google_search and "referer" not in request_headers_keys) else None- )-- for attempt in range(self._config.retries):- proxy: Optional[ProxyType] = None- if self._config.proxy_rotator and static_proxy is None:- proxy = self._config.proxy_rotator.get_proxy()- else:- proxy = static_proxy-- async with self._page_generator(- params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains- ) as page_info:- final_response: List = [None]- page = page_info.page- handler = self._create_response_handler(page_info, final_response)- page.on("response", handler)- try:- if params.page_setup:- try:- await params.page_setup(page)- except Exception as e: # pragma: no cover- log.error(f"Error executing page_setup: {e}")-- try:- first_response = await page.goto(url, referer=referer)- await self._wait_for_page_stability(page, params.load_dom, params.network_idle)-- if not first_response:- raise RuntimeError(f"Failed to get response for {url}")-- if params.page_action:- try:- _ = await params.page_action(page)- except Exception as e: # pragma: no cover- log.error(f"Error executing page_action: {e}")-- if params.wait_selector:- try:- waiter: AsyncLocator = page.locator(params.wait_selector)- await waiter.first.wait_for(state=params.wait_selector_state)- await self._wait_for_page_stability(page, params.load_dom, params.network_idle)- except Exception as e: # pragma: no cover- log.error(f"Error waiting for selector {params.wait_selector}: {e}")-- await page.wait_for_timeout(params.wait)-- response = await ResponseFactory.from_async_playwright_response(- page,- first_response,- final_response[0],- params.selector_config,- meta={"proxy": proxy},- pierce_shadow=params.pierce_shadow,- )- return response-- except Exception as e:- page_info.mark_error()- if attempt < self._config.retries - 1:- if is_proxy_error(e):- log.warning(- f"Proxy '{proxy}' failed (attempt {attempt + 1}) | Retrying in {self._config.retry_delay}s..."- )- else:- log.warning(- f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s..."- )- await asyncio_sleep(self._config.retry_delay)- else:- log.error(f"Failed after {self._config.retries} attempts: {e}")- raise- finally:- page.remove_listener("response", handler)-- raise RuntimeError("Request failed") # pragma: no coverdiff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.pyindex 2b32e35..6bf3b66 100644--- a/scrapling/engines/_browsers/_page.py+++ b/scrapling/engines/_browsers/_page.py@@ -1,8 +1,8 @@from threading import RLockfrom dataclasses import dataclass-from playwright.sync_api._generated import Page as SyncPage-from playwright.async_api._generated import Page as AsyncPage+from patchright.sync_api import Page as SyncPage+from patchright.async_api import Page as AsyncPagefrom scrapling.core._types import Optional, List, Literal, overload, TypeVar, Generic, castdiff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.pyindex ff86e73..875b5cb 100644--- a/scrapling/engines/_browsers/_stealth.py+++ b/scrapling/engines/_browsers/_stealth.py@@ -3,8 +3,8 @@ from re import compile as re_compilefrom time import sleep as time_sleepfrom asyncio import sleep as asyncio_sleep-from playwright.sync_api import Locator, Page-from playwright.async_api import Page as async_Page, Locator as AsyncLocator+from patchright.sync_api import Locator, Page+from patchright.async_api import Page as async_Page, Locator as AsyncLocatorfrom patchright.sync_api import sync_playwrightfrom patchright.async_api import async_playwrightdiff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.pyindex 72cc089..4030188 100644--- a/scrapling/engines/toolbelt/convertor.py+++ b/scrapling/engines/toolbelt/convertor.py@@ -4,10 +4,9 @@ from functools import lru_cachefrom lxml.etree import LxmlErrorfrom curl_cffi.requests import Response as CurlResponse-from playwright._impl._errors import Error as PlaywrightError-from patchright._impl._errors import Error as PatchrightError-from playwright.sync_api import Page as SyncPage, Response as SyncResponse-from playwright.async_api import Page as AsyncPage, Response as AsyncResponse+from patchright.sync_api import Error as PatchrightError+from patchright.sync_api import Page as SyncPage, Response as SyncResponse+from patchright.async_api import Page as AsyncPage, Response as AsyncResponsefrom scrapling.core.utils import logfrom .custom import Response, StatusText@@ -175,7 +174,7 @@ class ResponseFactory:are also derived from the provided response headers or reasonable defaults.Additionally, the page content and cookies are extracted for further use.- :param page: A synchronous Playwright `Page` instance that represents the current browser page. Required to retrieve the page's URL, cookies, and content.+ :param page: A synchronous Patchright `Page` instance that represents the current browser page. Required to retrieve the page's URL, cookies, and content.:param final_response: The last response received for the given request from the Playwright instance. Typically used as the main response object to derive status, headers, and other metadata.:param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one.:param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into@@ -310,7 +309,7 @@ class ResponseFactory:for _ in range(max_retries):try:return page.content() or ""- except (PlaywrightError, PatchrightError):+ except PatchrightError:page.wait_for_timeout(500)raise RuntimeError(f"Failed to retrieve the page content after retrying for {max_retries * 500}ms.")@@ -325,7 +324,7 @@ class ResponseFactory:for _ in range(max_retries):try:return (await page.content()) or ""- except (PlaywrightError, PatchrightError):+ except PatchrightError:await page.wait_for_timeout(500)raise RuntimeError(f"Failed to retrieve the page content after retrying for {max_retries * 500}ms.")@@ -350,7 +349,7 @@ class ResponseFactory:are also derived from the provided response headers or reasonable defaults.Additionally, the page content and cookies are extracted for further use.- :param page: An asynchronous Playwright `Page` instance that represents the current browser page. Required to retrieve the page's URL, cookies, and content.+ :param page: An asynchronous Patchright `Page` instance that represents the current browser page. Required to retrieve the page's URL, cookies, and content.:param final_response: The last response received for the given request from the Playwright instance. Typically used as the main response object to derive status, headers, and other metadata.:param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one.:param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked intodiff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.pyindex 80b004d..16e1920 100644--- a/scrapling/engines/toolbelt/fingerprints.py+++ b/scrapling/engines/toolbelt/fingerprints.py@@ -43,10 +43,10 @@ def get_os_name() -> OSName | Tuple:@lru_cache(2, typed=True)-def driven_browser_version(package: str = "playwright") -> int | None:+def driven_browser_version(package: str = "patchright") -> int | None:"""Get the Chromium major version the installed automation package drives, read from its bundled `browsers.json`.- :param package: The automation package to inspect, e.g. `"playwright"` or `"patchright"`+ :param package: The automation package to inspect. Defaults to `"patchright"`:return: The Chromium major version, or `None` if it couldn't be determined"""spec = find_spec(package)@@ -66,7 +66,7 @@ def driven_browser_version(package: str = "playwright") -> int | None:def generate_headers(browser_mode: bool | str = False) -> Dict:"""Generate real browser-like headers using browserforge's generator- :param browser_mode: If enabled, the headers created are used for playwright, so it has to match everything+ :param browser_mode: If enabled, the headers created are used for the browser, so it has to match everything:return: A dictionary of the generated headers"""os_name = get_os_name()@@ -84,7 +84,7 @@ def generate_headers(browser_mode: bool | str = False) -> Dict:headers = HeaderGenerator(browser=[Browser(name=b.name) for b in browsers], os=os_name, device=device).generate()- if browser_mode and (driven := driven_browser_version() or driven_browser_version("patchright")):+ if browser_mode and (driven := driven_browser_version()):if user_agent := headers.get("User-Agent"):headers["User-Agent"] = re.sub(r"Chrome/[\d.]+", f"Chrome/{driven}.0.0.0", user_agent)return headersdiff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.pyindex 01bee07..c23532a 100644--- a/scrapling/engines/toolbelt/navigation.py+++ b/scrapling/engines/toolbelt/navigation.py@@ -4,9 +4,9 @@ Functions related to files and URLsfrom urllib.parse import urlparse-from playwright.async_api import Route as async_Route+from patchright.async_api import Route as async_Routefrom msgspec import Struct, structs, convert, ValidationError-from playwright.sync_api import Route+from patchright.sync_api import Routefrom scrapling.core.utils import logfrom scrapling.core._types import Dict, Set, Tuple, Optional, Callablediff --git a/scrapling/fetchers/__init__.py b/scrapling/fetchers/__init__.pyindex 7db355d..1ef500e 100644--- a/scrapling/fetchers/__init__.py+++ b/scrapling/fetchers/__init__.py@@ -3,7 +3,6 @@ from scrapling.engines.toolbelt import ProxyRotatorif TYPE_CHECKING:from scrapling.fetchers.requests import Fetcher, AsyncFetcher, FetcherSession- from scrapling.fetchers.chrome import DynamicFetcher, DynamicSession, AsyncDynamicSessionfrom scrapling.fetchers.stealth_chrome import StealthyFetcher, StealthySession, AsyncStealthySession@@ -12,9 +11,6 @@ _LAZY_IMPORTS = {"Fetcher": ("scrapling.fetchers.requests", "Fetcher"),"AsyncFetcher": ("scrapling.fetchers.requests", "AsyncFetcher"),"FetcherSession": ("scrapling.fetchers.requests", "FetcherSession"),- "DynamicFetcher": ("scrapling.fetchers.chrome", "DynamicFetcher"),- "DynamicSession": ("scrapling.fetchers.chrome", "DynamicSession"),- "AsyncDynamicSession": ("scrapling.fetchers.chrome", "AsyncDynamicSession"),"StealthyFetcher": ("scrapling.fetchers.stealth_chrome", "StealthyFetcher"),"StealthySession": ("scrapling.fetchers.stealth_chrome", "StealthySession"),"AsyncStealthySession": ("scrapling.fetchers.stealth_chrome", "AsyncStealthySession"),@@ -25,9 +21,6 @@ __all__ = ["AsyncFetcher","ProxyRotator","FetcherSession",- "DynamicFetcher",- "DynamicSession",- "AsyncDynamicSession","StealthyFetcher","StealthySession","AsyncStealthySession",diff --git a/scrapling/fetchers/chrome.py b/scrapling/fetchers/chrome.pydeleted file mode 100644index b03285a..0000000--- a/scrapling/fetchers/chrome.py+++ /dev/null@@ -1,99 +0,0 @@-from scrapling.core._types import Unpack-from scrapling.engines._browsers._types import PlaywrightSession-from scrapling.engines.toolbelt.custom import BaseFetcher, Response-from scrapling.engines._browsers._controllers import DynamicSession, AsyncDynamicSession---class DynamicFetcher(BaseFetcher):- """A `Fetcher` that provide many options to fetch/load websites' pages through chromium-based browsers."""-- @classmethod- def fetch(cls, url: str, **kwargs: Unpack[PlaywrightSession]) -> Response:- """Opens up a browser and do your request based on your chosen options below.-- :param url: Target url.- :param headless: Run the browser in headless/hidden (default), or headful/visible mode.- :param disable_resources: Drop requests for unnecessary resources for a speed boost.- :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).- :param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.- :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.- :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.- :param cookies: Set cookies for the next request.- :param network_idle: Wait for the page until there are no network connections for at least 500 ms.- :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.- :param pierce_shadow: Include open Shadow DOM content in the response HTML. Defaults to False.- :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000- :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object.- :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.- :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.- :param wait_selector: Wait for a specific CSS selector to be in a specific state.- :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.- :param locale: Set the locale for the browser if wanted. Defaults to the system default locale.- :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.- :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.- :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.- :param google_search: Enabled by default, Scrapling will set a Google referer header.- :param extra_headers: A dictionary of extra headers to add to the request.- :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.- :param extra_flags: A list of additional browser flags to pass to the browser on launch.- :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.- :param additional_args: Additional arguments to be passed to Playwright's context as additional settings.- :return: A `Response` object.- """- selector_config = kwargs.get("selector_config", {}) or kwargs.get(- "custom_config", {}- ) # Checking `custom_config` for backward compatibility- if not isinstance(selector_config, dict):- raise TypeError("Argument `selector_config` must be a dictionary.")-- kwargs["selector_config"] = {**cls._generate_parser_arguments(), **selector_config}-- with DynamicSession(**kwargs) as session:- return session.fetch(url)-- @classmethod- async def async_fetch(cls, url: str, **kwargs: Unpack[PlaywrightSession]) -> Response:- """Opens up a browser and do your request based on your chosen options below.-- :param url: Target url.- :param headless: Run the browser in headless/hidden (default), or headful/visible mode.- :param disable_resources: Drop requests for unnecessary resources for a speed boost.- :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched (e.g., ``"example.com"`` blocks ``"sub.example.com"`` too).- :param block_ads: Block requests to ~3,500 known ad/tracking domains. Can be combined with ``blocked_domains``.- :param dns_over_https: Route DNS queries through Cloudflare's DNS-over-HTTPS to prevent DNS leaks when using proxies.- :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.- :param cookies: Set cookies for the next request.- :param network_idle: Wait for the page until there are no network connections for at least 500 ms.- :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.- :param pierce_shadow: Include open Shadow DOM content in the response HTML. Defaults to False.- :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000- :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object.- :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.- :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.- :param wait_selector: Wait for a specific CSS selector to be in a specific state.- :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.- :param locale: Set the locale for the browser if wanted. Defaults to the system default locale.- :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.- :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.- :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.- :param google_search: Enabled by default, Scrapling will set a Google referer header.- :param extra_headers: A dictionary of extra headers to add to the request.- :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.- :param extra_flags: A list of additional browser flags to pass to the browser on launch.- :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.- :param additional_args: Additional arguments to be passed to Playwright's context as additional settings.- :return: A `Response` object.- """- selector_config = kwargs.get("selector_config", {}) or kwargs.get(- "custom_config", {}- ) # Checking `custom_config` for backward compatibility- if not isinstance(selector_config, dict):- raise TypeError("Argument `selector_config` must be a dictionary.")-- kwargs["selector_config"] = {**cls._generate_parser_arguments(), **selector_config}-- async with AsyncDynamicSession(**kwargs) as session:- return await session.fetch(url)---PlayWrightFetcher = DynamicFetcher # For backward-compatibilitydiff --git a/scrapling/spiders/session.py b/scrapling/spiders/session.pyindex 513b423..89e008b 100644--- a/scrapling/spiders/session.py+++ b/scrapling/spiders/session.py@@ -4,9 +4,9 @@ from scrapling.spiders.request import Requestfrom scrapling.engines.static import _ASyncSessionLogicfrom scrapling.engines.toolbelt.convertor import Responsefrom scrapling.core._types import Set, cast, SUPPORTED_HTTP_METHODS-from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, FetcherSession+from scrapling.fetchers import AsyncStealthySession, FetcherSession-Session = FetcherSession | AsyncDynamicSession | AsyncStealthySession+Session = FetcherSession | AsyncStealthySessionclass SessionManager:diff --git a/zensical.toml b/zensical.tomlindex 980bcb7..03c8b40 100644--- a/zensical.toml+++ b/zensical.toml@@ -27,8 +27,7 @@ nav = [{Fetching = [{"Fetchers basics" = "fetching/choosing.md"},{"HTTP requests" = "fetching/static.md"},- {"Dynamic websites" = "fetching/dynamic.md"},- {"Dynamic websites with hard protections" = "fetching/stealthy.md"}+ {"Browser fetching" = "fetching/stealthy.md"}]},{Spiders = [{"Architecture" = "spiders/architecture.md"},