🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(browsers): add a pre-navigation hook to allow page setup
Karim shoair5e13d3ece6d9
diff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.pyindex b2340fb..00f1fe9 100644--- a/scrapling/core/_shell_signatures.py+++ b/scrapling/core/_shell_signatures.py@@ -47,6 +47,7 @@ _FETCH_PARAMS = {"wait": int | float,"timezone_id": str | None,"page_action": Optional[Callable],+ "page_setup": Optional[Callable],"proxy": Optional[str | Dict[str, str] | Tuple],"extra_headers": Optional[Dict[str, str]],"timeout": int | float,@@ -80,6 +81,7 @@ _STEALTHY_FETCH_PARAMS = {"wait": int | float,"timezone_id": str | None,"page_action": Optional[Callable],+ "page_setup": Optional[Callable],"proxy": Optional[str | Dict[str, str] | Tuple],"extra_headers": Optional[Dict[str, str]],"timeout": int | float,diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex ce4b643..edabb17 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -47,7 +47,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin)::param network_idle: Wait for the page until there are no network connections for at least 500 ms.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.:param wait_selector: Wait for a specific CSS selector to be in a specific state.:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting@@ -105,7 +106,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin)::param google_search: Enabled by default, Scrapling will set a Google referer header.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._:param disable_resources: Drop requests for unnecessary resources for a speed boost.Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.@@ -152,6 +154,12 @@ class DynamicSession(SyncSession, DynamicSessionMixin):),)+ if params.page_setup:+ try:+ params.page_setup(page)+ except Exception as e: # pragma: no cover+ log.error(f"Error executing page_setup: {e}")+try:first_response = page.goto(url, referer=referer)self._wait_for_page_stability(page, params.load_dom, params.network_idle)@@ -228,7 +236,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin)::param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.:param wait_selector: Wait for a specific CSS selector to be in a specific state.:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting@@ -285,7 +294,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin)::param google_search: Enabled by default, Scrapling will set a Google referer header.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._:param disable_resources: Drop requests for unnecessary resources for a speed boost.Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.@@ -333,6 +343,12 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):),)+ if params.page_setup:+ try:+ await params.page_setup(page)+ except Exception as e: # pragma: no cover+ log.error(f"Error executing page_setup: {e}")+try:first_response = await page.goto(url, referer=referer)await self._wait_for_page_stability(page, params.load_dom, params.network_idle)diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.pyindex f06c62e..efa195e 100644--- a/scrapling/engines/_browsers/_stealth.py+++ b/scrapling/engines/_browsers/_stealth.py@@ -47,7 +47,8 @@ class StealthySession(SyncSession, StealthySessionMixin)::param network_idle: Wait for the page until there are no network connections for at least 500 ms.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.:param wait_selector: Wait for a specific CSS selector to be in a specific state.:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting@@ -187,7 +188,8 @@ class StealthySession(SyncSession, StealthySessionMixin)::param google_search: Enabled by default, Scrapling will set a Google referer header.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._:param disable_resources: Drop requests for unnecessary resources for a speed boost.Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.@@ -235,6 +237,12 @@ class StealthySession(SyncSession, StealthySessionMixin):),)+ if params.page_setup:+ try:+ params.page_setup(page)+ except Exception as e: # pragma: no cover+ log.error(f"Error executing page_setup: {e}")+try:first_response = page.goto(url, referer=referer)self._wait_for_page_stability(page, params.load_dom, params.network_idle)@@ -315,7 +323,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin)::param network_idle: Wait for the page until there are no network connections for at least 500 ms.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.:param wait_selector: Wait for a specific CSS selector to be in a specific state.:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting@@ -454,7 +463,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin)::param google_search: Enabled by default, Scrapling will set a Google referer header.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._:param disable_resources: Drop requests for unnecessary resources for a speed boost.Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.@@ -503,6 +513,12 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):),)+ if params.page_setup:+ try:+ await params.page_setup(page)+ except Exception as e: # pragma: no cover+ log.error(f"Error executing page_setup: {e}")+try:first_response = await page.goto(url, referer=referer)await self._wait_for_page_stability(page, params.load_dom, params.network_idle)diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.pyindex 5030480..b436c59 100644--- a/scrapling/engines/_browsers/_types.py+++ b/scrapling/engines/_browsers/_types.py@@ -74,6 +74,7 @@ class PlaywrightSession(TypedDict, total=False):wait: int | floattimezone_id: str | Nonepage_action: Optional[Callable]+ page_setup: Optional[Callable]proxy: Optional[str | Dict[str, str] | Tuple]proxy_rotator: Optional[ProxyRotator]extra_headers: Optional[Dict[str, str]]@@ -105,6 +106,7 @@ class PlaywrightFetchParams(TypedDict, total=False):disable_resources: boolwait_selector: Optional[str]page_action: Optional[Callable]+ page_setup: Optional[Callable]selector_config: Optional[Dict]extra_headers: Optional[Dict[str, str]]wait_selector_state: SelectorWaitStatesdiff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.pyindex 504ba78..77dc81d 100644--- a/scrapling/engines/_browsers/_validators.py+++ b/scrapling/engines/_browsers/_validators.py@@ -71,6 +71,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):wait: Seconds = 0timezone_id: str | None = ""page_action: Optional[Callable] = None+ page_setup: Optional[Callable] = Noneproxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None`proxy_rotator: Optional[ProxyRotator] = Noneextra_headers: Optional[Dict[str, str]] = None@@ -96,6 +97,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):"""Custom validation after msgspec validation"""if self.page_action and not callable(self.page_action):raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")+ if self.page_setup and not callable(self.page_setup):+ raise TypeError(f"page_setup must be callable, got {type(self.page_setup).__name__}")if self.proxy and self.proxy_rotator:raise ValueError("Cannot use 'proxy_rotator' together with 'proxy'. "@@ -160,6 +163,7 @@ class _fetch_params:timeout: Secondswait: Secondspage_action: Optional[Callable]+ page_setup: Optional[Callable]extra_headers: Optional[Dict[str, str]]disable_resources: boolwait_selector: Optional[str]diff --git a/scrapling/fetchers/chrome.py b/scrapling/fetchers/chrome.pyindex f7795b7..3c2efcf 100644--- a/scrapling/fetchers/chrome.py+++ b/scrapling/fetchers/chrome.py@@ -23,7 +23,8 @@ class DynamicFetcher(BaseFetcher)::param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object.- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.:param wait_selector: Wait for a specific CSS selector to be in a specific state.:param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.:param locale: Set the locale for the browser if wanted. Defaults to the system default locale.@@ -65,7 +66,8 @@ class DynamicFetcher(BaseFetcher)::param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object.- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.:param wait_selector: Wait for a specific CSS selector to be in a specific state.:param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.:param locale: Set the locale for the browser if wanted. Defaults to the system default locale.diff --git a/scrapling/fetchers/stealth_chrome.py b/scrapling/fetchers/stealth_chrome.pyindex 70ce937..ad6e01f 100644--- a/scrapling/fetchers/stealth_chrome.py+++ b/scrapling/fetchers/stealth_chrome.py@@ -27,7 +27,8 @@ class StealthyFetcher(BaseFetcher)::param network_idle: Wait for the page until there are no network connections for at least 500 ms.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.:param wait_selector: Wait for a specific CSS selector to be in a specific state.:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting@@ -78,7 +79,8 @@ class StealthyFetcher(BaseFetcher)::param network_idle: Wait for the page until there are no network connections for at least 500 ms.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.:param wait_selector: Wait for a specific CSS selector to be in a specific state.:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting