Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(browsers): add a pre-navigation hook to allow page setup

Karim shoair5e13d3ece6d9
diff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.py
index b2340fb..00f1fe9 100644
--- a/scrapling/core/_shell_signatures.py
+++ b/scrapling/core/_shell_signatures.py
@@ -47,6 +47,7 @@ _FETCH_PARAMS = {
"wait": int | float,
"timezone_id": str | None,
"page_action": Optional[Callable],
+ "page_setup": Optional[Callable],
"proxy": Optional[str | Dict[str, str] | Tuple],
"extra_headers": Optional[Dict[str, str]],
"timeout": int | float,
@@ -80,6 +81,7 @@ _STEALTHY_FETCH_PARAMS = {
"wait": int | float,
"timezone_id": str | None,
"page_action": Optional[Callable],
+ "page_setup": Optional[Callable],
"proxy": Optional[str | Dict[str, str] | Tuple],
"extra_headers": Optional[Dict[str, str]],
"timeout": int | float,
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index ce4b643..edabb17 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -47,7 +47,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
@@ -105,7 +106,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
:param google_search: Enabled by default, Scrapling will set a Google referer header.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
@@ -152,6 +154,12 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
),
)
+ if params.page_setup:
+ try:
+ params.page_setup(page)
+ except Exception as e: # pragma: no cover
+ log.error(f"Error executing page_setup: {e}")
+
try:
first_response = page.goto(url, referer=referer)
self._wait_for_page_stability(page, params.load_dom, params.network_idle)
@@ -228,7 +236,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
@@ -285,7 +294,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
:param google_search: Enabled by default, Scrapling will set a Google referer header.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
@@ -333,6 +343,12 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
),
)
+ if params.page_setup:
+ try:
+ await params.page_setup(page)
+ except Exception as e: # pragma: no cover
+ log.error(f"Error executing page_setup: {e}")
+
try:
first_response = await page.goto(url, referer=referer)
await self._wait_for_page_stability(page, params.load_dom, params.network_idle)
diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py
index f06c62e..efa195e 100644
--- a/scrapling/engines/_browsers/_stealth.py
+++ b/scrapling/engines/_browsers/_stealth.py
@@ -47,7 +47,8 @@ class StealthySession(SyncSession, StealthySessionMixin):
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
@@ -187,7 +188,8 @@ class StealthySession(SyncSession, StealthySessionMixin):
:param google_search: Enabled by default, Scrapling will set a Google referer header.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
@@ -235,6 +237,12 @@ class StealthySession(SyncSession, StealthySessionMixin):
),
)
+ if params.page_setup:
+ try:
+ params.page_setup(page)
+ except Exception as e: # pragma: no cover
+ log.error(f"Error executing page_setup: {e}")
+
try:
first_response = page.goto(url, referer=referer)
self._wait_for_page_stability(page, params.load_dom, params.network_idle)
@@ -315,7 +323,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
@@ -454,7 +463,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
:param google_search: Enabled by default, Scrapling will set a Google referer header.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
@@ -503,6 +513,12 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
),
)
+ if params.page_setup:
+ try:
+ await params.page_setup(page)
+ except Exception as e: # pragma: no cover
+ log.error(f"Error executing page_setup: {e}")
+
try:
first_response = await page.goto(url, referer=referer)
await self._wait_for_page_stability(page, params.load_dom, params.network_idle)
diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py
index 5030480..b436c59 100644
--- a/scrapling/engines/_browsers/_types.py
+++ b/scrapling/engines/_browsers/_types.py
@@ -74,6 +74,7 @@ class PlaywrightSession(TypedDict, total=False):
wait: int | float
timezone_id: str | None
page_action: Optional[Callable]
+ page_setup: Optional[Callable]
proxy: Optional[str | Dict[str, str] | Tuple]
proxy_rotator: Optional[ProxyRotator]
extra_headers: Optional[Dict[str, str]]
@@ -105,6 +106,7 @@ class PlaywrightFetchParams(TypedDict, total=False):
disable_resources: bool
wait_selector: Optional[str]
page_action: Optional[Callable]
+ page_setup: Optional[Callable]
selector_config: Optional[Dict]
extra_headers: Optional[Dict[str, str]]
wait_selector_state: SelectorWaitStates
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index 504ba78..77dc81d 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -71,6 +71,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
wait: Seconds = 0
timezone_id: str | None = ""
page_action: Optional[Callable] = None
+ page_setup: Optional[Callable] = None
proxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None`
proxy_rotator: Optional[ProxyRotator] = None
extra_headers: Optional[Dict[str, str]] = None
@@ -96,6 +97,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
"""Custom validation after msgspec validation"""
if self.page_action and not callable(self.page_action):
raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
+ if self.page_setup and not callable(self.page_setup):
+ raise TypeError(f"page_setup must be callable, got {type(self.page_setup).__name__}")
if self.proxy and self.proxy_rotator:
raise ValueError(
"Cannot use 'proxy_rotator' together with 'proxy'. "
@@ -160,6 +163,7 @@ class _fetch_params:
timeout: Seconds
wait: Seconds
page_action: Optional[Callable]
+ page_setup: Optional[Callable]
extra_headers: Optional[Dict[str, str]]
disable_resources: bool
wait_selector: Optional[str]
diff --git a/scrapling/fetchers/chrome.py b/scrapling/fetchers/chrome.py
index f7795b7..3c2efcf 100644
--- a/scrapling/fetchers/chrome.py
+++ b/scrapling/fetchers/chrome.py
@@ -23,7 +23,8 @@ class DynamicFetcher(BaseFetcher):
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object.
- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.
:param locale: Set the locale for the browser if wanted. Defaults to the system default locale.
@@ -65,7 +66,8 @@ class DynamicFetcher(BaseFetcher):
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the Response object.
- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.
:param locale: Set the locale for the browser if wanted. Defaults to the system default locale.
diff --git a/scrapling/fetchers/stealth_chrome.py b/scrapling/fetchers/stealth_chrome.py
index 70ce937..ad6e01f 100644
--- a/scrapling/fetchers/stealth_chrome.py
+++ b/scrapling/fetchers/stealth_chrome.py
@@ -27,7 +27,8 @@ class StealthyFetcher(BaseFetcher):
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting
@@ -78,7 +79,8 @@ class StealthyFetcher(BaseFetcher):
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.
+ :param page_action: Added for automation. A function that takes the `page` object, runs after navigation, and does the automation you need.
+ :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. Locale will affect navigator.language value, Accept-Language request header value as well as number and date formatting