Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(StealthyFetcher): add the ability to add `init_script` to context

Karim shoair9543f171aa2f
diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py
index 55f185d..d6150e4 100644
--- a/scrapling/engines/_browsers/_camoufox.py
+++ b/scrapling/engines/_browsers/_camoufox.py
@@ -60,6 +60,7 @@ class StealthySession:
"timeout",
"page_action",
"wait_selector",
+ "init_script",
"addons",
"wait_selector_state",
"cookies",
@@ -95,6 +96,7 @@ class StealthySession:
timeout: int | float = 30000,
page_action: Optional[Callable] = None,
wait_selector: Optional[str] = None,
+ init_script: Optional[str] = None,
addons: Optional[List[str]] = None,
wait_selector_state: SelectorWaitStates = "attached",
cookies: Optional[List[Dict]] = None,
@@ -128,6 +130,7 @@ class StealthySession:
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
+ :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
:param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.
It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
@@ -153,6 +156,7 @@ class StealthySession:
"timeout": timeout,
"page_action": page_action,
"wait_selector": wait_selector,
+ "init_script": init_script,
"addons": addons,
"wait_selector_state": wait_selector_state,
"cookies": cookies,
@@ -180,6 +184,7 @@ class StealthySession:
self.timeout = config.timeout
self.page_action = config.page_action
self.wait_selector = config.wait_selector
+ self.init_script = config.init_script
self.addons = config.addons
self.wait_selector_state = config.wait_selector_state
self.cookies = config.cookies
@@ -234,6 +239,9 @@ class StealthySession:
**self.launch_options
)
)
+ if self.init_script: # pragma: no cover
+ self.context.add_init_script(path=self.init_script)
+
if self.cookies: # pragma: no cover
self.context.add_cookies(self.cookies)
@@ -474,6 +482,7 @@ class AsyncStealthySession(StealthySession):
timeout: int | float = 30000,
page_action: Optional[Callable] = None,
wait_selector: Optional[str] = None,
+ init_script: Optional[str] = None,
addons: Optional[List[str]] = None,
wait_selector_state: SelectorWaitStates = "attached",
cookies: Optional[List[Dict]] = None,
@@ -507,6 +516,7 @@ class AsyncStealthySession(StealthySession):
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
+ :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
:param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.
It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
@@ -531,6 +541,7 @@ class AsyncStealthySession(StealthySession):
timeout,
page_action,
wait_selector,
+ init_script,
addons,
wait_selector_state,
cookies,
@@ -557,6 +568,9 @@ class AsyncStealthySession(StealthySession):
**self.launch_options
)
)
+ if self.init_script: # pragma: no cover
+ await self.context.add_init_script(path=self.init_script)
+
if self.cookies:
await self.context.add_cookies(self.cookies)
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index 24bd314..d2b321e 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -90,6 +90,7 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
solve_cloudflare: bool = False
wait: int | float = 0
timeout: int | float = 30000
+ init_script: Optional[str] = None
page_action: Optional[Callable] = None
wait_selector: Optional[str] = None
addons: Optional[List[str]] = None
@@ -131,6 +132,15 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
f"Addon's path is not a folder, you need to pass a folder of the extracted addon: {addon}"
)
+ if self.init_script is not None:
+ script_path = Path(self.init_script)
+ if not script_path.exists():
+ raise ValueError("Init script path not found")
+ elif not script_path.is_file():
+ raise ValueError("Init script is not a file")
+ elif not script_path.is_absolute():
+ raise ValueError("Init script is not a absolute path")
+
if not self.cookies:
self.cookies = []
if self.solve_cloudflare and self.timeout < 60_000:
diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py
index 055ea33..054906c 100644
--- a/scrapling/fetchers.py
+++ b/scrapling/fetchers.py
@@ -62,6 +62,7 @@ class StealthyFetcher(BaseFetcher):
timeout: int | float = 30000,
page_action: Optional[Callable] = None,
wait_selector: Optional[str] = None,
+ init_script: Optional[str] = None,
addons: Optional[List[str]] = None,
wait_selector_state: SelectorWaitStates = "attached",
cookies: Optional[List[Dict]] = None,
@@ -97,6 +98,7 @@ class StealthyFetcher(BaseFetcher):
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
+ :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.
:param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.
It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
@@ -127,6 +129,7 @@ class StealthyFetcher(BaseFetcher):
disable_ads=disable_ads,
allow_webgl=allow_webgl,
page_action=page_action,
+ init_script=init_script,
network_idle=network_idle,
block_images=block_images,
block_webrtc=block_webrtc,
@@ -158,6 +161,7 @@ class StealthyFetcher(BaseFetcher):
timeout: int | float = 30000,
page_action: Optional[Callable] = None,
wait_selector: Optional[str] = None,
+ init_script: Optional[str] = None,
addons: Optional[List[str]] = None,
wait_selector_state: SelectorWaitStates = "attached",
cookies: Optional[List[Dict]] = None,
@@ -193,6 +197,7 @@ class StealthyFetcher(BaseFetcher):
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
+ :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.
:param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.
It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
@@ -223,6 +228,7 @@ class StealthyFetcher(BaseFetcher):
disable_ads=disable_ads,
allow_webgl=allow_webgl,
page_action=page_action,
+ init_script=init_script,
network_idle=network_idle,
block_images=block_images,
block_webrtc=block_webrtc,