Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(DynamicFetcher): add the ability to add `init_script` to context (#56)

Karim shoair6ad1bab2e4d6
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index 285553f..2747ed9 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -60,6 +60,7 @@ class DynamicSession:
"disable_resources",
"network_idle",
"wait_selector",
+ "init_script",
"wait_selector_state",
"wait",
"playwright",
@@ -94,6 +95,7 @@ class DynamicSession:
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
+ init_script: Optional[str] = None,
cookies: Optional[List[Dict]] = None,
network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached",
@@ -112,6 +114,7 @@ class DynamicSession:
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
+ :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
@@ -143,6 +146,7 @@ class DynamicSession:
"selector_config": selector_config,
"disable_resources": disable_resources,
"wait_selector": wait_selector,
+ "init_script": init_script,
"cookies": cookies,
"network_idle": network_idle,
"wait_selector_state": wait_selector_state,
@@ -168,6 +172,7 @@ class DynamicSession:
self.cdp_url = config.cdp_url
self.network_idle = config.network_idle
self.wait_selector = config.wait_selector
+ self.init_script = config.init_script
self.wait_selector_state = config.wait_selector_state
self.playwright: Optional[Playwright] = None
@@ -243,6 +248,9 @@ class DynamicSession:
user_data_dir="", **self.launch_options
)
+ if self.init_script: # pragma: no cover
+ self.context.add_init_script(path=self.init_script)
+
if self.cookies: # pragma: no cover
self.context.add_cookies(self.cookies)
@@ -409,6 +417,7 @@ class AsyncDynamicSession(DynamicSession):
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
+ init_script: Optional[str] = None,
cookies: Optional[List[Dict]] = None,
network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached",
@@ -427,6 +436,7 @@ class AsyncDynamicSession(DynamicSession):
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
+ :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
@@ -459,6 +469,7 @@ class AsyncDynamicSession(DynamicSession):
timeout,
disable_resources,
wait_selector,
+ init_script,
cookies,
network_idle,
wait_selector_state,
@@ -494,6 +505,9 @@ class AsyncDynamicSession(DynamicSession):
)
)
+ if self.init_script: # pragma: no cover
+ await self.context.add_init_script(path=self.init_script)
+
if self.cookies:
await self.context.add_cookies(self.cookies)
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index d2b321e..6363df7 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -32,6 +32,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
extra_headers: Optional[Dict[str, str]] = None
useragent: Optional[str] = None
timeout: int | float = 30000
+ init_script: Optional[str] = None
disable_resources: bool = False
wait_selector: Optional[str] = None
cookies: Optional[List[Dict]] = None
@@ -58,6 +59,15 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
if not self.selector_config:
self.selector_config = {}
+ if self.init_script is not None:
+ script_path = Path(self.init_script)
+ if not script_path.exists():
+ raise ValueError("Init script path not found")
+ elif not script_path.is_file():
+ raise ValueError("Init script is not a file")
+ elif not script_path.is_absolute():
+ raise ValueError("Init script is not a absolute path")
+
@staticmethod
def __validate_cdp(cdp_url):
try:
diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py
index 054906c..c1ce8f7 100644
--- a/scrapling/fetchers.py
+++ b/scrapling/fetchers.py
@@ -282,6 +282,7 @@ class DynamicFetcher(BaseFetcher):
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
+ init_script: Optional[str] = None,
cookies: Optional[Iterable[Dict]] = None,
network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached",
@@ -301,6 +302,7 @@ class DynamicFetcher(BaseFetcher):
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
+ :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
@@ -334,6 +336,7 @@ class DynamicFetcher(BaseFetcher):
real_chrome=real_chrome,
page_action=page_action,
hide_canvas=hide_canvas,
+ init_script=init_script,
network_idle=network_idle,
google_search=google_search,
extra_headers=extra_headers,
@@ -365,6 +368,7 @@ class DynamicFetcher(BaseFetcher):
timeout: int | float = 30000,
disable_resources: bool = False,
wait_selector: Optional[str] = None,
+ init_script: Optional[str] = None,
cookies: Optional[Iterable[Dict]] = None,
network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached",
@@ -384,6 +388,7 @@ class DynamicFetcher(BaseFetcher):
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
+ :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.
:param locale: Set the locale for the browser if wanted. The default value is `en-US`.
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
@@ -418,6 +423,7 @@ class DynamicFetcher(BaseFetcher):
real_chrome=real_chrome,
page_action=page_action,
hide_canvas=hide_canvas,
+ init_script=init_script,
network_idle=network_idle,
google_search=google_search,
extra_headers=extra_headers,