🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(DynamicFetcher): add the ability to add `init_script` to context (#56)
Karim shoair6ad1bab2e4d6
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex 285553f..2747ed9 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -60,6 +60,7 @@ class DynamicSession:"disable_resources","network_idle","wait_selector",+ "init_script","wait_selector_state","wait","playwright",@@ -94,6 +95,7 @@ class DynamicSession:timeout: int | float = 30000,disable_resources: bool = False,wait_selector: Optional[str] = None,+ init_script: Optional[str] = None,cookies: Optional[List[Dict]] = None,network_idle: bool = False,wait_selector_state: SelectorWaitStates = "attached",@@ -112,6 +114,7 @@ class DynamicSession::param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.:param wait_selector: Wait for a specific CSS selector to be in a specific state.+ :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.:param locale: Set the locale for the browser if wanted. The default value is `en-US`.:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.@@ -143,6 +146,7 @@ class DynamicSession:"selector_config": selector_config,"disable_resources": disable_resources,"wait_selector": wait_selector,+ "init_script": init_script,"cookies": cookies,"network_idle": network_idle,"wait_selector_state": wait_selector_state,@@ -168,6 +172,7 @@ class DynamicSession:self.cdp_url = config.cdp_urlself.network_idle = config.network_idleself.wait_selector = config.wait_selector+ self.init_script = config.init_scriptself.wait_selector_state = config.wait_selector_stateself.playwright: Optional[Playwright] = None@@ -243,6 +248,9 @@ class DynamicSession:user_data_dir="", **self.launch_options)+ if self.init_script: # pragma: no cover+ self.context.add_init_script(path=self.init_script)+if self.cookies: # pragma: no coverself.context.add_cookies(self.cookies)@@ -409,6 +417,7 @@ class AsyncDynamicSession(DynamicSession):timeout: int | float = 30000,disable_resources: bool = False,wait_selector: Optional[str] = None,+ init_script: Optional[str] = None,cookies: Optional[List[Dict]] = None,network_idle: bool = False,wait_selector_state: SelectorWaitStates = "attached",@@ -427,6 +436,7 @@ class AsyncDynamicSession(DynamicSession)::param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.:param wait_selector: Wait for a specific CSS selector to be in a specific state.+ :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.:param locale: Set the locale for the browser if wanted. The default value is `en-US`.:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.@@ -459,6 +469,7 @@ class AsyncDynamicSession(DynamicSession):timeout,disable_resources,wait_selector,+ init_script,cookies,network_idle,wait_selector_state,@@ -494,6 +505,9 @@ class AsyncDynamicSession(DynamicSession):))+ if self.init_script: # pragma: no cover+ await self.context.add_init_script(path=self.init_script)+if self.cookies:await self.context.add_cookies(self.cookies)diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.pyindex d2b321e..6363df7 100644--- a/scrapling/engines/_browsers/_validators.py+++ b/scrapling/engines/_browsers/_validators.py@@ -32,6 +32,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):extra_headers: Optional[Dict[str, str]] = Noneuseragent: Optional[str] = Nonetimeout: int | float = 30000+ init_script: Optional[str] = Nonedisable_resources: bool = Falsewait_selector: Optional[str] = Nonecookies: Optional[List[Dict]] = None@@ -58,6 +59,15 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):if not self.selector_config:self.selector_config = {}+ if self.init_script is not None:+ script_path = Path(self.init_script)+ if not script_path.exists():+ raise ValueError("Init script path not found")+ elif not script_path.is_file():+ raise ValueError("Init script is not a file")+ elif not script_path.is_absolute():+ raise ValueError("Init script is not a absolute path")+@staticmethoddef __validate_cdp(cdp_url):try:diff --git a/scrapling/fetchers.py b/scrapling/fetchers.pyindex 054906c..c1ce8f7 100644--- a/scrapling/fetchers.py+++ b/scrapling/fetchers.py@@ -282,6 +282,7 @@ class DynamicFetcher(BaseFetcher):timeout: int | float = 30000,disable_resources: bool = False,wait_selector: Optional[str] = None,+ init_script: Optional[str] = None,cookies: Optional[Iterable[Dict]] = None,network_idle: bool = False,wait_selector_state: SelectorWaitStates = "attached",@@ -301,6 +302,7 @@ class DynamicFetcher(BaseFetcher)::param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.:param wait_selector: Wait for a specific CSS selector to be in a specific state.+ :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.:param locale: Set the locale for the browser if wanted. The default value is `en-US`.:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.@@ -334,6 +336,7 @@ class DynamicFetcher(BaseFetcher):real_chrome=real_chrome,page_action=page_action,hide_canvas=hide_canvas,+ init_script=init_script,network_idle=network_idle,google_search=google_search,extra_headers=extra_headers,@@ -365,6 +368,7 @@ class DynamicFetcher(BaseFetcher):timeout: int | float = 30000,disable_resources: bool = False,wait_selector: Optional[str] = None,+ init_script: Optional[str] = None,cookies: Optional[Iterable[Dict]] = None,network_idle: bool = False,wait_selector_state: SelectorWaitStates = "attached",@@ -384,6 +388,7 @@ class DynamicFetcher(BaseFetcher)::param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.:param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.:param wait_selector: Wait for a specific CSS selector to be in a specific state.+ :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.:param locale: Set the locale for the browser if wanted. The default value is `en-US`.:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.:param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.@@ -418,6 +423,7 @@ class DynamicFetcher(BaseFetcher):real_chrome=real_chrome,page_action=page_action,hide_canvas=hide_canvas,+ init_script=init_script,network_idle=network_idle,google_search=google_search,extra_headers=extra_headers,