🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(fetchers): Add argument to control the user data directory
Karim shoairc34e10faed19
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex f16ef14..ce82dbb 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -149,7 +149,8 @@ class DynamicSessionMixin:self.wait_selector_state = config.wait_selector_stateself.selector_config = config.selector_configself.page_action = config.page_action- self._headers_keys = set(map(str.lower, self.extra_headers.keys())) if self.extra_headers else set()+ self.user_data_dir = config.user_data_dir+ self._headers_keys = {header.lower() for header in self.extra_headers.keys()} if self.extra_headers else set()self.__initiate_browser_options__()def __initiate_browser_options__(self):@@ -173,6 +174,7 @@ class DynamicSessionMixin:)self.launch_options["extra_http_headers"] = dict(self.launch_options["extra_http_headers"])self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None+ self.launch_options["user_data_dir"] = self.user_data_dirself.context_options = dict()else:# while `context_options` is left to be used when cdp mode is enabled@@ -221,7 +223,8 @@ class StealthySessionMixin:self.selector_config = config.selector_configself.additional_args = config.additional_argsself.page_action = config.page_action- self._headers_keys = set(map(str.lower, self.extra_headers.keys())) if self.extra_headers else set()+ self.user_data_dir = config.user_data_dir+ self._headers_keys = {header.lower() for header in self.extra_headers.keys()} if self.extra_headers else set()self.__initiate_browser_options__()def __initiate_browser_options__(self):@@ -239,7 +242,7 @@ class StealthySessionMixin:"block_webrtc": self.block_webrtc,"block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode."os": None if self.os_randomize else get_os_name(),- "user_data_dir": "",+ "user_data_dir": self.user_data_dir,"ff_version": __ff_version_str__,"firefox_user_prefs": {# This is what enabling `enable_cache` does internally, so we do it from here insteaddiff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.pyindex 6e2b620..7076ff8 100644--- a/scrapling/engines/_browsers/_camoufox.py+++ b/scrapling/engines/_browsers/_camoufox.py@@ -104,6 +104,7 @@ class StealthySession(StealthySessionMixin, SyncSession):os_randomize: bool = False,disable_ads: bool = False,geoip: bool = False,+ user_data_dir: str = "",selector_config: Optional[Dict] = None,additional_args: Optional[Dict] = None,):@@ -136,6 +137,7 @@ class StealthySession(StealthySessionMixin, SyncSession)::param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.+ :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings."""@@ -159,6 +161,7 @@ class StealthySession(StealthySessionMixin, SyncSession):block_images=block_images,block_webrtc=block_webrtc,os_randomize=os_randomize,+ user_data_dir=user_data_dir,wait_selector=wait_selector,google_search=google_search,extra_headers=extra_headers,@@ -173,9 +176,7 @@ class StealthySession(StealthySessionMixin, SyncSession):def __create__(self):"""Create a browser for this instance and context."""self.playwright = sync_playwright().start()- self.context = self.playwright.firefox.launch_persistent_context( # pragma: no cover- **self.launch_options- )+ self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options)if self.init_script: # pragma: no coverself.context.add_init_script(path=self.init_script)@@ -226,7 +227,6 @@ class StealthySession(StealthySessionMixin, SyncSession)::param page: The targeted page:return:"""- page.wait_for_load_state("networkidle")challenge_type = self._detect_cloudflare(self._get_page_content(page))if not challenge_type:log.error("No Cloudflare challenge found.")@@ -441,6 +441,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):os_randomize: bool = False,disable_ads: bool = False,geoip: bool = False,+ user_data_dir: str = "",selector_config: Optional[Dict] = None,additional_args: Optional[Dict] = None,):@@ -474,6 +475,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession)::param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.:param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.+ :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings."""@@ -499,6 +501,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):wait_selector=wait_selector,google_search=google_search,extra_headers=extra_headers,+ user_data_dir=user_data_dir,additional_args=additional_args,selector_config=selector_config,solve_cloudflare=solve_cloudflare,@@ -509,8 +512,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):async def __create__(self):"""Create a browser for this instance and context."""- self.playwright: AsyncPlaywright | None = await async_playwright().start()- self.context: AsyncBrowserContext | None = await self.playwright.firefox.launch_persistent_context(+ self.playwright: AsyncPlaywright = await async_playwright().start()+ self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context(**self.launch_options)@@ -534,11 +537,11 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):if self.context:await self.context.close()- self.context = None+ self.context = None # pyright: ignoreif self.playwright:await self.playwright.stop()- self.playwright = None+ self.playwright = None # pyright: ignoreself._closed = True@@ -563,7 +566,6 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession)::param page: The async targeted page:return:"""- await page.wait_for_load_state("networkidle")challenge_type = self._detect_cloudflare(await self._get_page_content(page))if not challenge_type:log.error("No Cloudflare challenge found.")diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex 1fbb9a3..ccfbb11 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -97,6 +97,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):network_idle: bool = False,load_dom: bool = True,wait_selector_state: SelectorWaitStates = "attached",+ user_data_dir: str = "",selector_config: Optional[Dict] = None,):"""A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.@@ -124,6 +125,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession)::param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.+ :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.:param selector_config: The arguments that will be passed in the end while creating the final Selector's class."""self.__validate__(@@ -143,6 +145,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):hide_canvas=hide_canvas,init_script=init_script,network_idle=network_idle,+ user_data_dir=user_data_dir,google_search=google_search,extra_headers=extra_headers,wait_selector=wait_selector,@@ -164,7 +167,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):**self.context_options)else:- self.context = self.playwright.chromium.launch_persistent_context(user_data_dir="", **self.launch_options)+ self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options)if self.init_script: # pragma: no coverself.context.add_init_script(path=self.init_script)@@ -341,6 +344,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):network_idle: bool = False,load_dom: bool = True,wait_selector_state: SelectorWaitStates = "attached",+ user_data_dir: str = "",selector_config: Optional[Dict] = None,):"""A Browser session manager with page pooling@@ -369,6 +373,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession)::param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.:param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.+ :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.:param selector_config: The arguments that will be passed in the end while creating the final Selector's class."""@@ -389,6 +394,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):hide_canvas=hide_canvas,init_script=init_script,network_idle=network_idle,+ user_data_dir=user_data_dir,google_search=google_search,extra_headers=extra_headers,wait_selector=wait_selector,@@ -410,7 +416,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):self.context: AsyncBrowserContext = await browser.new_context(**self.context_options)else:self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(- user_data_dir="", **self.launch_options+ **self.launch_options)if self.init_script: # pragma: no coverdiff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.pyindex 448f0de..af90d9e 100644--- a/scrapling/engines/_browsers/_validators.py+++ b/scrapling/engines/_browsers/_validators.py@@ -87,6 +87,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):network_idle: bool = Falseload_dom: bool = Truewait_selector_state: SelectorWaitStates = "attached"+ user_data_dir: str = ""selector_config: Optional[Dict] = {}def __post_init__(self):@@ -134,6 +135,7 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):os_randomize: bool = Falsedisable_ads: bool = Falsegeoip: bool = False+ user_data_dir: str = ""selector_config: Optional[Dict] = {}additional_args: Optional[Dict] = {}