Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

refactor(DynamicSession): Optimization + Removing `max_pages` from sync version

Karim shoair63a9db21a192
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index 1c5dd89..13c0f61 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -80,7 +80,7 @@ class DynamicSession:
def __init__(
self,
- max_pages: int = 1,
+ __max_pages: int = 1,
headless: bool = True,
google_search: bool = True,
hide_canvas: bool = False,
@@ -102,7 +102,7 @@ class DynamicSession:
wait_selector_state: SelectorWaitStates = "attached",
adaptor_arguments: Optional[Dict] = None,
):
- """A Browser session manager with page pooling
+ """A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
@@ -125,12 +125,11 @@ class DynamicSession:
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
- :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
"""
params = {
- "max_pages": max_pages,
+ "max_pages": __max_pages,
"headless": headless,
"google_search": google_search,
"hide_canvas": hide_canvas,
@@ -188,38 +187,46 @@ class DynamicSession:
self.__initiate_browser_options__()
def __initiate_browser_options__(self):
- # `launch_options` is used with persistent context
- self.launch_options = dict(
- _launch_kwargs(
- self.headless,
- self.proxy,
- self.locale,
- tuple(self.extra_headers.items()) if self.extra_headers else tuple(),
- self.useragent,
- self.real_chrome,
- self.stealth,
- self.hide_canvas,
- self.disable_webgl,
+ if self.cdp_url:
+ # `launch_options` is used with persistent context
+ self.launch_options = dict(
+ _launch_kwargs(
+ self.headless,
+ self.proxy,
+ self.locale,
+ tuple(self.extra_headers.items())
+ if self.extra_headers
+ else tuple(),
+ self.useragent,
+ self.real_chrome,
+ self.stealth,
+ self.hide_canvas,
+ self.disable_webgl,
+ )
)
- )
- self.launch_options["extra_http_headers"] = dict(
- self.launch_options["extra_http_headers"]
- )
- self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None
- # while `context_options` is left to be used when cdp mode is enabled
- self.context_options = dict(
- _context_kwargs(
- self.proxy,
- self.locale,
- tuple(self.extra_headers.items()) if self.extra_headers else tuple(),
- self.useragent,
- self.stealth,
+ self.launch_options["extra_http_headers"] = dict(
+ self.launch_options["extra_http_headers"]
)
- )
- self.context_options["extra_http_headers"] = dict(
- self.context_options["extra_http_headers"]
- )
- self.context_options["proxy"] = dict(self.context_options["proxy"]) or None
+ self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None
+ self.context_options = dict()
+ else:
+ # while `context_options` is left to be used when cdp mode is enabled
+ self.launch_options = dict()
+ self.context_options = dict(
+ _context_kwargs(
+ self.proxy,
+ self.locale,
+ tuple(self.extra_headers.items())
+ if self.extra_headers
+ else tuple(),
+ self.useragent,
+ self.stealth,
+ )
+ )
+ self.context_options["extra_http_headers"] = dict(
+ self.context_options["extra_http_headers"]
+ )
+ self.context_options["proxy"] = dict(self.context_options["proxy"]) or None
def __create__(self):
"""Create a browser for this instance and context."""
@@ -386,7 +393,7 @@ class DynamicSession:
class AsyncDynamicSession(DynamicSession):
- """A Browser session manager with page pooling"""
+ """An async Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory."""
def __init__(
self,