🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(browser fetchers): A new option to control page JS
Karim shoair8c4d6158d6d6
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex 6412698..5bb86ab 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -162,6 +162,7 @@ class DynamicSessionMixin:self.disable_resources = config.disable_resourcesself.cdp_url = config.cdp_urlself.network_idle = config.network_idle+ self.load_dom = config.load_domself.wait_selector = config.wait_selectorself.init_script = config.init_scriptself.wait_selector_state = config.wait_selector_state@@ -216,6 +217,7 @@ class StealthySessionMixin:self.block_webrtc = config.block_webrtcself.allow_webgl = config.allow_webglself.network_idle = config.network_idle+ self.load_dom = config.load_domself.humanize = config.humanizeself.solve_cloudflare = config.solve_cloudflareself.wait = config.waitdiff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.pyindex fcb77ac..31113be 100644--- a/scrapling/engines/_browsers/_camoufox.py+++ b/scrapling/engines/_browsers/_camoufox.py@@ -46,6 +46,7 @@ class StealthySession(StealthySessionMixin, SyncSession):"block_webrtc","allow_webgl","network_idle",+ "load_dom","humanize","solve_cloudflare","wait",@@ -82,6 +83,7 @@ class StealthySession(StealthySessionMixin, SyncSession):block_webrtc: bool = False,allow_webgl: bool = True,network_idle: bool = False,+ load_dom: bool = True,humanize: bool | float = True,solve_cloudflare: bool = False,wait: int | float = 0,@@ -116,6 +118,7 @@ class StealthySession(StealthySessionMixin, SyncSession)::param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.:param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.+ :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.:param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.:param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.@@ -142,6 +145,7 @@ class StealthySession(StealthySessionMixin, SyncSession):cookies=cookies,headless=headless,humanize=humanize,+ load_dom=load_dom,max_pages=__max_pages,disable_ads=disable_ads,allow_webgl=allow_webgl,@@ -259,6 +263,7 @@ class StealthySession(StealthySessionMixin, SyncSession):wait_selector: Optional[str] = _UNSET,wait_selector_state: SelectorWaitStates = _UNSET,network_idle: bool = _UNSET,+ load_dom: bool = _UNSET,solve_cloudflare: bool = _UNSET,selector_config: Optional[Dict] = _UNSET,) -> Response:@@ -276,6 +281,7 @@ class StealthySession(StealthySessionMixin, SyncSession)::param wait_selector: Wait for a specific CSS selector to be in a specific state.:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.+ :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.:param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:return: A `Response` object.@@ -292,6 +298,7 @@ class StealthySession(StealthySessionMixin, SyncSession):wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),+ load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),),@@ -321,7 +328,8 @@ class StealthySession(StealthySessionMixin, SyncSession):# Navigate to URL and wait for a specified statepage_info.page.on("response", handle_response)first_response = page_info.page.goto(url, referer=referer)- page_info.page.wait_for_load_state(state="domcontentloaded")+ if params.load_dom:+ page_info.page.wait_for_load_state(state="domcontentloaded")if params.network_idle:page_info.page.wait_for_load_state("networkidle")@@ -333,7 +341,8 @@ class StealthySession(StealthySessionMixin, SyncSession):self._solve_cloudflare(page_info.page)# Make sure the page is fully loaded after the captchapage_info.page.wait_for_load_state(state="load")- page_info.page.wait_for_load_state(state="domcontentloaded")+ if params.load_dom:+ page_info.page.wait_for_load_state(state="domcontentloaded")if params.network_idle:page_info.page.wait_for_load_state("networkidle")@@ -349,7 +358,8 @@ class StealthySession(StealthySessionMixin, SyncSession):waiter.first.wait_for(state=params.wait_selector_state)# Wait again after waiting for the selector, helpful with protections like Cloudflarepage_info.page.wait_for_load_state(state="load")- page_info.page.wait_for_load_state(state="domcontentloaded")+ if params.load_dom:+ page_info.page.wait_for_load_state(state="domcontentloaded")if params.network_idle:page_info.page.wait_for_load_state("networkidle")except Exception as e:@@ -382,6 +392,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):block_webrtc: bool = False,allow_webgl: bool = True,network_idle: bool = False,+ load_dom: bool = True,humanize: bool | float = True,solve_cloudflare: bool = False,wait: int | float = 0,@@ -416,6 +427,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession)::param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.:param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.+ :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.:param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.:param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.@@ -441,6 +453,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):timeout=timeout,cookies=cookies,headless=headless,+ load_dom=load_dom,humanize=humanize,max_pages=max_pages,disable_ads=disable_ads,@@ -559,6 +572,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):wait_selector: Optional[str] = _UNSET,wait_selector_state: SelectorWaitStates = _UNSET,network_idle: bool = _UNSET,+ load_dom: bool = _UNSET,solve_cloudflare: bool = _UNSET,selector_config: Optional[Dict] = _UNSET,) -> Response:@@ -576,6 +590,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession)::param wait_selector: Wait for a specific CSS selector to be in a specific state.:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.+ :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.:param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:return: A `Response` object.@@ -591,6 +606,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),+ load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),),@@ -620,7 +636,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):# Navigate to URL and wait for a specified statepage_info.page.on("response", handle_response)first_response = await page_info.page.goto(url, referer=referer)- await page_info.page.wait_for_load_state(state="domcontentloaded")+ if params.load_dom:+ await page_info.page.wait_for_load_state(state="domcontentloaded")if params.network_idle:await page_info.page.wait_for_load_state("networkidle")@@ -632,7 +649,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):await self._solve_cloudflare(page_info.page)# Make sure the page is fully loaded after the captchaawait page_info.page.wait_for_load_state(state="load")- await page_info.page.wait_for_load_state(state="domcontentloaded")+ if params.load_dom:+ await page_info.page.wait_for_load_state(state="domcontentloaded")if params.network_idle:await page_info.page.wait_for_load_state("networkidle")@@ -648,7 +666,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):await waiter.first.wait_for(state=params.wait_selector_state)# Wait again after waiting for the selector, helpful with protections like Cloudflareawait page_info.page.wait_for_load_state(state="load")- await page_info.page.wait_for_load_state(state="domcontentloaded")+ if params.load_dom:+ await page_info.page.wait_for_load_state(state="domcontentloaded")if params.network_idle:await page_info.page.wait_for_load_state("networkidle")except Exception as e:diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex a53fa62..96d7291 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -54,6 +54,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):"cookies","disable_resources","network_idle",+ "load_dom","wait_selector","init_script","wait_selector_state",@@ -93,6 +94,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):init_script: Optional[str] = None,cookies: Optional[List[Dict]] = None,network_idle: bool = False,+ load_dom: bool = True,wait_selector_state: SelectorWaitStates = "attached",selector_config: Optional[Dict] = None,):@@ -116,6 +118,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession)::param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.:param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.+ :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._@@ -130,6 +133,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):stealth=stealth,cdp_url=cdp_url,cookies=cookies,+ load_dom=load_dom,headless=headless,useragent=useragent,max_pages=__max_pages,@@ -208,6 +212,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):wait_selector: Optional[str] = _UNSET,wait_selector_state: SelectorWaitStates = _UNSET,network_idle: bool = _UNSET,+ load_dom: bool = _UNSET,selector_config: Optional[Dict] = _UNSET,) -> Response:"""Opens up the browser and do your request based on your chosen options.@@ -224,6 +229,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession)::param wait_selector: Wait for a specific CSS selector to be in a specific state.:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.+ :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:return: A `Response` object."""@@ -239,6 +245,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),+ load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),),PlaywrightConfig,@@ -267,7 +274,8 @@ class DynamicSession(DynamicSessionMixin, SyncSession):# Navigate to URL and wait for a specified statepage_info.page.on("response", handle_response)first_response = page_info.page.goto(url, referer=referer)- page_info.page.wait_for_load_state(state="domcontentloaded")+ if params.load_dom:+ page_info.page.wait_for_load_state(state="domcontentloaded")if params.network_idle:page_info.page.wait_for_load_state("networkidle")@@ -287,7 +295,8 @@ class DynamicSession(DynamicSessionMixin, SyncSession):waiter.first.wait_for(state=params.wait_selector_state)# Wait again after waiting for the selector, helpful with protections like Cloudflarepage_info.page.wait_for_load_state(state="load")- page_info.page.wait_for_load_state(state="domcontentloaded")+ if params.load_dom:+ page_info.page.wait_for_load_state(state="domcontentloaded")if params.network_idle:page_info.page.wait_for_load_state("networkidle")except Exception as e: # pragma: no cover@@ -335,6 +344,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):init_script: Optional[str] = None,cookies: Optional[List[Dict]] = None,network_idle: bool = False,+ load_dom: bool = True,wait_selector_state: SelectorWaitStates = "attached",selector_config: Optional[Dict] = None,):@@ -347,6 +357,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession)::param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.:param cookies: Set cookies for the next request.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.+ :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.@@ -374,6 +385,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):stealth=stealth,cdp_url=cdp_url,cookies=cookies,+ load_dom=load_dom,headless=headless,useragent=useragent,max_pages=max_pages,@@ -453,6 +465,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):wait_selector: Optional[str] = _UNSET,wait_selector_state: SelectorWaitStates = _UNSET,network_idle: bool = _UNSET,+ load_dom: bool = _UNSET,selector_config: Optional[Dict] = _UNSET,) -> Response:"""Opens up the browser and do your request based on your chosen options.@@ -469,6 +482,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession)::param wait_selector: Wait for a specific CSS selector to be in a specific state.:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.+ :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:return: A `Response` object."""@@ -484,6 +498,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),+ load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),),PlaywrightConfig,@@ -512,7 +527,8 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):# Navigate to URL and wait for a specified statepage_info.page.on("response", handle_response)first_response = await page_info.page.goto(url, referer=referer)- await page_info.page.wait_for_load_state(state="domcontentloaded")+ if self.load_dom:+ await page_info.page.wait_for_load_state(state="domcontentloaded")if params.network_idle:await page_info.page.wait_for_load_state("networkidle")@@ -532,7 +548,8 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):await waiter.first.wait_for(state=params.wait_selector_state)# Wait again after waiting for the selector, helpful with protections like Cloudflareawait page_info.page.wait_for_load_state(state="load")- await page_info.page.wait_for_load_state(state="domcontentloaded")+ if self.load_dom:+ await page_info.page.wait_for_load_state(state="domcontentloaded")if params.network_idle:await page_info.page.wait_for_load_state("networkidle")except Exception as e:diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.pyindex c34d6fe..a20ae55 100644--- a/scrapling/engines/_browsers/_validators.py+++ b/scrapling/engines/_browsers/_validators.py@@ -35,6 +35,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):wait_selector: Optional[str] = Nonecookies: Optional[List[Dict]] = Nonenetwork_idle: bool = False+ load_dom: bool = Truewait_selector_state: SelectorWaitStates = "attached"selector_config: Optional[Dict] = None@@ -92,6 +93,7 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):block_webrtc: bool = Falseallow_webgl: bool = Truenetwork_idle: bool = False+ load_dom: bool = Truehumanize: bool | float = Truesolve_cloudflare: bool = Falsewait: int | float = 0diff --git a/scrapling/fetchers.py b/scrapling/fetchers.pyindex fc72de4..2421b42 100644--- a/scrapling/fetchers.py+++ b/scrapling/fetchers.py@@ -56,6 +56,7 @@ class StealthyFetcher(BaseFetcher):block_webrtc: bool = False,allow_webgl: bool = True,network_idle: bool = False,+ load_dom: bool = True,humanize: bool | float = True,solve_cloudflare: bool = False,wait: int | float = 0,@@ -92,6 +93,7 @@ class StealthyFetcher(BaseFetcher)::param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.:param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.+ :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.:param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.:param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.@@ -123,6 +125,7 @@ class StealthyFetcher(BaseFetcher):cookies=cookies,headless=headless,humanize=humanize,+ load_dom=load_dom,disable_ads=disable_ads,allow_webgl=allow_webgl,page_action=page_action,@@ -152,6 +155,7 @@ class StealthyFetcher(BaseFetcher):block_webrtc: bool = False,allow_webgl: bool = True,network_idle: bool = False,+ load_dom: bool = True,humanize: bool | float = True,solve_cloudflare: bool = False,wait: int | float = 0,@@ -188,6 +192,7 @@ class StealthyFetcher(BaseFetcher)::param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.:param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.+ :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.:param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.:param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.@@ -220,6 +225,7 @@ class StealthyFetcher(BaseFetcher):cookies=cookies,headless=headless,humanize=humanize,+ load_dom=load_dom,disable_ads=disable_ads,allow_webgl=allow_webgl,page_action=page_action,@@ -280,6 +286,7 @@ class DynamicFetcher(BaseFetcher):init_script: Optional[str] = None,cookies: Optional[Iterable[Dict]] = None,network_idle: bool = False,+ load_dom: bool = True,wait_selector_state: SelectorWaitStates = "attached",custom_config: Optional[Dict] = None,) -> Response:@@ -293,6 +300,7 @@ class DynamicFetcher(BaseFetcher)::param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.:param cookies: Set cookies for the next request.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.+ :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.@@ -325,6 +333,7 @@ class DynamicFetcher(BaseFetcher):cdp_url=cdp_url,cookies=cookies,headless=headless,+ load_dom=load_dom,useragent=useragent,real_chrome=real_chrome,page_action=page_action,@@ -364,6 +373,7 @@ class DynamicFetcher(BaseFetcher):init_script: Optional[str] = None,cookies: Optional[Iterable[Dict]] = None,network_idle: bool = False,+ load_dom: bool = True,wait_selector_state: SelectorWaitStates = "attached",custom_config: Optional[Dict] = None,) -> Response:@@ -377,6 +387,7 @@ class DynamicFetcher(BaseFetcher)::param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.:param cookies: Set cookies for the next request.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.+ :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.:param page_action: Added for automation. A function that takes the `page` object and does the automation you need.@@ -402,6 +413,7 @@ class DynamicFetcher(BaseFetcher):async with AsyncDynamicSession(wait=wait,+ max_pages=1,proxy=proxy,locale=locale,timeout=timeout,@@ -409,8 +421,8 @@ class DynamicFetcher(BaseFetcher):cdp_url=cdp_url,cookies=cookies,headless=headless,+ load_dom=load_dom,useragent=useragent,- max_pages=1,real_chrome=real_chrome,page_action=page_action,hide_canvas=hide_canvas,