🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor/feat(browser fetchers): make it possible to have a configuration per page in sessions
Karim shoair9838d741d0de
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex 55b3310..6b91d8f 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -22,6 +22,7 @@ from ._config_tools import _compiled_stealth_scriptsfrom ._validators import validate, PlaywrightConfig, CamoufoxConfigfrom ._config_tools import _launch_kwargs, _context_kwargsfrom scrapling.core._types import (+ Any,Dict,Optional,)@@ -38,7 +39,12 @@ class SyncSession:self.context: Optional[BrowserContext] = Noneself._closed = False- def _get_page(self) -> PageInfo: # pragma: no cover+ def _get_page(+ self,+ timeout: int | float,+ extra_headers: Optional[Dict[str, str]],+ disable_resources: bool,+ ) -> PageInfo: # pragma: no cover"""Get a new page to use"""# Close all finished pages to ensure clean state@@ -59,13 +65,12 @@ class SyncSession:)page = self.context.new_page()- timeout = getattr(self, "timeout", 30000)page.set_default_navigation_timeout(timeout)page.set_default_timeout(timeout)- if getattr(self, "extra_headers", False):- page.set_extra_http_headers(getattr(self, "extra_headers"))+ if extra_headers:+ page.set_extra_http_headers(extra_headers)- if getattr(self, "disable_resources", False):+ if disable_resources:page.route("**/*", intercept_route)if getattr(self, "stealth", False):@@ -74,6 +79,13 @@ class SyncSession:return self.page_pool.add_page(page)+ @staticmethod+ def _get_with_precedence(+ request_value: Any, session_value: Any, sentinel_value: object+ ) -> Any:+ """Get value with request-level priority over session-level"""+ return request_value if request_value is not sentinel_value else session_value+def get_pool_stats(self) -> Dict[str, int]:"""Get statistics about the current page pool"""return {@@ -90,7 +102,12 @@ class AsyncSession(SyncSession):self.context: Optional[AsyncBrowserContext] = Noneself._lock = Lock()- async def _get_page(self) -> PageInfo: # pragma: no cover+ async def _get_page(+ self,+ timeout: int | float,+ extra_headers: Optional[Dict[str, str]],+ disable_resources: bool,+ ) -> PageInfo: # pragma: no cover"""Get a new page to use"""async with self._lock:# Close all finished pages to ensure clean state@@ -111,13 +128,12 @@ class AsyncSession(SyncSession):)page = await self.context.new_page()- timeout = getattr(self, "timeout", 30000)page.set_default_navigation_timeout(timeout)page.set_default_timeout(timeout)- if getattr(self, "extra_headers", False):- await page.set_extra_http_headers(getattr(self, "extra_headers"))+ if extra_headers:+ await page.set_extra_http_headers(extra_headers)- if getattr(self, "disable_resources", False):+ if disable_resources:await page.route("**/*", async_intercept_route)if getattr(self, "stealth", False):@@ -334,7 +350,6 @@ class StealthySessionMixin:self.geoip = config.geoipself.selector_config = config.selector_configself.additional_args = config.additional_args- self.selector_config = config.selector_configself.page_action = config.page_actionself._headers_keys = (set(map(str.lower, self.extra_headers.keys()))diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.pyindex cd814c1..926191a 100644--- a/scrapling/engines/_browsers/_camoufox.py+++ b/scrapling/engines/_browsers/_camoufox.py@@ -31,6 +31,7 @@ from scrapling.engines.toolbelt import ()__CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*")+_UNSET = object()class StealthySession(StealthySessionMixin, SyncSession):@@ -247,19 +248,74 @@ class StealthySession(StealthySessionMixin, SyncSession):log.info("Cloudflare captcha is solved")return- def fetch(self, url: str) -> Response:+ def fetch(+ self,+ url: str,+ google_search: bool = _UNSET,+ timeout: int | float = _UNSET,+ wait: int | float = _UNSET,+ page_action: Optional[Callable] = _UNSET,+ extra_headers: Optional[Dict[str, str]] = _UNSET,+ disable_resources: bool = _UNSET,+ wait_selector: Optional[str] = _UNSET,+ wait_selector_state: SelectorWaitStates = _UNSET,+ network_idle: bool = _UNSET,+ solve_cloudflare: bool = _UNSET,+ selector_config: Optional[Dict] = _UNSET,+ ) -> Response:"""Opens up the browser and do your request based on your chosen options.:param url: The Target url.+ :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.+ :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000+ :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.+ :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.+ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._+ :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.+ Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.+ This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.+ :param wait_selector: Wait for a specific CSS selector to be in a specific state.+ :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.+ :param network_idle: Wait for the page until there are no network connections for at least 500 ms.+ :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.+ :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:return: A `Response` object."""+ google_search = self._get_with_precedence(+ google_search, self.google_search, _UNSET+ )+ timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)+ wait = self._get_with_precedence(wait, self.wait, _UNSET)+ page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)+ extra_headers = self._get_with_precedence(+ extra_headers, self.extra_headers, _UNSET+ )+ disable_resources = self._get_with_precedence(+ disable_resources, self.disable_resources, _UNSET+ )+ wait_selector = self._get_with_precedence(+ wait_selector, self.wait_selector, _UNSET+ )+ wait_selector_state = self._get_with_precedence(+ wait_selector_state, self.wait_selector_state, _UNSET+ )+ network_idle = self._get_with_precedence(+ network_idle, self.network_idle, _UNSET+ )+ solve_cloudflare = self._get_with_precedence(+ solve_cloudflare, self.solve_cloudflare, _UNSET+ )+ selector_config = self._get_with_precedence(+ selector_config, self.selector_config, _UNSET+ )+if self._closed: # pragma: no coverraise RuntimeError("Context manager has been closed")final_response = Nonereferer = (generate_convincing_referer(url)- if (self.google_search and "referer" not in self._headers_keys)+ if (google_search and "referer" not in self._headers_keys)else None)@@ -271,7 +327,7 @@ class StealthySession(StealthySessionMixin, SyncSession):):final_response = finished_response- page_info = self._get_page()+ page_info = self._get_page(timeout, extra_headers, disable_resources)page_info.mark_busy(url=url)try: # pragma: no cover@@ -280,41 +336,41 @@ class StealthySession(StealthySessionMixin, SyncSession):first_response = page_info.page.goto(url, referer=referer)page_info.page.wait_for_load_state(state="domcontentloaded")- if self.network_idle:+ if network_idle:page_info.page.wait_for_load_state("networkidle")if not first_response:raise RuntimeError(f"Failed to get response for {url}")- if self.solve_cloudflare:+ if solve_cloudflare:self._solve_cloudflare(page_info.page)# Make sure the page is fully loaded after the captchapage_info.page.wait_for_load_state(state="load")page_info.page.wait_for_load_state(state="domcontentloaded")- if self.network_idle:+ if network_idle:page_info.page.wait_for_load_state("networkidle")- if self.page_action is not None:+ if page_action is not None:try:- page_info.page = self.page_action(page_info.page)+ _ = page_action(page_info.page)except Exception as e:log.error(f"Error executing page_action: {e}")- if self.wait_selector:+ if wait_selector:try:- waiter: Locator = page_info.page.locator(self.wait_selector)- waiter.first.wait_for(state=self.wait_selector_state)+ waiter: Locator = page_info.page.locator(wait_selector)+ waiter.first.wait_for(state=wait_selector_state)# Wait again after waiting for the selector, helpful with protections like Cloudflarepage_info.page.wait_for_load_state(state="load")page_info.page.wait_for_load_state(state="domcontentloaded")- if self.network_idle:+ if network_idle:page_info.page.wait_for_load_state("networkidle")except Exception as e:- log.error(f"Error waiting for selector {self.wait_selector}: {e}")+ log.error(f"Error waiting for selector {wait_selector}: {e}")- page_info.page.wait_for_timeout(self.wait)+ page_info.page.wait_for_timeout(wait)response = ResponseFactory.from_playwright_response(- page_info.page, first_response, final_response, self.selector_config+ page_info.page, first_response, final_response, selector_config)# Mark the page as finished for next use@@ -508,19 +564,74 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):log.info("Cloudflare captcha is solved")return- async def fetch(self, url: str) -> Response:+ async def fetch(+ self,+ url: str,+ google_search: bool = _UNSET,+ timeout: int | float = _UNSET,+ wait: int | float = _UNSET,+ page_action: Optional[Callable] = _UNSET,+ extra_headers: Optional[Dict[str, str]] = _UNSET,+ disable_resources: bool = _UNSET,+ wait_selector: Optional[str] = _UNSET,+ wait_selector_state: SelectorWaitStates = _UNSET,+ network_idle: bool = _UNSET,+ solve_cloudflare: bool = _UNSET,+ selector_config: Optional[Dict] = _UNSET,+ ) -> Response:"""Opens up the browser and do your request based on your chosen options.:param url: The Target url.+ :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.+ :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000+ :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.+ :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.+ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._+ :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.+ Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.+ This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.+ :param wait_selector: Wait for a specific CSS selector to be in a specific state.+ :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.+ :param network_idle: Wait for the page until there are no network connections for at least 500 ms.+ :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.+ :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:return: A `Response` object."""+ google_search = self._get_with_precedence(+ google_search, self.google_search, _UNSET+ )+ timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)+ wait = self._get_with_precedence(wait, self.wait, _UNSET)+ page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)+ extra_headers = self._get_with_precedence(+ extra_headers, self.extra_headers, _UNSET+ )+ disable_resources = self._get_with_precedence(+ disable_resources, self.disable_resources, _UNSET+ )+ wait_selector = self._get_with_precedence(+ wait_selector, self.wait_selector, _UNSET+ )+ wait_selector_state = self._get_with_precedence(+ wait_selector_state, self.wait_selector_state, _UNSET+ )+ network_idle = self._get_with_precedence(+ network_idle, self.network_idle, _UNSET+ )+ solve_cloudflare = self._get_with_precedence(+ solve_cloudflare, self.solve_cloudflare, _UNSET+ )+ selector_config = self._get_with_precedence(+ selector_config, self.selector_config, _UNSET+ )+if self._closed: # pragma: no coverraise RuntimeError("Context manager has been closed")final_response = Nonereferer = (generate_convincing_referer(url)- if (self.google_search and "referer" not in self._headers_keys)+ if (google_search and "referer" not in self._headers_keys)else None)@@ -532,7 +643,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):):final_response = finished_response- page_info = await self._get_page()+ page_info = await self._get_page(timeout, extra_headers, disable_resources)page_info.mark_busy(url=url)try:@@ -541,43 +652,43 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):first_response = await page_info.page.goto(url, referer=referer)await page_info.page.wait_for_load_state(state="domcontentloaded")- if self.network_idle:+ if network_idle:await page_info.page.wait_for_load_state("networkidle")if not first_response:raise RuntimeError(f"Failed to get response for {url}")- if self.solve_cloudflare:+ if solve_cloudflare:await self._solve_cloudflare(page_info.page)# Make sure the page is fully loaded after the captchaawait page_info.page.wait_for_load_state(state="load")await page_info.page.wait_for_load_state(state="domcontentloaded")- if self.network_idle:+ if network_idle:await page_info.page.wait_for_load_state("networkidle")- if self.page_action is not None:+ if page_action is not None:try:- page_info.page = await self.page_action(page_info.page)+ _ = await page_action(page_info.page)except Exception as e:log.error(f"Error executing page_action: {e}")- if self.wait_selector:+ if wait_selector:try:- waiter: AsyncLocator = page_info.page.locator(self.wait_selector)- await waiter.first.wait_for(state=self.wait_selector_state)+ waiter: AsyncLocator = page_info.page.locator(wait_selector)+ await waiter.first.wait_for(state=wait_selector_state)# Wait again after waiting for the selector, helpful with protections like Cloudflareawait page_info.page.wait_for_load_state(state="load")await page_info.page.wait_for_load_state(state="domcontentloaded")- if self.network_idle:+ if network_idle:await page_info.page.wait_for_load_state("networkidle")except Exception as e:- log.error(f"Error waiting for selector {self.wait_selector}: {e}")+ log.error(f"Error waiting for selector {wait_selector}: {e}")- await page_info.page.wait_for_timeout(self.wait)+ await page_info.page.wait_for_timeout(wait)# Create response objectresponse = await ResponseFactory.from_async_playwright_response(- page_info.page, first_response, final_response, self.selector_config+ page_info.page, first_response, final_response, selector_config)# Mark the page as finished for next usediff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex ef3b906..04a9d35 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -31,6 +31,8 @@ from scrapling.engines.toolbelt import (generate_convincing_referer,)+_UNSET = object()+class DynamicSession(DynamicSessionMixin, SyncSession):"""A Browser session manager with page pooling."""@@ -198,19 +200,66 @@ class DynamicSession(DynamicSessionMixin, SyncSession):def fetch(self,url: str,+ google_search: bool = _UNSET,+ timeout: int | float = _UNSET,+ wait: int | float = _UNSET,+ page_action: Optional[Callable] = _UNSET,+ extra_headers: Optional[Dict[str, str]] = _UNSET,+ disable_resources: bool = _UNSET,+ wait_selector: Optional[str] = _UNSET,+ wait_selector_state: SelectorWaitStates = _UNSET,+ network_idle: bool = _UNSET,+ selector_config: Optional[Dict] = _UNSET,) -> Response:"""Opens up the browser and do your request based on your chosen options.:param url: The Target url.+ :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.+ :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000+ :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.+ :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.+ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._+ :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.+ Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.+ This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.+ :param wait_selector: Wait for a specific CSS selector to be in a specific state.+ :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.+ :param network_idle: Wait for the page until there are no network connections for at least 500 ms.+ :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:return: A `Response` object."""+ google_search = self._get_with_precedence(+ google_search, self.google_search, _UNSET+ )+ timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)+ wait = self._get_with_precedence(wait, self.wait, _UNSET)+ page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)+ extra_headers = self._get_with_precedence(+ extra_headers, self.extra_headers, _UNSET+ )+ disable_resources = self._get_with_precedence(+ disable_resources, self.disable_resources, _UNSET+ )+ wait_selector = self._get_with_precedence(+ wait_selector, self.wait_selector, _UNSET+ )+ wait_selector_state = self._get_with_precedence(+ wait_selector_state, self.wait_selector_state, _UNSET+ )+ network_idle = self._get_with_precedence(+ network_idle, self.network_idle, _UNSET+ )+ selector_config = self._get_with_precedence(+ selector_config, self.selector_config, _UNSET+ )+if self._closed: # pragma: no coverraise RuntimeError("Context manager has been closed")final_response = Nonereferer = (generate_convincing_referer(url)- if (self.google_search and "referer" not in self._headers_keys)+ if (google_search and "referer" not in self._headers_keys)else None)@@ -222,7 +271,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):):final_response = finished_response- page_info = self._get_page()+ page_info = self._get_page(timeout, extra_headers, disable_resources)page_info.mark_busy(url=url)try: # pragma: no cover@@ -231,35 +280,35 @@ class DynamicSession(DynamicSessionMixin, SyncSession):first_response = page_info.page.goto(url, referer=referer)page_info.page.wait_for_load_state(state="domcontentloaded")- if self.network_idle:+ if network_idle:page_info.page.wait_for_load_state("networkidle")if not first_response:raise RuntimeError(f"Failed to get response for {url}")- if self.page_action is not None:+ if page_action is not None:try:- page_info.page = self.page_action(page_info.page)+ _ = page_action(page_info.page)except Exception as e: # pragma: no coverlog.error(f"Error executing page_action: {e}")- if self.wait_selector:+ if wait_selector:try:- waiter: Locator = page_info.page.locator(self.wait_selector)- waiter.first.wait_for(state=self.wait_selector_state)+ waiter: Locator = page_info.page.locator(wait_selector)+ waiter.first.wait_for(state=wait_selector_state)# Wait again after waiting for the selector, helpful with protections like Cloudflarepage_info.page.wait_for_load_state(state="load")page_info.page.wait_for_load_state(state="domcontentloaded")- if self.network_idle:+ if network_idle:page_info.page.wait_for_load_state("networkidle")except Exception as e: # pragma: no cover- log.error(f"Error waiting for selector {self.wait_selector}: {e}")+ log.error(f"Error waiting for selector {wait_selector}: {e}")- page_info.page.wait_for_timeout(self.wait)+ page_info.page.wait_for_timeout(wait)# Create response objectresponse = ResponseFactory.from_playwright_response(- page_info.page, first_response, final_response, self.selector_config+ page_info.page, first_response, final_response, selector_config)# Mark the page as finished for next use@@ -409,19 +458,69 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):self._closed = True- async def fetch(self, url: str) -> Response:+ async def fetch(+ self,+ url: str,+ google_search: bool = _UNSET,+ timeout: int | float = _UNSET,+ wait: int | float = _UNSET,+ page_action: Optional[Callable] = _UNSET,+ extra_headers: Optional[Dict[str, str]] = _UNSET,+ disable_resources: bool = _UNSET,+ wait_selector: Optional[str] = _UNSET,+ wait_selector_state: SelectorWaitStates = _UNSET,+ network_idle: bool = _UNSET,+ selector_config: Optional[Dict] = _UNSET,+ ) -> Response:"""Opens up the browser and do your request based on your chosen options.:param url: The Target url.+ :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.+ :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000+ :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.+ :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.+ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._+ :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.+ Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.+ This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.+ :param wait_selector: Wait for a specific CSS selector to be in a specific state.+ :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.+ :param network_idle: Wait for the page until there are no network connections for at least 500 ms.+ :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:return: A `Response` object."""+ google_search = self._get_with_precedence(+ google_search, self.google_search, _UNSET+ )+ timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)+ wait = self._get_with_precedence(wait, self.wait, _UNSET)+ page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)+ extra_headers = self._get_with_precedence(+ extra_headers, self.extra_headers, _UNSET+ )+ disable_resources = self._get_with_precedence(+ disable_resources, self.disable_resources, _UNSET+ )+ wait_selector = self._get_with_precedence(+ wait_selector, self.wait_selector, _UNSET+ )+ wait_selector_state = self._get_with_precedence(+ wait_selector_state, self.wait_selector_state, _UNSET+ )+ network_idle = self._get_with_precedence(+ network_idle, self.network_idle, _UNSET+ )+ selector_config = self._get_with_precedence(+ selector_config, self.selector_config, _UNSET+ )+if self._closed: # pragma: no coverraise RuntimeError("Context manager has been closed")final_response = Nonereferer = (generate_convincing_referer(url)- if (self.google_search and "referer" not in self._headers_keys)+ if (google_search and "referer" not in self._headers_keys)else None)@@ -433,7 +532,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):):final_response = finished_response- page_info = await self._get_page()+ page_info = await self._get_page(timeout, extra_headers, disable_resources)page_info.mark_busy(url=url)try:@@ -442,35 +541,35 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):first_response = await page_info.page.goto(url, referer=referer)await page_info.page.wait_for_load_state(state="domcontentloaded")- if self.network_idle:+ if network_idle:await page_info.page.wait_for_load_state("networkidle")if not first_response:raise RuntimeError(f"Failed to get response for {url}")- if self.page_action is not None:+ if page_action is not None:try:- page_info.page = await self.page_action(page_info.page)+ _ = await page_action(page_info.page)except Exception as e:log.error(f"Error executing page_action: {e}")- if self.wait_selector:+ if wait_selector:try:- waiter: AsyncLocator = page_info.page.locator(self.wait_selector)- await waiter.first.wait_for(state=self.wait_selector_state)+ waiter: AsyncLocator = page_info.page.locator(wait_selector)+ await waiter.first.wait_for(state=wait_selector_state)# Wait again after waiting for the selector, helpful with protections like Cloudflareawait page_info.page.wait_for_load_state(state="load")await page_info.page.wait_for_load_state(state="domcontentloaded")- if self.network_idle:+ if network_idle:await page_info.page.wait_for_load_state("networkidle")except Exception as e:- log.error(f"Error waiting for selector {self.wait_selector}: {e}")+ log.error(f"Error waiting for selector {wait_selector}: {e}")- await page_info.page.wait_for_timeout(self.wait)+ await page_info.page.wait_for_timeout(wait)# Create response objectresponse = await ResponseFactory.from_async_playwright_response(- page_info.page, first_response, final_response, self.selector_config+ page_info.page, first_response, final_response, selector_config)# Mark the page as finished for next use