Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

refactor/feat(browser fetchers): make it possible to have a configuration per page in sessions

Karim shoair9838d741d0de
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py
index 55b3310..6b91d8f 100644
--- a/scrapling/engines/_browsers/_base.py
+++ b/scrapling/engines/_browsers/_base.py
@@ -22,6 +22,7 @@ from ._config_tools import _compiled_stealth_scripts
from ._validators import validate, PlaywrightConfig, CamoufoxConfig
from ._config_tools import _launch_kwargs, _context_kwargs
from scrapling.core._types import (
+ Any,
Dict,
Optional,
)
@@ -38,7 +39,12 @@ class SyncSession:
self.context: Optional[BrowserContext] = None
self._closed = False
- def _get_page(self) -> PageInfo: # pragma: no cover
+ def _get_page(
+ self,
+ timeout: int | float,
+ extra_headers: Optional[Dict[str, str]],
+ disable_resources: bool,
+ ) -> PageInfo: # pragma: no cover
"""Get a new page to use"""
# Close all finished pages to ensure clean state
@@ -59,13 +65,12 @@ class SyncSession:
)
page = self.context.new_page()
- timeout = getattr(self, "timeout", 30000)
page.set_default_navigation_timeout(timeout)
page.set_default_timeout(timeout)
- if getattr(self, "extra_headers", False):
- page.set_extra_http_headers(getattr(self, "extra_headers"))
+ if extra_headers:
+ page.set_extra_http_headers(extra_headers)
- if getattr(self, "disable_resources", False):
+ if disable_resources:
page.route("**/*", intercept_route)
if getattr(self, "stealth", False):
@@ -74,6 +79,13 @@ class SyncSession:
return self.page_pool.add_page(page)
+ @staticmethod
+ def _get_with_precedence(
+ request_value: Any, session_value: Any, sentinel_value: object
+ ) -> Any:
+ """Get value with request-level priority over session-level"""
+ return request_value if request_value is not sentinel_value else session_value
+
def get_pool_stats(self) -> Dict[str, int]:
"""Get statistics about the current page pool"""
return {
@@ -90,7 +102,12 @@ class AsyncSession(SyncSession):
self.context: Optional[AsyncBrowserContext] = None
self._lock = Lock()
- async def _get_page(self) -> PageInfo: # pragma: no cover
+ async def _get_page(
+ self,
+ timeout: int | float,
+ extra_headers: Optional[Dict[str, str]],
+ disable_resources: bool,
+ ) -> PageInfo: # pragma: no cover
"""Get a new page to use"""
async with self._lock:
# Close all finished pages to ensure clean state
@@ -111,13 +128,12 @@ class AsyncSession(SyncSession):
)
page = await self.context.new_page()
- timeout = getattr(self, "timeout", 30000)
page.set_default_navigation_timeout(timeout)
page.set_default_timeout(timeout)
- if getattr(self, "extra_headers", False):
- await page.set_extra_http_headers(getattr(self, "extra_headers"))
+ if extra_headers:
+ await page.set_extra_http_headers(extra_headers)
- if getattr(self, "disable_resources", False):
+ if disable_resources:
await page.route("**/*", async_intercept_route)
if getattr(self, "stealth", False):
@@ -334,7 +350,6 @@ class StealthySessionMixin:
self.geoip = config.geoip
self.selector_config = config.selector_config
self.additional_args = config.additional_args
- self.selector_config = config.selector_config
self.page_action = config.page_action
self._headers_keys = (
set(map(str.lower, self.extra_headers.keys()))
diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py
index cd814c1..926191a 100644
--- a/scrapling/engines/_browsers/_camoufox.py
+++ b/scrapling/engines/_browsers/_camoufox.py
@@ -31,6 +31,7 @@ from scrapling.engines.toolbelt import (
)
__CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*")
+_UNSET = object()
class StealthySession(StealthySessionMixin, SyncSession):
@@ -247,19 +248,74 @@ class StealthySession(StealthySessionMixin, SyncSession):
log.info("Cloudflare captcha is solved")
return
- def fetch(self, url: str) -> Response:
+ def fetch(
+ self,
+ url: str,
+ google_search: bool = _UNSET,
+ timeout: int | float = _UNSET,
+ wait: int | float = _UNSET,
+ page_action: Optional[Callable] = _UNSET,
+ extra_headers: Optional[Dict[str, str]] = _UNSET,
+ disable_resources: bool = _UNSET,
+ wait_selector: Optional[str] = _UNSET,
+ wait_selector_state: SelectorWaitStates = _UNSET,
+ network_idle: bool = _UNSET,
+ solve_cloudflare: bool = _UNSET,
+ selector_config: Optional[Dict] = _UNSET,
+ ) -> Response:
"""Opens up the browser and do your request based on your chosen options.
:param url: The Target url.
+ :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
+ :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
+ :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
+ :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
+ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+ :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
+ Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+ This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+ :param wait_selector: Wait for a specific CSS selector to be in a specific state.
+ :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
+ :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
+ :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
+ :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
+ google_search = self._get_with_precedence(
+ google_search, self.google_search, _UNSET
+ )
+ timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)
+ wait = self._get_with_precedence(wait, self.wait, _UNSET)
+ page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)
+ extra_headers = self._get_with_precedence(
+ extra_headers, self.extra_headers, _UNSET
+ )
+ disable_resources = self._get_with_precedence(
+ disable_resources, self.disable_resources, _UNSET
+ )
+ wait_selector = self._get_with_precedence(
+ wait_selector, self.wait_selector, _UNSET
+ )
+ wait_selector_state = self._get_with_precedence(
+ wait_selector_state, self.wait_selector_state, _UNSET
+ )
+ network_idle = self._get_with_precedence(
+ network_idle, self.network_idle, _UNSET
+ )
+ solve_cloudflare = self._get_with_precedence(
+ solve_cloudflare, self.solve_cloudflare, _UNSET
+ )
+ selector_config = self._get_with_precedence(
+ selector_config, self.selector_config, _UNSET
+ )
+
if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed")
final_response = None
referer = (
generate_convincing_referer(url)
- if (self.google_search and "referer" not in self._headers_keys)
+ if (google_search and "referer" not in self._headers_keys)
else None
)
@@ -271,7 +327,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
):
final_response = finished_response
- page_info = self._get_page()
+ page_info = self._get_page(timeout, extra_headers, disable_resources)
page_info.mark_busy(url=url)
try: # pragma: no cover
@@ -280,41 +336,41 @@ class StealthySession(StealthySessionMixin, SyncSession):
first_response = page_info.page.goto(url, referer=referer)
page_info.page.wait_for_load_state(state="domcontentloaded")
- if self.network_idle:
+ if network_idle:
page_info.page.wait_for_load_state("networkidle")
if not first_response:
raise RuntimeError(f"Failed to get response for {url}")
- if self.solve_cloudflare:
+ if solve_cloudflare:
self._solve_cloudflare(page_info.page)
# Make sure the page is fully loaded after the captcha
page_info.page.wait_for_load_state(state="load")
page_info.page.wait_for_load_state(state="domcontentloaded")
- if self.network_idle:
+ if network_idle:
page_info.page.wait_for_load_state("networkidle")
- if self.page_action is not None:
+ if page_action is not None:
try:
- page_info.page = self.page_action(page_info.page)
+ _ = page_action(page_info.page)
except Exception as e:
log.error(f"Error executing page_action: {e}")
- if self.wait_selector:
+ if wait_selector:
try:
- waiter: Locator = page_info.page.locator(self.wait_selector)
- waiter.first.wait_for(state=self.wait_selector_state)
+ waiter: Locator = page_info.page.locator(wait_selector)
+ waiter.first.wait_for(state=wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
page_info.page.wait_for_load_state(state="load")
page_info.page.wait_for_load_state(state="domcontentloaded")
- if self.network_idle:
+ if network_idle:
page_info.page.wait_for_load_state("networkidle")
except Exception as e:
- log.error(f"Error waiting for selector {self.wait_selector}: {e}")
+ log.error(f"Error waiting for selector {wait_selector}: {e}")
- page_info.page.wait_for_timeout(self.wait)
+ page_info.page.wait_for_timeout(wait)
response = ResponseFactory.from_playwright_response(
- page_info.page, first_response, final_response, self.selector_config
+ page_info.page, first_response, final_response, selector_config
)
# Mark the page as finished for next use
@@ -508,19 +564,74 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
log.info("Cloudflare captcha is solved")
return
- async def fetch(self, url: str) -> Response:
+ async def fetch(
+ self,
+ url: str,
+ google_search: bool = _UNSET,
+ timeout: int | float = _UNSET,
+ wait: int | float = _UNSET,
+ page_action: Optional[Callable] = _UNSET,
+ extra_headers: Optional[Dict[str, str]] = _UNSET,
+ disable_resources: bool = _UNSET,
+ wait_selector: Optional[str] = _UNSET,
+ wait_selector_state: SelectorWaitStates = _UNSET,
+ network_idle: bool = _UNSET,
+ solve_cloudflare: bool = _UNSET,
+ selector_config: Optional[Dict] = _UNSET,
+ ) -> Response:
"""Opens up the browser and do your request based on your chosen options.
:param url: The Target url.
+ :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
+ :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
+ :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
+ :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
+ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+ :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
+ Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+ This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+ :param wait_selector: Wait for a specific CSS selector to be in a specific state.
+ :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
+ :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
+ :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.
+ :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
+ google_search = self._get_with_precedence(
+ google_search, self.google_search, _UNSET
+ )
+ timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)
+ wait = self._get_with_precedence(wait, self.wait, _UNSET)
+ page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)
+ extra_headers = self._get_with_precedence(
+ extra_headers, self.extra_headers, _UNSET
+ )
+ disable_resources = self._get_with_precedence(
+ disable_resources, self.disable_resources, _UNSET
+ )
+ wait_selector = self._get_with_precedence(
+ wait_selector, self.wait_selector, _UNSET
+ )
+ wait_selector_state = self._get_with_precedence(
+ wait_selector_state, self.wait_selector_state, _UNSET
+ )
+ network_idle = self._get_with_precedence(
+ network_idle, self.network_idle, _UNSET
+ )
+ solve_cloudflare = self._get_with_precedence(
+ solve_cloudflare, self.solve_cloudflare, _UNSET
+ )
+ selector_config = self._get_with_precedence(
+ selector_config, self.selector_config, _UNSET
+ )
+
if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed")
final_response = None
referer = (
generate_convincing_referer(url)
- if (self.google_search and "referer" not in self._headers_keys)
+ if (google_search and "referer" not in self._headers_keys)
else None
)
@@ -532,7 +643,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
):
final_response = finished_response
- page_info = await self._get_page()
+ page_info = await self._get_page(timeout, extra_headers, disable_resources)
page_info.mark_busy(url=url)
try:
@@ -541,43 +652,43 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
first_response = await page_info.page.goto(url, referer=referer)
await page_info.page.wait_for_load_state(state="domcontentloaded")
- if self.network_idle:
+ if network_idle:
await page_info.page.wait_for_load_state("networkidle")
if not first_response:
raise RuntimeError(f"Failed to get response for {url}")
- if self.solve_cloudflare:
+ if solve_cloudflare:
await self._solve_cloudflare(page_info.page)
# Make sure the page is fully loaded after the captcha
await page_info.page.wait_for_load_state(state="load")
await page_info.page.wait_for_load_state(state="domcontentloaded")
- if self.network_idle:
+ if network_idle:
await page_info.page.wait_for_load_state("networkidle")
- if self.page_action is not None:
+ if page_action is not None:
try:
- page_info.page = await self.page_action(page_info.page)
+ _ = await page_action(page_info.page)
except Exception as e:
log.error(f"Error executing page_action: {e}")
- if self.wait_selector:
+ if wait_selector:
try:
- waiter: AsyncLocator = page_info.page.locator(self.wait_selector)
- await waiter.first.wait_for(state=self.wait_selector_state)
+ waiter: AsyncLocator = page_info.page.locator(wait_selector)
+ await waiter.first.wait_for(state=wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
await page_info.page.wait_for_load_state(state="load")
await page_info.page.wait_for_load_state(state="domcontentloaded")
- if self.network_idle:
+ if network_idle:
await page_info.page.wait_for_load_state("networkidle")
except Exception as e:
- log.error(f"Error waiting for selector {self.wait_selector}: {e}")
+ log.error(f"Error waiting for selector {wait_selector}: {e}")
- await page_info.page.wait_for_timeout(self.wait)
+ await page_info.page.wait_for_timeout(wait)
# Create response object
response = await ResponseFactory.from_async_playwright_response(
- page_info.page, first_response, final_response, self.selector_config
+ page_info.page, first_response, final_response, selector_config
)
# Mark the page as finished for next use
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index ef3b906..04a9d35 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -31,6 +31,8 @@ from scrapling.engines.toolbelt import (
generate_convincing_referer,
)
+_UNSET = object()
+
class DynamicSession(DynamicSessionMixin, SyncSession):
"""A Browser session manager with page pooling."""
@@ -198,19 +200,66 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
def fetch(
self,
url: str,
+ google_search: bool = _UNSET,
+ timeout: int | float = _UNSET,
+ wait: int | float = _UNSET,
+ page_action: Optional[Callable] = _UNSET,
+ extra_headers: Optional[Dict[str, str]] = _UNSET,
+ disable_resources: bool = _UNSET,
+ wait_selector: Optional[str] = _UNSET,
+ wait_selector_state: SelectorWaitStates = _UNSET,
+ network_idle: bool = _UNSET,
+ selector_config: Optional[Dict] = _UNSET,
) -> Response:
"""Opens up the browser and do your request based on your chosen options.
:param url: The Target url.
+ :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
+ :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
+ :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
+ :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
+ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+ :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
+ Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+ This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+ :param wait_selector: Wait for a specific CSS selector to be in a specific state.
+ :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
+ :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
+ :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
+ google_search = self._get_with_precedence(
+ google_search, self.google_search, _UNSET
+ )
+ timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)
+ wait = self._get_with_precedence(wait, self.wait, _UNSET)
+ page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)
+ extra_headers = self._get_with_precedence(
+ extra_headers, self.extra_headers, _UNSET
+ )
+ disable_resources = self._get_with_precedence(
+ disable_resources, self.disable_resources, _UNSET
+ )
+ wait_selector = self._get_with_precedence(
+ wait_selector, self.wait_selector, _UNSET
+ )
+ wait_selector_state = self._get_with_precedence(
+ wait_selector_state, self.wait_selector_state, _UNSET
+ )
+ network_idle = self._get_with_precedence(
+ network_idle, self.network_idle, _UNSET
+ )
+ selector_config = self._get_with_precedence(
+ selector_config, self.selector_config, _UNSET
+ )
+
if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed")
final_response = None
referer = (
generate_convincing_referer(url)
- if (self.google_search and "referer" not in self._headers_keys)
+ if (google_search and "referer" not in self._headers_keys)
else None
)
@@ -222,7 +271,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
):
final_response = finished_response
- page_info = self._get_page()
+ page_info = self._get_page(timeout, extra_headers, disable_resources)
page_info.mark_busy(url=url)
try: # pragma: no cover
@@ -231,35 +280,35 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
first_response = page_info.page.goto(url, referer=referer)
page_info.page.wait_for_load_state(state="domcontentloaded")
- if self.network_idle:
+ if network_idle:
page_info.page.wait_for_load_state("networkidle")
if not first_response:
raise RuntimeError(f"Failed to get response for {url}")
- if self.page_action is not None:
+ if page_action is not None:
try:
- page_info.page = self.page_action(page_info.page)
+ _ = page_action(page_info.page)
except Exception as e: # pragma: no cover
log.error(f"Error executing page_action: {e}")
- if self.wait_selector:
+ if wait_selector:
try:
- waiter: Locator = page_info.page.locator(self.wait_selector)
- waiter.first.wait_for(state=self.wait_selector_state)
+ waiter: Locator = page_info.page.locator(wait_selector)
+ waiter.first.wait_for(state=wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
page_info.page.wait_for_load_state(state="load")
page_info.page.wait_for_load_state(state="domcontentloaded")
- if self.network_idle:
+ if network_idle:
page_info.page.wait_for_load_state("networkidle")
except Exception as e: # pragma: no cover
- log.error(f"Error waiting for selector {self.wait_selector}: {e}")
+ log.error(f"Error waiting for selector {wait_selector}: {e}")
- page_info.page.wait_for_timeout(self.wait)
+ page_info.page.wait_for_timeout(wait)
# Create response object
response = ResponseFactory.from_playwright_response(
- page_info.page, first_response, final_response, self.selector_config
+ page_info.page, first_response, final_response, selector_config
)
# Mark the page as finished for next use
@@ -409,19 +458,69 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
self._closed = True
- async def fetch(self, url: str) -> Response:
+ async def fetch(
+ self,
+ url: str,
+ google_search: bool = _UNSET,
+ timeout: int | float = _UNSET,
+ wait: int | float = _UNSET,
+ page_action: Optional[Callable] = _UNSET,
+ extra_headers: Optional[Dict[str, str]] = _UNSET,
+ disable_resources: bool = _UNSET,
+ wait_selector: Optional[str] = _UNSET,
+ wait_selector_state: SelectorWaitStates = _UNSET,
+ network_idle: bool = _UNSET,
+ selector_config: Optional[Dict] = _UNSET,
+ ) -> Response:
"""Opens up the browser and do your request based on your chosen options.
:param url: The Target url.
+ :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.
+ :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
+ :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
+ :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
+ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+ :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.
+ Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+ This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+ :param wait_selector: Wait for a specific CSS selector to be in a specific state.
+ :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
+ :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
+ :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
+ google_search = self._get_with_precedence(
+ google_search, self.google_search, _UNSET
+ )
+ timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)
+ wait = self._get_with_precedence(wait, self.wait, _UNSET)
+ page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)
+ extra_headers = self._get_with_precedence(
+ extra_headers, self.extra_headers, _UNSET
+ )
+ disable_resources = self._get_with_precedence(
+ disable_resources, self.disable_resources, _UNSET
+ )
+ wait_selector = self._get_with_precedence(
+ wait_selector, self.wait_selector, _UNSET
+ )
+ wait_selector_state = self._get_with_precedence(
+ wait_selector_state, self.wait_selector_state, _UNSET
+ )
+ network_idle = self._get_with_precedence(
+ network_idle, self.network_idle, _UNSET
+ )
+ selector_config = self._get_with_precedence(
+ selector_config, self.selector_config, _UNSET
+ )
+
if self._closed: # pragma: no cover
raise RuntimeError("Context manager has been closed")
final_response = None
referer = (
generate_convincing_referer(url)
- if (self.google_search and "referer" not in self._headers_keys)
+ if (google_search and "referer" not in self._headers_keys)
else None
)
@@ -433,7 +532,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
):
final_response = finished_response
- page_info = await self._get_page()
+ page_info = await self._get_page(timeout, extra_headers, disable_resources)
page_info.mark_busy(url=url)
try:
@@ -442,35 +541,35 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
first_response = await page_info.page.goto(url, referer=referer)
await page_info.page.wait_for_load_state(state="domcontentloaded")
- if self.network_idle:
+ if network_idle:
await page_info.page.wait_for_load_state("networkidle")
if not first_response:
raise RuntimeError(f"Failed to get response for {url}")
- if self.page_action is not None:
+ if page_action is not None:
try:
- page_info.page = await self.page_action(page_info.page)
+ _ = await page_action(page_info.page)
except Exception as e:
log.error(f"Error executing page_action: {e}")
- if self.wait_selector:
+ if wait_selector:
try:
- waiter: AsyncLocator = page_info.page.locator(self.wait_selector)
- await waiter.first.wait_for(state=self.wait_selector_state)
+ waiter: AsyncLocator = page_info.page.locator(wait_selector)
+ await waiter.first.wait_for(state=wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
await page_info.page.wait_for_load_state(state="load")
await page_info.page.wait_for_load_state(state="domcontentloaded")
- if self.network_idle:
+ if network_idle:
await page_info.page.wait_for_load_state("networkidle")
except Exception as e:
- log.error(f"Error waiting for selector {self.wait_selector}: {e}")
+ log.error(f"Error waiting for selector {wait_selector}: {e}")
- await page_info.page.wait_for_timeout(self.wait)
+ await page_info.page.wait_for_timeout(wait)
# Create response object
response = await ResponseFactory.from_async_playwright_response(
- page_info.page, first_response, final_response, self.selector_config
+ page_info.page, first_response, final_response, selector_config
)
# Mark the page as finished for next use