🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(browser sessions): Collect XHR requests done while loading the page
Karim shoair68f7c5c36f04
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex 501db76..15371f7 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -1,4 +1,5 @@from time import time+from re import search as re_searchfrom asyncio import sleep as asyncio_sleep, Lockfrom contextlib import contextmanager, asynccontextmanager@@ -146,11 +147,18 @@ class SyncSession:self._wait_for_networkidle(page)@staticmethod- def _create_response_handler(page_info: PageInfo[Page], response_container: List) -> Callable:- """Create a response handler that captures the final navigation response.+ def _create_response_handler(+ page_info: PageInfo[Page],+ response_container: List,+ xhr_pattern: Optional[str] = None,+ xhr_container: Optional[List] = None,+ ) -> Callable:+ """Create a response handler that captures the final navigation response and optionally XHR/fetch responses.:param page_info: The PageInfo object containing the page:param response_container: A list to store the final response (mutable container)+ :param xhr_pattern: Optional regex pattern to match XHR/fetch response URLs+ :param xhr_container: Optional list to store captured XHR/fetch responses:return: A callback function for page.on("response", ...)"""@@ -161,6 +169,13 @@ class SyncSession:and finished_response.request.frame == page_info.page.main_frame):response_container[0] = finished_response+ elif (+ xhr_pattern+ and xhr_container is not None+ and finished_response.request.resource_type in ("xhr", "fetch")+ and re_search(xhr_pattern, finished_response.url)+ ):+ xhr_container.append(finished_response)return handle_response@@ -317,11 +332,18 @@ class AsyncSession:await self._wait_for_networkidle(page)@staticmethod- def _create_response_handler(page_info: PageInfo[AsyncPage], response_container: List) -> Callable:- """Create an async response handler that captures the final navigation response.+ def _create_response_handler(+ page_info: PageInfo[AsyncPage],+ response_container: List,+ xhr_pattern: Optional[str] = None,+ xhr_container: Optional[List] = None,+ ) -> Callable:+ """Create an async response handler that captures the final navigation response and optionally XHR/fetch responses.:param page_info: The PageInfo object containing the page:param response_container: A list to store the final response (mutable container)+ :param xhr_pattern: Optional regex pattern to match XHR/fetch response URLs+ :param xhr_container: Optional list to store captured XHR/fetch responses:return: A callback function for page.on("response", ...)"""@@ -332,6 +354,13 @@ class AsyncSession:and finished_response.request.frame == page_info.page.main_frame):response_container[0] = finished_response+ elif (+ xhr_pattern+ and xhr_container is not None+ and finished_response.request.resource_type in ("xhr", "fetch")+ and re_search(xhr_pattern, finished_response.url)+ ):+ xhr_container.append(finished_response)return handle_responsediff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex fa6e8b6..f788819 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -139,9 +139,17 @@ class DynamicSession(SyncSession, DynamicSessionMixin):with self._page_generator(params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains) as page_info:- final_response = [None]+ final_response, xhr_captured = [None], []page = page_info.page- page.on("response", self._create_response_handler(page_info, final_response))+ page.on(+ "response",+ self._create_response_handler(+ page_info,+ final_response,+ xhr_pattern=self._config.capture_xhr,+ xhr_container=xhr_captured,+ ),+ )try:first_response = page.goto(url, referer=referer)@@ -167,7 +175,12 @@ class DynamicSession(SyncSession, DynamicSessionMixin):page.wait_for_timeout(params.wait)response = ResponseFactory.from_playwright_response(- page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy}+ page,+ first_response,+ final_response[0],+ params.selector_config,+ meta={"proxy": proxy},+ xhr_captured=xhr_captured,)return response@@ -306,9 +319,17 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):async with self._page_generator(params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains) as page_info:- final_response = [None]+ final_response, xhr_captured = [None], []page = page_info.page- page.on("response", self._create_response_handler(page_info, final_response))+ page.on(+ "response",+ self._create_response_handler(+ page_info,+ final_response,+ xhr_pattern=self._config.capture_xhr,+ xhr_container=xhr_captured,+ ),+ )try:first_response = await page.goto(url, referer=referer)@@ -334,7 +355,12 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):await page.wait_for_timeout(params.wait)response = await ResponseFactory.from_async_playwright_response(- page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy}+ page,+ first_response,+ final_response[0],+ params.selector_config,+ meta={"proxy": proxy},+ xhr_captured=xhr_captured,)return responsediff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.pyindex 8fb248f..ef3a445 100644--- a/scrapling/engines/_browsers/_stealth.py+++ b/scrapling/engines/_browsers/_stealth.py@@ -3,12 +3,8 @@ from re import compile as re_compilefrom time import sleep as time_sleepfrom asyncio import sleep as asyncio_sleep-from playwright.sync_api import Locator, Page, BrowserContext-from playwright.async_api import (- Page as async_Page,- Locator as AsyncLocator,- BrowserContext as AsyncBrowserContext,-)+from playwright.sync_api import Locator, Page+from playwright.async_api import Page as async_Page, Locator as AsyncLocatorfrom patchright.sync_api import sync_playwrightfrom patchright.async_api import async_playwright@@ -226,9 +222,17 @@ class StealthySession(SyncSession, StealthySessionMixin):with self._page_generator(params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains) as page_info:- final_response = [None]+ final_response, xhr_captured = [None], []page = page_info.page- page.on("response", self._create_response_handler(page_info, final_response))+ page.on(+ "response",+ self._create_response_handler(+ page_info,+ final_response,+ xhr_pattern=self._config.capture_xhr,+ xhr_container=xhr_captured,+ ),+ )try:first_response = page.goto(url, referer=referer)@@ -259,7 +263,12 @@ class StealthySession(SyncSession, StealthySessionMixin):page.wait_for_timeout(params.wait)response = ResponseFactory.from_playwright_response(- page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy}+ page,+ first_response,+ final_response[0],+ params.selector_config,+ meta={"proxy": proxy},+ xhr_captured=xhr_captured,)return response@@ -480,9 +489,17 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):async with self._page_generator(params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains) as page_info:- final_response = [None]+ final_response, xhr_captured = [None], []page = page_info.page- page.on("response", self._create_response_handler(page_info, final_response))+ page.on(+ "response",+ self._create_response_handler(+ page_info,+ final_response,+ xhr_pattern=self._config.capture_xhr,+ xhr_container=xhr_captured,+ ),+ )try:first_response = await page.goto(url, referer=referer)@@ -513,7 +530,12 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):await page.wait_for_timeout(params.wait)response = await ResponseFactory.from_async_playwright_response(- page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy}+ page,+ first_response,+ final_response[0],+ params.selector_config,+ meta={"proxy": proxy},+ xhr_captured=xhr_captured,)return responsediff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.pyindex 30d6af1..f4f31cd 100644--- a/scrapling/engines/_browsers/_types.py+++ b/scrapling/engines/_browsers/_types.py@@ -89,6 +89,7 @@ class PlaywrightSession(TypedDict, total=False):blocked_domains: Optional[Set[str]]retries: intretry_delay: int | float+ capture_xhr: str | Noneclass PlaywrightFetchParams(TypedDict, total=False):diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.pyindex e2424f4..dfe7310 100644--- a/scrapling/engines/_browsers/_validators.py+++ b/scrapling/engines/_browsers/_validators.py@@ -87,6 +87,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):blocked_domains: Optional[Set[str]] = Noneretries: RetriesCount = 3retry_delay: Seconds = 1+ capture_xhr: str | None = Nonedef __post_init__(self): # pragma: no cover"""Custom validation after msgspec validation"""@@ -112,6 +113,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):self.selector_config = {}if not self.additional_args:self.additional_args = {}+ if not self.capture_xhr:+ self.capture_xhr = Noneif self.init_script is not None:validation_msg = _is_invalid_file_path(self.init_script)diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.pyindex 8606003..8d624e0 100644--- a/scrapling/engines/toolbelt/convertor.py+++ b/scrapling/engines/toolbelt/convertor.py@@ -8,7 +8,7 @@ from playwright.async_api import Page as AsyncPage, Response as AsyncResponsefrom scrapling.core.utils import logfrom .custom import Response, StatusText-from scrapling.core._types import Dict, Optional+from scrapling.core._types import Dict, List, Optional__CHARSET_RE__ = re_compile(r"charset=([\w-]+)")@@ -81,11 +81,13 @@ class ResponseFactory:@classmethoddef from_playwright_response(cls,- page: SyncPage,+ page: Optional[SyncPage],first_response: SyncResponse,final_response: Optional[SyncResponse],parser_arguments: Dict,meta: Optional[Dict] = None,+ xhr_captured: Optional[List[SyncResponse]] = None,+ collect_history: bool = True,) -> Response:"""Transforms a Playwright response into an internal `Response` object, encapsulating@@ -102,7 +104,8 @@ class ResponseFactory::param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked intothe `Response` object.:param meta: Additional meta data to be saved with the response.-+ :param xhr_captured: Optional list of captured Playwright XHR/fetch responses to convert and attach to the returned Response.+ :param collect_history: Optional boolean indicating whether to collect redirections history or not.:return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.:rtype: Response"""@@ -115,9 +118,9 @@ class ResponseFactory:# PlayWright API sometimes give empty status text for some reason!status_text = final_response.status_text or StatusText.get(final_response.status)- history = cls._process_response_history(first_response, parser_arguments)+ history = cls._process_response_history(first_response, parser_arguments) if collect_history else []try:- if "html" in final_response.all_headers().get("content-type", ""):+ if page and "html" in final_response.all_headers().get("content-type", ""):page_content = cls._get_page_content(page).encode("utf-8")else:page_content = final_response.body()@@ -125,14 +128,14 @@ class ResponseFactory:log.error(f"Error getting page content: {e}")page_content = b""- return Response(+ response = Response(**{- "url": page.url,+ "url": page.url if page else first_response.url,"content": page_content,"status": final_response.status,"reason": status_text,"encoding": encoding,- "cookies": tuple(dict(cookie) for cookie in page.context.cookies()),+ "cookies": tuple(dict(cookie) for cookie in page.context.cookies()) if page else {},"headers": first_response.all_headers(),"request_headers": first_response.request.all_headers(),"history": history,@@ -140,6 +143,11 @@ class ResponseFactory:**parser_arguments,})+ if xhr_captured:+ response.captured_xhr = [+ cls.from_playwright_response(None, p, None, {}, collect_history=False) for p in xhr_captured+ ]+ return response@classmethodasync def _async_process_response_history(@@ -219,11 +227,13 @@ class ResponseFactory:@classmethodasync def from_async_playwright_response(cls,- page: AsyncPage,+ page: Optional[AsyncPage],first_response: AsyncResponse,final_response: Optional[AsyncResponse],parser_arguments: Dict,meta: Optional[Dict] = None,+ xhr_captured: Optional[List[AsyncResponse]] = None,+ collect_history: bool = True,) -> Response:"""Transforms a Playwright response into an internal `Response` object, encapsulating@@ -240,6 +250,8 @@ class ResponseFactory::param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked intothe `Response` object.:param meta: Additional meta data to be saved with the response.+ :param xhr_captured: Optional list of captured async Playwright XHR/fetch responses to convert and attach to the returned Response.+ :param collect_history: Optional boolean indicating whether to collect redirections history or not.:return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.:rtype: Response@@ -253,9 +265,9 @@ class ResponseFactory:# PlayWright API sometimes give empty status text for some reason!status_text = final_response.status_text or StatusText.get(final_response.status)- history = await cls._async_process_response_history(first_response, parser_arguments)+ history = await cls._async_process_response_history(first_response, parser_arguments) if collect_history else []try:- if "html" in (await final_response.all_headers()).get("content-type", ""):+ if page and "html" in (await final_response.all_headers()).get("content-type", ""):page_content = (await cls._get_async_page_content(page)).encode("utf-8")else:page_content = await final_response.body()@@ -263,14 +275,14 @@ class ResponseFactory:log.error(f"Error getting page content in async: {e}")page_content = b""- return Response(+ response = Response(**{- "url": page.url,+ "url": page.url if page else first_response.url,"content": page_content,"status": final_response.status,"reason": status_text,"encoding": encoding,- "cookies": tuple(dict(cookie) for cookie in await page.context.cookies()),+ "cookies": tuple(dict(cookie) for cookie in await page.context.cookies()) if page else {},"headers": await first_response.all_headers(),"request_headers": await first_response.request.all_headers(),"history": history,@@ -278,6 +290,11 @@ class ResponseFactory:**parser_arguments,})+ if xhr_captured:+ response.captured_xhr = [+ await cls.from_async_playwright_response(None, p, None, {}, collect_history=False) for p in xhr_captured+ ]+ return response@staticmethoddef from_http_request(response: CurlResponse, parser_arguments: Dict, meta: Optional[Dict] = None) -> Response:diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex 58a7485..483a32b 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -67,6 +67,7 @@ class Response(Selector):self.meta: Dict[str, Any] = meta or {}self.request: Optional["Request"] = None # Will be set by crawler+ self.captured_xhr: List["Response"] = []@propertydef body(self) -> bytes: