Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(browser sessions): Collect XHR requests done while loading the page

Karim shoair68f7c5c36f04
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py
index 501db76..15371f7 100644
--- a/scrapling/engines/_browsers/_base.py
+++ b/scrapling/engines/_browsers/_base.py
@@ -1,4 +1,5 @@
from time import time
+from re import search as re_search
from asyncio import sleep as asyncio_sleep, Lock
from contextlib import contextmanager, asynccontextmanager
@@ -146,11 +147,18 @@ class SyncSession:
self._wait_for_networkidle(page)
@staticmethod
- def _create_response_handler(page_info: PageInfo[Page], response_container: List) -> Callable:
- """Create a response handler that captures the final navigation response.
+ def _create_response_handler(
+ page_info: PageInfo[Page],
+ response_container: List,
+ xhr_pattern: Optional[str] = None,
+ xhr_container: Optional[List] = None,
+ ) -> Callable:
+ """Create a response handler that captures the final navigation response and optionally XHR/fetch responses.
:param page_info: The PageInfo object containing the page
:param response_container: A list to store the final response (mutable container)
+ :param xhr_pattern: Optional regex pattern to match XHR/fetch response URLs
+ :param xhr_container: Optional list to store captured XHR/fetch responses
:return: A callback function for page.on("response", ...)
"""
@@ -161,6 +169,13 @@ class SyncSession:
and finished_response.request.frame == page_info.page.main_frame
):
response_container[0] = finished_response
+ elif (
+ xhr_pattern
+ and xhr_container is not None
+ and finished_response.request.resource_type in ("xhr", "fetch")
+ and re_search(xhr_pattern, finished_response.url)
+ ):
+ xhr_container.append(finished_response)
return handle_response
@@ -317,11 +332,18 @@ class AsyncSession:
await self._wait_for_networkidle(page)
@staticmethod
- def _create_response_handler(page_info: PageInfo[AsyncPage], response_container: List) -> Callable:
- """Create an async response handler that captures the final navigation response.
+ def _create_response_handler(
+ page_info: PageInfo[AsyncPage],
+ response_container: List,
+ xhr_pattern: Optional[str] = None,
+ xhr_container: Optional[List] = None,
+ ) -> Callable:
+ """Create an async response handler that captures the final navigation response and optionally XHR/fetch responses.
:param page_info: The PageInfo object containing the page
:param response_container: A list to store the final response (mutable container)
+ :param xhr_pattern: Optional regex pattern to match XHR/fetch response URLs
+ :param xhr_container: Optional list to store captured XHR/fetch responses
:return: A callback function for page.on("response", ...)
"""
@@ -332,6 +354,13 @@ class AsyncSession:
and finished_response.request.frame == page_info.page.main_frame
):
response_container[0] = finished_response
+ elif (
+ xhr_pattern
+ and xhr_container is not None
+ and finished_response.request.resource_type in ("xhr", "fetch")
+ and re_search(xhr_pattern, finished_response.url)
+ ):
+ xhr_container.append(finished_response)
return handle_response
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index fa6e8b6..f788819 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -139,9 +139,17 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
with self._page_generator(
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
) as page_info:
- final_response = [None]
+ final_response, xhr_captured = [None], []
page = page_info.page
- page.on("response", self._create_response_handler(page_info, final_response))
+ page.on(
+ "response",
+ self._create_response_handler(
+ page_info,
+ final_response,
+ xhr_pattern=self._config.capture_xhr,
+ xhr_container=xhr_captured,
+ ),
+ )
try:
first_response = page.goto(url, referer=referer)
@@ -167,7 +175,12 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
page.wait_for_timeout(params.wait)
response = ResponseFactory.from_playwright_response(
- page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy}
+ page,
+ first_response,
+ final_response[0],
+ params.selector_config,
+ meta={"proxy": proxy},
+ xhr_captured=xhr_captured,
)
return response
@@ -306,9 +319,17 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
async with self._page_generator(
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
) as page_info:
- final_response = [None]
+ final_response, xhr_captured = [None], []
page = page_info.page
- page.on("response", self._create_response_handler(page_info, final_response))
+ page.on(
+ "response",
+ self._create_response_handler(
+ page_info,
+ final_response,
+ xhr_pattern=self._config.capture_xhr,
+ xhr_container=xhr_captured,
+ ),
+ )
try:
first_response = await page.goto(url, referer=referer)
@@ -334,7 +355,12 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
await page.wait_for_timeout(params.wait)
response = await ResponseFactory.from_async_playwright_response(
- page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy}
+ page,
+ first_response,
+ final_response[0],
+ params.selector_config,
+ meta={"proxy": proxy},
+ xhr_captured=xhr_captured,
)
return response
diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py
index 8fb248f..ef3a445 100644
--- a/scrapling/engines/_browsers/_stealth.py
+++ b/scrapling/engines/_browsers/_stealth.py
@@ -3,12 +3,8 @@ from re import compile as re_compile
from time import sleep as time_sleep
from asyncio import sleep as asyncio_sleep
-from playwright.sync_api import Locator, Page, BrowserContext
-from playwright.async_api import (
- Page as async_Page,
- Locator as AsyncLocator,
- BrowserContext as AsyncBrowserContext,
-)
+from playwright.sync_api import Locator, Page
+from playwright.async_api import Page as async_Page, Locator as AsyncLocator
from patchright.sync_api import sync_playwright
from patchright.async_api import async_playwright
@@ -226,9 +222,17 @@ class StealthySession(SyncSession, StealthySessionMixin):
with self._page_generator(
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
) as page_info:
- final_response = [None]
+ final_response, xhr_captured = [None], []
page = page_info.page
- page.on("response", self._create_response_handler(page_info, final_response))
+ page.on(
+ "response",
+ self._create_response_handler(
+ page_info,
+ final_response,
+ xhr_pattern=self._config.capture_xhr,
+ xhr_container=xhr_captured,
+ ),
+ )
try:
first_response = page.goto(url, referer=referer)
@@ -259,7 +263,12 @@ class StealthySession(SyncSession, StealthySessionMixin):
page.wait_for_timeout(params.wait)
response = ResponseFactory.from_playwright_response(
- page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy}
+ page,
+ first_response,
+ final_response[0],
+ params.selector_config,
+ meta={"proxy": proxy},
+ xhr_captured=xhr_captured,
)
return response
@@ -480,9 +489,17 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
async with self._page_generator(
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
) as page_info:
- final_response = [None]
+ final_response, xhr_captured = [None], []
page = page_info.page
- page.on("response", self._create_response_handler(page_info, final_response))
+ page.on(
+ "response",
+ self._create_response_handler(
+ page_info,
+ final_response,
+ xhr_pattern=self._config.capture_xhr,
+ xhr_container=xhr_captured,
+ ),
+ )
try:
first_response = await page.goto(url, referer=referer)
@@ -513,7 +530,12 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
await page.wait_for_timeout(params.wait)
response = await ResponseFactory.from_async_playwright_response(
- page, first_response, final_response[0], params.selector_config, meta={"proxy": proxy}
+ page,
+ first_response,
+ final_response[0],
+ params.selector_config,
+ meta={"proxy": proxy},
+ xhr_captured=xhr_captured,
)
return response
diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py
index 30d6af1..f4f31cd 100644
--- a/scrapling/engines/_browsers/_types.py
+++ b/scrapling/engines/_browsers/_types.py
@@ -89,6 +89,7 @@ class PlaywrightSession(TypedDict, total=False):
blocked_domains: Optional[Set[str]]
retries: int
retry_delay: int | float
+ capture_xhr: str | None
class PlaywrightFetchParams(TypedDict, total=False):
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index e2424f4..dfe7310 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -87,6 +87,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
blocked_domains: Optional[Set[str]] = None
retries: RetriesCount = 3
retry_delay: Seconds = 1
+ capture_xhr: str | None = None
def __post_init__(self): # pragma: no cover
"""Custom validation after msgspec validation"""
@@ -112,6 +113,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
self.selector_config = {}
if not self.additional_args:
self.additional_args = {}
+ if not self.capture_xhr:
+ self.capture_xhr = None
if self.init_script is not None:
validation_msg = _is_invalid_file_path(self.init_script)
diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py
index 8606003..8d624e0 100644
--- a/scrapling/engines/toolbelt/convertor.py
+++ b/scrapling/engines/toolbelt/convertor.py
@@ -8,7 +8,7 @@ from playwright.async_api import Page as AsyncPage, Response as AsyncResponse
from scrapling.core.utils import log
from .custom import Response, StatusText
-from scrapling.core._types import Dict, Optional
+from scrapling.core._types import Dict, List, Optional
__CHARSET_RE__ = re_compile(r"charset=([\w-]+)")
@@ -81,11 +81,13 @@ class ResponseFactory:
@classmethod
def from_playwright_response(
cls,
- page: SyncPage,
+ page: Optional[SyncPage],
first_response: SyncResponse,
final_response: Optional[SyncResponse],
parser_arguments: Dict,
meta: Optional[Dict] = None,
+ xhr_captured: Optional[List[SyncResponse]] = None,
+ collect_history: bool = True,
) -> Response:
"""
Transforms a Playwright response into an internal `Response` object, encapsulating
@@ -102,7 +104,8 @@ class ResponseFactory:
:param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into
the `Response` object.
:param meta: Additional meta data to be saved with the response.
-
+ :param xhr_captured: Optional list of captured Playwright XHR/fetch responses to convert and attach to the returned Response.
+ :param collect_history: Optional boolean indicating whether to collect redirections history or not.
:return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.
:rtype: Response
"""
@@ -115,9 +118,9 @@ class ResponseFactory:
# PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(final_response.status)
- history = cls._process_response_history(first_response, parser_arguments)
+ history = cls._process_response_history(first_response, parser_arguments) if collect_history else []
try:
- if "html" in final_response.all_headers().get("content-type", ""):
+ if page and "html" in final_response.all_headers().get("content-type", ""):
page_content = cls._get_page_content(page).encode("utf-8")
else:
page_content = final_response.body()
@@ -125,14 +128,14 @@ class ResponseFactory:
log.error(f"Error getting page content: {e}")
page_content = b""
- return Response(
+ response = Response(
**{
- "url": page.url,
+ "url": page.url if page else first_response.url,
"content": page_content,
"status": final_response.status,
"reason": status_text,
"encoding": encoding,
- "cookies": tuple(dict(cookie) for cookie in page.context.cookies()),
+ "cookies": tuple(dict(cookie) for cookie in page.context.cookies()) if page else {},
"headers": first_response.all_headers(),
"request_headers": first_response.request.all_headers(),
"history": history,
@@ -140,6 +143,11 @@ class ResponseFactory:
**parser_arguments,
}
)
+ if xhr_captured:
+ response.captured_xhr = [
+ cls.from_playwright_response(None, p, None, {}, collect_history=False) for p in xhr_captured
+ ]
+ return response
@classmethod
async def _async_process_response_history(
@@ -219,11 +227,13 @@ class ResponseFactory:
@classmethod
async def from_async_playwright_response(
cls,
- page: AsyncPage,
+ page: Optional[AsyncPage],
first_response: AsyncResponse,
final_response: Optional[AsyncResponse],
parser_arguments: Dict,
meta: Optional[Dict] = None,
+ xhr_captured: Optional[List[AsyncResponse]] = None,
+ collect_history: bool = True,
) -> Response:
"""
Transforms a Playwright response into an internal `Response` object, encapsulating
@@ -240,6 +250,8 @@ class ResponseFactory:
:param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into
the `Response` object.
:param meta: Additional meta data to be saved with the response.
+ :param xhr_captured: Optional list of captured async Playwright XHR/fetch responses to convert and attach to the returned Response.
+ :param collect_history: Optional boolean indicating whether to collect redirections history or not.
:return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.
:rtype: Response
@@ -253,9 +265,9 @@ class ResponseFactory:
# PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(final_response.status)
- history = await cls._async_process_response_history(first_response, parser_arguments)
+ history = await cls._async_process_response_history(first_response, parser_arguments) if collect_history else []
try:
- if "html" in (await final_response.all_headers()).get("content-type", ""):
+ if page and "html" in (await final_response.all_headers()).get("content-type", ""):
page_content = (await cls._get_async_page_content(page)).encode("utf-8")
else:
page_content = await final_response.body()
@@ -263,14 +275,14 @@ class ResponseFactory:
log.error(f"Error getting page content in async: {e}")
page_content = b""
- return Response(
+ response = Response(
**{
- "url": page.url,
+ "url": page.url if page else first_response.url,
"content": page_content,
"status": final_response.status,
"reason": status_text,
"encoding": encoding,
- "cookies": tuple(dict(cookie) for cookie in await page.context.cookies()),
+ "cookies": tuple(dict(cookie) for cookie in await page.context.cookies()) if page else {},
"headers": await first_response.all_headers(),
"request_headers": await first_response.request.all_headers(),
"history": history,
@@ -278,6 +290,11 @@ class ResponseFactory:
**parser_arguments,
}
)
+ if xhr_captured:
+ response.captured_xhr = [
+ await cls.from_async_playwright_response(None, p, None, {}, collect_history=False) for p in xhr_captured
+ ]
+ return response
@staticmethod
def from_http_request(response: CurlResponse, parser_arguments: Dict, meta: Optional[Dict] = None) -> Response:
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index 58a7485..483a32b 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -67,6 +67,7 @@ class Response(Selector):
self.meta: Dict[str, Any] = meta or {}
self.request: Optional["Request"] = None # Will be set by crawler
+ self.captured_xhr: List["Response"] = []
@property
def body(self) -> bytes: