Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

refactor(fetchers)!: Remove `capture_xhr` parameter while we have `record_requests` parameter for sessions

Karim shoair7c2ff106ab34
diff --git a/scrapling/core/_network_formatting.py b/scrapling/core/_network_formatting.py
index d2455b8..678ed2d 100644
--- a/scrapling/core/_network_formatting.py
+++ b/scrapling/core/_network_formatting.py
@@ -1,7 +1,7 @@
-from pydantic import BaseModel
+from pydantic import AliasPath, BaseModel, Field
from scrapling.core._types import Dict, Literal, Optional
-from scrapling.engines._browsers._network import NetworkRequest
+from scrapling.engines.toolbelt.custom import Response
from scrapling.engines.toolbelt.convertor import ResponseFactory
NetworkPart = Literal["summary", "request_headers", "request_body", "response_headers", "response_body"]
@@ -10,10 +10,12 @@ NetworkPart = Literal["summary", "request_headers", "request_body", "response_he
class NetworkRequestInfo(BaseModel):
"""Summary of a recorded browser request."""
- id: int
+ model_config = {"populate_by_name": True}
+
+ id: int = Field(validation_alias=AliasPath("meta", "network_id"))
url: str
method: str
- resource_type: str
+ resource_type: str = Field(validation_alias=AliasPath("meta", "resource_type"))
status: int
@@ -33,21 +35,21 @@ def _decode(body: bytes, encoding: str) -> str:
return body.decode("utf-8", errors="replace")
-def _request_details(record: NetworkRequest, part: NetworkPart) -> NetworkRequestModel:
+def _request_details(record: Response, part: NetworkPart) -> NetworkRequestModel:
"""Read one saved request part without browser access."""
- result = NetworkRequestModel(request_id=record.id, part=part)
+ result = NetworkRequestModel(request_id=record.meta["network_id"], part=part)
if part == "summary":
result.data = NetworkRequestInfo.model_validate(record, from_attributes=True)
return result
- headers = record.request_headers if part.startswith("request_") else record.response_headers
+ headers = record.request_headers if part.startswith("request_") else record.headers
if part.endswith("headers"):
result.data = headers.copy()
- if record.response.meta.get("request_headers_partial" if part == "request_headers" else "headers_partial"):
+ if record.meta.get("request_headers_partial" if part == "request_headers" else "headers_partial"):
result.note = "Headers are partial; full headers unavailable."
return result
- body = record.request_body if part == "request_body" else record.response.body
- if part == "response_body" and record.response.meta.get("body_note"):
- result.note = record.response.meta["body_note"]
+ body = record.meta["request_body"] if part == "request_body" else record.body
+ if part == "response_body" and record.meta.get("body_note"):
+ result.note = record.meta["body_note"]
elif body is None:
result.note = "Body is absent."
elif body and not ResponseFactory._text_content(headers):
@@ -56,7 +58,7 @@ def _request_details(record: NetworkRequest, part: NetworkPart) -> NetworkReques
encoding = (
ResponseFactory._extract_encoding(headers.get("content-type"))
if part == "request_body"
- else record.response.encoding
+ else record.encoding
)
result.data = _decode(body, encoding)
return result
diff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.py
index a4e709f..0a90e64 100644
--- a/scrapling/core/_shell_signatures.py
+++ b/scrapling/core/_shell_signatures.py
@@ -65,7 +65,6 @@ _FETCH_PARAMS = {
"block_ads": bool,
"retries": int,
"retry_delay": int | float,
- "capture_xhr": str | None,
"record_requests": bool,
"max_recorded_requests": int,
"executable_path": Optional[str],
diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py
index 2a09900..c57cecc 100644
--- a/scrapling/core/ai.py
+++ b/scrapling/core/ai.py
@@ -383,7 +383,7 @@ class ScraplingMCPServer:
shown = records[:limit]
return NetworkRequestsModel(
requests=[NetworkRequestInfo.model_validate(record, from_attributes=True) for record in shown],
- next_cursor=shown[-1].id if shown else max(after_id, network.last_id),
+ next_cursor=shown[-1].meta["network_id"] if shown else max(after_id, network.last_id),
has_more=len(records) > limit,
dropped_count=network.dropped_count,
)
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py
index a906a04..42ab729 100644
--- a/scrapling/engines/_browsers/_base.py
+++ b/scrapling/engines/_browsers/_base.py
@@ -1,5 +1,4 @@
from time import time
-from re import search as re_search
from asyncio import sleep as asyncio_sleep, Lock
from contextlib import contextmanager, asynccontextmanager, suppress
@@ -174,15 +173,11 @@ class SyncSession:
def _create_response_handler(
page_info: PageInfo[Page],
response_container: List,
- xhr_pattern: Optional[str] = None,
- xhr_container: Optional[List] = None,
) -> Callable[[SyncPlaywrightResponse], None]:
- """Create a response handler that captures the final navigation response and optionally XHR/fetch responses.
+ """Create a response handler that captures the final navigation response.
:param page_info: The PageInfo object containing the page
:param response_container: A list to store the final response (mutable container)
- :param xhr_pattern: Optional regex pattern to match XHR/fetch response URLs
- :param xhr_container: Optional list to store captured XHR/fetch responses
:return: A callback function for page.on("response", ...)
"""
@@ -193,13 +188,6 @@ class SyncSession:
and finished_response.request.frame == page_info.page.main_frame
):
response_container[0] = finished_response
- elif (
- xhr_pattern
- and xhr_container is not None
- and finished_response.request.resource_type in ("xhr", "fetch")
- and re_search(xhr_pattern, finished_response.url)
- ):
- xhr_container.append(finished_response)
return handle_response
@@ -393,15 +381,11 @@ class AsyncSession:
def _create_response_handler(
page_info: PageInfo[AsyncPage],
response_container: List,
- xhr_pattern: Optional[str] = None,
- xhr_container: Optional[List] = None,
) -> Callable[[AsyncPlaywrightResponse], Awaitable[None]]:
- """Create an async response handler that captures the final navigation response and optionally XHR/fetch responses.
+ """Create an async response handler that captures the final navigation response.
:param page_info: The PageInfo object containing the page
:param response_container: A list to store the final response (mutable container)
- :param xhr_pattern: Optional regex pattern to match XHR/fetch response URLs
- :param xhr_container: Optional list to store captured XHR/fetch responses
:return: A callback function for page.on("response", ...)
"""
@@ -412,13 +396,6 @@ class AsyncSession:
and finished_response.request.frame == page_info.page.main_frame
):
response_container[0] = finished_response
- elif (
- xhr_pattern
- and xhr_container is not None
- and finished_response.request.resource_type in ("xhr", "fetch")
- and re_search(xhr_pattern, finished_response.url)
- ):
- xhr_container.append(finished_response)
return handle_response
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index 31c4179..e53d27f 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -149,14 +149,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
) as page_info:
final_response: List = [None]
- xhr_captured: List = []
page = page_info.page
- handler = self._create_response_handler(
- page_info,
- final_response,
- xhr_pattern=self._config.capture_xhr,
- xhr_container=xhr_captured,
- )
+ handler = self._create_response_handler(page_info, final_response)
page.on("response", handler)
try:
if params.page_setup:
@@ -194,7 +188,6 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
final_response[0],
params.selector_config,
meta={"proxy": proxy},
- xhr_captured=xhr_captured,
pierce_shadow=params.pierce_shadow,
)
return response
@@ -347,14 +340,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
) as page_info:
final_response: List = [None]
- xhr_captured: List = []
page = page_info.page
- handler = self._create_response_handler(
- page_info,
- final_response,
- xhr_pattern=self._config.capture_xhr,
- xhr_container=xhr_captured,
- )
+ handler = self._create_response_handler(page_info, final_response)
page.on("response", handler)
try:
if params.page_setup:
@@ -392,7 +379,6 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
final_response[0],
params.selector_config,
meta={"proxy": proxy},
- xhr_captured=xhr_captured,
pierce_shadow=params.pierce_shadow,
)
return response
diff --git a/scrapling/engines/_browsers/_network.py b/scrapling/engines/_browsers/_network.py
index 27b4648..835f055 100644
--- a/scrapling/engines/_browsers/_network.py
+++ b/scrapling/engines/_browsers/_network.py
@@ -1,7 +1,6 @@
from re import compile as compile_regex, error as RegexError
from collections import OrderedDict
from contextlib import suppress
-from dataclasses import dataclass
from scrapling.core._types import Any, Callable, Dict, List, Optional
from scrapling.engines.toolbelt.custom import Response
@@ -11,21 +10,6 @@ MAX_BODY_BYTES = 1024 * 1024
MAX_TOTAL_BODY_BYTES = 20 * 1024 * 1024
-@dataclass(slots=True)
-class NetworkRequest:
- """Saved request data and response, with a session-local ID."""
-
- id: int
- url: str
- method: str
- resource_type: str
- request_headers: Dict[str, str]
- request_body: Optional[bytes]
- response_headers: Dict[str, str]
- status: int
- response: Response
-
-
class NetworkRecorder:
"""Keep bounded request history and saved responses for browser sessions."""
@@ -36,7 +20,7 @@ class NetworkRecorder:
self.max_requests = max_requests
self.dropped_count = 0
self.last_id = 0
- self._records: OrderedDict[int, NetworkRequest] = OrderedDict()
+ self._records: OrderedDict[int, Response] = OrderedDict()
self._contexts: Dict[Any, Dict[str, Callable[..., Any]]] = {}
self._body_bytes = 0
self._asynchronous = asynchronous
@@ -86,24 +70,17 @@ class NetworkRecorder:
if generation != self._generation or self._contexts.get(context) is not listeners:
return
self.last_id += 1
- self._records[self.last_id] = NetworkRequest(
- self.last_id,
- request.url,
- request.method,
- request.resource_type,
- response.request_headers,
- request.post_data_buffer,
- response.headers,
- response.status,
- response,
+ response.meta.update(
+ network_id=self.last_id, resource_type=request.resource_type, request_body=request.post_data_buffer
)
+ self._records[self.last_id] = response
self._body_bytes += len(response.body)
while len(self._records) > self.max_requests or self._body_bytes > MAX_TOTAL_BODY_BYTES:
- self._body_bytes -= len(self._records.popitem(last=False)[1].response.body)
+ self._body_bytes -= len(self._records.popitem(last=False)[1].body)
self.dropped_count += 1
- def get(self, request_id: int) -> Optional[NetworkRequest]:
- """Return one retained request, or None if its ID is unavailable."""
+ def get(self, request_id: int) -> Optional[Response]:
+ """Return one saved response, or None if its recording ID is unavailable."""
return self._records.get(request_id)
def search(
@@ -113,25 +90,25 @@ class NetworkRecorder:
resource_type: Optional[str] = None,
status: Optional[int] = None,
after_id: int = 0,
- limit: int = 100,
+ limit: Optional[int] = 100,
include_static: bool = True,
- ) -> List[NetworkRequest]:
- """Return matching requests in ID order; the URL pattern is a regular expression."""
- if limit < 1 or after_id < 0:
- raise ValueError("limit must be positive and after_id must be nonnegative.")
+ ) -> List[Response]:
+ """Return responses in recording ID order; filter by URL regex and use limit=None for all matches."""
+ if (limit is not None and limit < 1) or after_id < 0:
+ raise ValueError("limit must be positive or None and after_id must be nonnegative.")
try:
pattern = compile_regex(url_pattern) if url_pattern is not None else None
except RegexError as error:
raise ValueError(f"Invalid URL pattern: {error}") from error
found = []
- for record in self._records.values():
+ for request_id, record in self._records.items():
if (
- record.id <= after_id
+ request_id <= after_id
or (pattern is not None and not pattern.search(record.url))
or (method is not None and record.method != method.upper())
- or (resource_type is not None and record.resource_type != resource_type)
+ or (resource_type is not None and record.meta["resource_type"] != resource_type)
or (status is not None and record.status != status)
- or (not include_static and record.resource_type not in {"xhr", "fetch"} and record.status < 400)
+ or (not include_static and record.meta["resource_type"] not in {"xhr", "fetch"} and record.status < 400)
):
continue
found.append(record)
diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py
index 419f8e0..5c06bfc 100644
--- a/scrapling/engines/_browsers/_stealth.py
+++ b/scrapling/engines/_browsers/_stealth.py
@@ -243,14 +243,8 @@ class StealthySession(SyncSession, StealthySessionMixin):
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
) as page_info:
final_response: List = [None]
- xhr_captured: List = []
page = page_info.page
- handler = self._create_response_handler(
- page_info,
- final_response,
- xhr_pattern=self._config.capture_xhr,
- xhr_container=xhr_captured,
- )
+ handler = self._create_response_handler(page_info, final_response)
page.on("response", handler)
try:
if params.page_setup:
@@ -293,7 +287,6 @@ class StealthySession(SyncSession, StealthySessionMixin):
final_response[0],
params.selector_config,
meta={"proxy": proxy},
- xhr_captured=xhr_captured,
pierce_shadow=params.pierce_shadow,
)
return response
@@ -540,14 +533,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
) as page_info:
final_response: List = [None]
- xhr_captured: List = []
page = page_info.page
- handler = self._create_response_handler(
- page_info,
- final_response,
- xhr_pattern=self._config.capture_xhr,
- xhr_container=xhr_captured,
- )
+ handler = self._create_response_handler(page_info, final_response)
page.on("response", handler)
try:
if params.page_setup:
@@ -590,7 +577,6 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
final_response[0],
params.selector_config,
meta={"proxy": proxy},
- xhr_captured=xhr_captured,
pierce_shadow=params.pierce_shadow,
)
return response
diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py
index 6e8d5ca..7309172 100644
--- a/scrapling/engines/_browsers/_types.py
+++ b/scrapling/engines/_browsers/_types.py
@@ -93,7 +93,6 @@ class PlaywrightSession(TypedDict, total=False):
block_ads: bool
retries: int
retry_delay: int | float
- capture_xhr: str | None
record_requests: bool
max_recorded_requests: int
executable_path: Optional[str]
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index a349a77..0202752 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -1,5 +1,4 @@
from pathlib import Path
-from re import compile as re_compile, error as RegexError
from typing import Annotated
from functools import lru_cache
from urllib.parse import urlparse
@@ -91,7 +90,6 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
block_ads: bool = False
retries: RetriesCount = 3
retry_delay: Seconds = 1
- capture_xhr: str | None = None
record_requests: bool = False
max_recorded_requests: Annotated[int, Meta(ge=1)] = 1000
executable_path: Optional[str] = None
@@ -123,13 +121,6 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
self.selector_config = {}
if not self.additional_args:
self.additional_args = {}
- if not self.capture_xhr:
- self.capture_xhr = None
- else:
- try:
- re_compile(self.capture_xhr)
- except RegexError as error:
- raise ValueError(f"Invalid capture_xhr regex: {error}. Use '.*' to match all URLs.") from error
if self.init_script is not None:
validation_msg = _is_invalid_file_path(self.init_script)
@@ -190,6 +181,8 @@ def validate_fetch(
session: Any,
model: type[PlaywrightConfig] | type[StealthConfig],
) -> _fetch_params: # pragma: no cover
+ if "capture_xhr" in method_kwargs:
+ raise TypeError("capture_xhr was removed; use record_requests=True and session.network.")
result: Dict[str, Any] = {}
overrides: Dict[str, Any] = {}
kwargs_dict: Dict[str, Any] = dict(method_kwargs)
@@ -254,6 +247,8 @@ def validate(params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: .
def validate(params: Dict, model: type[PlaywrightConfig] | type[StealthConfig]) -> PlaywrightConfig | StealthConfig:
+ if "capture_xhr" in params:
+ raise TypeError("capture_xhr was removed; use record_requests=True and session.network.")
try:
# Filter out params with the default values (no need to validate them) to speed up validation
filtered = _filter_defaults(params, model.__name__)
diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py
index 14e4151..72cc089 100644
--- a/scrapling/engines/toolbelt/convertor.py
+++ b/scrapling/engines/toolbelt/convertor.py
@@ -11,7 +11,7 @@ from playwright.async_api import Page as AsyncPage, Response as AsyncResponse
from scrapling.core.utils import log
from .custom import Response, StatusText
-from scrapling.core._types import Dict, List, Optional
+from scrapling.core._types import Dict, Optional
SHADOW_SNAPSHOT_JS = r"""
() => {
@@ -162,7 +162,6 @@ class ResponseFactory:
final_response: Optional[SyncResponse],
parser_arguments: Dict,
meta: Optional[Dict] = None,
- xhr_captured: Optional[List[SyncResponse]] = None,
collect_history: bool = True,
pierce_shadow: bool = False,
max_body_bytes: Optional[int] = None,
@@ -182,7 +181,6 @@ class ResponseFactory:
:param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into
the `Response` object.
:param meta: Additional meta data to be saved with the response.
- :param xhr_captured: Optional list of captured Playwright XHR/fetch responses to convert and attach to the returned Response.
:param collect_history: Optional boolean indicating whether to collect redirections history or not.
:param pierce_shadow: Include open shadow roots in the HTML snapshot. Disabled by default.
:param max_body_bytes: Optional text-only recorded-body limit. Skips non-text or unknown content types before reading; preserves headers and notes for unavailable bodies.
@@ -241,7 +239,7 @@ class ResponseFactory:
raise
request_headers = first_response.request.headers
body_meta["request_headers_partial"] = True
- response = cls._build_browser_response(
+ return cls._build_browser_response(
{
"url": page.url if page else first_response.url,
"content": page_content,
@@ -257,11 +255,6 @@ class ResponseFactory:
},
max_body_bytes,
)
- if xhr_captured:
- response.captured_xhr = [
- cls.from_playwright_response(None, p, None, {}, collect_history=False) for p in xhr_captured
- ]
- return response
@classmethod
async def _async_process_response_history(
@@ -344,7 +337,6 @@ class ResponseFactory:
final_response: Optional[AsyncResponse],
parser_arguments: Dict,
meta: Optional[Dict] = None,
- xhr_captured: Optional[List[AsyncResponse]] = None,
collect_history: bool = True,
pierce_shadow: bool = False,
max_body_bytes: Optional[int] = None,
@@ -364,7 +356,6 @@ class ResponseFactory:
:param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into
the `Response` object.
:param meta: Additional meta data to be saved with the response.
- :param xhr_captured: Optional list of captured async Playwright XHR/fetch responses to convert and attach to the returned Response.
:param collect_history: Optional boolean indicating whether to collect redirections history or not.
:param pierce_shadow: Include open shadow roots in the HTML snapshot. Disabled by default.
:param max_body_bytes: Optional text-only recorded-body limit. Skips non-text or unknown content types before reading; preserves headers and notes for unavailable bodies.
@@ -424,7 +415,7 @@ class ResponseFactory:
raise
request_headers = first_response.request.headers
body_meta["request_headers_partial"] = True
- response = cls._build_browser_response(
+ return cls._build_browser_response(
{
"url": page.url if page else first_response.url,
"content": page_content,
@@ -440,11 +431,6 @@ class ResponseFactory:
},
max_body_bytes,
)
- if xhr_captured:
- response.captured_xhr = [
- await cls.from_async_playwright_response(None, p, None, {}, collect_history=False) for p in xhr_captured
- ]
- return response
@staticmethod
def from_http_request(response: CurlResponse, parser_arguments: Dict, meta: Optional[Dict] = None) -> Response:
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index 729f2b0..0527844 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -36,7 +36,6 @@ class Response(Selector):
:param history: List of redirect responses, if any.
:param meta: Metadata dictionary (e.g., proxy used).
:param request: Associated spider Request object (set by crawler, in the spiders framework).
- :param captured_xhr: List of captured XHR/fetch ``Response`` objects. Populated when ``capture_xhr`` is set on a browser session.
"""
def __init__(
@@ -61,6 +60,7 @@ class Response(Selector):
adaptive_domain: str = cast(str, selector_config.pop("adaptive_domain", ""))
self.status = status
+ self.method = method
self.reason = reason
self.cookies = cookies
self.headers = headers
@@ -80,7 +80,6 @@ class Response(Selector):
self.meta: Dict[str, Any] = meta or {}
self.request: Optional["Request"] = None # Will be set by crawler
- self.captured_xhr: List["Response"] = []
@property
def body(self) -> bytes: