🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor(fetchers)!: Remove `capture_xhr` parameter while we have `record_requests` parameter for sessions
Karim shoair7c2ff106ab34
diff --git a/scrapling/core/_network_formatting.py b/scrapling/core/_network_formatting.pyindex d2455b8..678ed2d 100644--- a/scrapling/core/_network_formatting.py+++ b/scrapling/core/_network_formatting.py@@ -1,7 +1,7 @@-from pydantic import BaseModel+from pydantic import AliasPath, BaseModel, Fieldfrom scrapling.core._types import Dict, Literal, Optional-from scrapling.engines._browsers._network import NetworkRequest+from scrapling.engines.toolbelt.custom import Responsefrom scrapling.engines.toolbelt.convertor import ResponseFactoryNetworkPart = Literal["summary", "request_headers", "request_body", "response_headers", "response_body"]@@ -10,10 +10,12 @@ NetworkPart = Literal["summary", "request_headers", "request_body", "response_heclass NetworkRequestInfo(BaseModel):"""Summary of a recorded browser request."""- id: int+ model_config = {"populate_by_name": True}++ id: int = Field(validation_alias=AliasPath("meta", "network_id"))url: strmethod: str- resource_type: str+ resource_type: str = Field(validation_alias=AliasPath("meta", "resource_type"))status: int@@ -33,21 +35,21 @@ def _decode(body: bytes, encoding: str) -> str:return body.decode("utf-8", errors="replace")-def _request_details(record: NetworkRequest, part: NetworkPart) -> NetworkRequestModel:+def _request_details(record: Response, part: NetworkPart) -> NetworkRequestModel:"""Read one saved request part without browser access."""- result = NetworkRequestModel(request_id=record.id, part=part)+ result = NetworkRequestModel(request_id=record.meta["network_id"], part=part)if part == "summary":result.data = NetworkRequestInfo.model_validate(record, from_attributes=True)return result- headers = record.request_headers if part.startswith("request_") else record.response_headers+ headers = record.request_headers if part.startswith("request_") else record.headersif part.endswith("headers"):result.data = headers.copy()- if record.response.meta.get("request_headers_partial" if part == "request_headers" else "headers_partial"):+ if record.meta.get("request_headers_partial" if part == "request_headers" else "headers_partial"):result.note = "Headers are partial; full headers unavailable."return result- body = record.request_body if part == "request_body" else record.response.body- if part == "response_body" and record.response.meta.get("body_note"):- result.note = record.response.meta["body_note"]+ body = record.meta["request_body"] if part == "request_body" else record.body+ if part == "response_body" and record.meta.get("body_note"):+ result.note = record.meta["body_note"]elif body is None:result.note = "Body is absent."elif body and not ResponseFactory._text_content(headers):@@ -56,7 +58,7 @@ def _request_details(record: NetworkRequest, part: NetworkPart) -> NetworkRequesencoding = (ResponseFactory._extract_encoding(headers.get("content-type"))if part == "request_body"- else record.response.encoding+ else record.encoding)result.data = _decode(body, encoding)return resultdiff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.pyindex a4e709f..0a90e64 100644--- a/scrapling/core/_shell_signatures.py+++ b/scrapling/core/_shell_signatures.py@@ -65,7 +65,6 @@ _FETCH_PARAMS = {"block_ads": bool,"retries": int,"retry_delay": int | float,- "capture_xhr": str | None,"record_requests": bool,"max_recorded_requests": int,"executable_path": Optional[str],diff --git a/scrapling/core/ai.py b/scrapling/core/ai.pyindex 2a09900..c57cecc 100644--- a/scrapling/core/ai.py+++ b/scrapling/core/ai.py@@ -383,7 +383,7 @@ class ScraplingMCPServer:shown = records[:limit]return NetworkRequestsModel(requests=[NetworkRequestInfo.model_validate(record, from_attributes=True) for record in shown],- next_cursor=shown[-1].id if shown else max(after_id, network.last_id),+ next_cursor=shown[-1].meta["network_id"] if shown else max(after_id, network.last_id),has_more=len(records) > limit,dropped_count=network.dropped_count,)diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex a906a04..42ab729 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -1,5 +1,4 @@from time import time-from re import search as re_searchfrom asyncio import sleep as asyncio_sleep, Lockfrom contextlib import contextmanager, asynccontextmanager, suppress@@ -174,15 +173,11 @@ class SyncSession:def _create_response_handler(page_info: PageInfo[Page],response_container: List,- xhr_pattern: Optional[str] = None,- xhr_container: Optional[List] = None,) -> Callable[[SyncPlaywrightResponse], None]:- """Create a response handler that captures the final navigation response and optionally XHR/fetch responses.+ """Create a response handler that captures the final navigation response.:param page_info: The PageInfo object containing the page:param response_container: A list to store the final response (mutable container)- :param xhr_pattern: Optional regex pattern to match XHR/fetch response URLs- :param xhr_container: Optional list to store captured XHR/fetch responses:return: A callback function for page.on("response", ...)"""@@ -193,13 +188,6 @@ class SyncSession:and finished_response.request.frame == page_info.page.main_frame):response_container[0] = finished_response- elif (- xhr_pattern- and xhr_container is not None- and finished_response.request.resource_type in ("xhr", "fetch")- and re_search(xhr_pattern, finished_response.url)- ):- xhr_container.append(finished_response)return handle_response@@ -393,15 +381,11 @@ class AsyncSession:def _create_response_handler(page_info: PageInfo[AsyncPage],response_container: List,- xhr_pattern: Optional[str] = None,- xhr_container: Optional[List] = None,) -> Callable[[AsyncPlaywrightResponse], Awaitable[None]]:- """Create an async response handler that captures the final navigation response and optionally XHR/fetch responses.+ """Create an async response handler that captures the final navigation response.:param page_info: The PageInfo object containing the page:param response_container: A list to store the final response (mutable container)- :param xhr_pattern: Optional regex pattern to match XHR/fetch response URLs- :param xhr_container: Optional list to store captured XHR/fetch responses:return: A callback function for page.on("response", ...)"""@@ -412,13 +396,6 @@ class AsyncSession:and finished_response.request.frame == page_info.page.main_frame):response_container[0] = finished_response- elif (- xhr_pattern- and xhr_container is not None- and finished_response.request.resource_type in ("xhr", "fetch")- and re_search(xhr_pattern, finished_response.url)- ):- xhr_container.append(finished_response)return handle_responsediff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex 31c4179..e53d27f 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -149,14 +149,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin):params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains) as page_info:final_response: List = [None]- xhr_captured: List = []page = page_info.page- handler = self._create_response_handler(- page_info,- final_response,- xhr_pattern=self._config.capture_xhr,- xhr_container=xhr_captured,- )+ handler = self._create_response_handler(page_info, final_response)page.on("response", handler)try:if params.page_setup:@@ -194,7 +188,6 @@ class DynamicSession(SyncSession, DynamicSessionMixin):final_response[0],params.selector_config,meta={"proxy": proxy},- xhr_captured=xhr_captured,pierce_shadow=params.pierce_shadow,)return response@@ -347,14 +340,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains) as page_info:final_response: List = [None]- xhr_captured: List = []page = page_info.page- handler = self._create_response_handler(- page_info,- final_response,- xhr_pattern=self._config.capture_xhr,- xhr_container=xhr_captured,- )+ handler = self._create_response_handler(page_info, final_response)page.on("response", handler)try:if params.page_setup:@@ -392,7 +379,6 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):final_response[0],params.selector_config,meta={"proxy": proxy},- xhr_captured=xhr_captured,pierce_shadow=params.pierce_shadow,)return responsediff --git a/scrapling/engines/_browsers/_network.py b/scrapling/engines/_browsers/_network.pyindex 27b4648..835f055 100644--- a/scrapling/engines/_browsers/_network.py+++ b/scrapling/engines/_browsers/_network.py@@ -1,7 +1,6 @@from re import compile as compile_regex, error as RegexErrorfrom collections import OrderedDictfrom contextlib import suppress-from dataclasses import dataclassfrom scrapling.core._types import Any, Callable, Dict, List, Optionalfrom scrapling.engines.toolbelt.custom import Response@@ -11,21 +10,6 @@ MAX_BODY_BYTES = 1024 * 1024MAX_TOTAL_BODY_BYTES = 20 * 1024 * 1024-@dataclass(slots=True)-class NetworkRequest:- """Saved request data and response, with a session-local ID."""-- id: int- url: str- method: str- resource_type: str- request_headers: Dict[str, str]- request_body: Optional[bytes]- response_headers: Dict[str, str]- status: int- response: Response--class NetworkRecorder:"""Keep bounded request history and saved responses for browser sessions."""@@ -36,7 +20,7 @@ class NetworkRecorder:self.max_requests = max_requestsself.dropped_count = 0self.last_id = 0- self._records: OrderedDict[int, NetworkRequest] = OrderedDict()+ self._records: OrderedDict[int, Response] = OrderedDict()self._contexts: Dict[Any, Dict[str, Callable[..., Any]]] = {}self._body_bytes = 0self._asynchronous = asynchronous@@ -86,24 +70,17 @@ class NetworkRecorder:if generation != self._generation or self._contexts.get(context) is not listeners:returnself.last_id += 1- self._records[self.last_id] = NetworkRequest(- self.last_id,- request.url,- request.method,- request.resource_type,- response.request_headers,- request.post_data_buffer,- response.headers,- response.status,- response,+ response.meta.update(+ network_id=self.last_id, resource_type=request.resource_type, request_body=request.post_data_buffer)+ self._records[self.last_id] = responseself._body_bytes += len(response.body)while len(self._records) > self.max_requests or self._body_bytes > MAX_TOTAL_BODY_BYTES:- self._body_bytes -= len(self._records.popitem(last=False)[1].response.body)+ self._body_bytes -= len(self._records.popitem(last=False)[1].body)self.dropped_count += 1- def get(self, request_id: int) -> Optional[NetworkRequest]:- """Return one retained request, or None if its ID is unavailable."""+ def get(self, request_id: int) -> Optional[Response]:+ """Return one saved response, or None if its recording ID is unavailable."""return self._records.get(request_id)def search(@@ -113,25 +90,25 @@ class NetworkRecorder:resource_type: Optional[str] = None,status: Optional[int] = None,after_id: int = 0,- limit: int = 100,+ limit: Optional[int] = 100,include_static: bool = True,- ) -> List[NetworkRequest]:- """Return matching requests in ID order; the URL pattern is a regular expression."""- if limit < 1 or after_id < 0:- raise ValueError("limit must be positive and after_id must be nonnegative.")+ ) -> List[Response]:+ """Return responses in recording ID order; filter by URL regex and use limit=None for all matches."""+ if (limit is not None and limit < 1) or after_id < 0:+ raise ValueError("limit must be positive or None and after_id must be nonnegative.")try:pattern = compile_regex(url_pattern) if url_pattern is not None else Noneexcept RegexError as error:raise ValueError(f"Invalid URL pattern: {error}") from errorfound = []- for record in self._records.values():+ for request_id, record in self._records.items():if (- record.id <= after_id+ request_id <= after_idor (pattern is not None and not pattern.search(record.url))or (method is not None and record.method != method.upper())- or (resource_type is not None and record.resource_type != resource_type)+ or (resource_type is not None and record.meta["resource_type"] != resource_type)or (status is not None and record.status != status)- or (not include_static and record.resource_type not in {"xhr", "fetch"} and record.status < 400)+ or (not include_static and record.meta["resource_type"] not in {"xhr", "fetch"} and record.status < 400)):continuefound.append(record)diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.pyindex 419f8e0..5c06bfc 100644--- a/scrapling/engines/_browsers/_stealth.py+++ b/scrapling/engines/_browsers/_stealth.py@@ -243,14 +243,8 @@ class StealthySession(SyncSession, StealthySessionMixin):params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains) as page_info:final_response: List = [None]- xhr_captured: List = []page = page_info.page- handler = self._create_response_handler(- page_info,- final_response,- xhr_pattern=self._config.capture_xhr,- xhr_container=xhr_captured,- )+ handler = self._create_response_handler(page_info, final_response)page.on("response", handler)try:if params.page_setup:@@ -293,7 +287,6 @@ class StealthySession(SyncSession, StealthySessionMixin):final_response[0],params.selector_config,meta={"proxy": proxy},- xhr_captured=xhr_captured,pierce_shadow=params.pierce_shadow,)return response@@ -540,14 +533,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains) as page_info:final_response: List = [None]- xhr_captured: List = []page = page_info.page- handler = self._create_response_handler(- page_info,- final_response,- xhr_pattern=self._config.capture_xhr,- xhr_container=xhr_captured,- )+ handler = self._create_response_handler(page_info, final_response)page.on("response", handler)try:if params.page_setup:@@ -590,7 +577,6 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):final_response[0],params.selector_config,meta={"proxy": proxy},- xhr_captured=xhr_captured,pierce_shadow=params.pierce_shadow,)return responsediff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.pyindex 6e8d5ca..7309172 100644--- a/scrapling/engines/_browsers/_types.py+++ b/scrapling/engines/_browsers/_types.py@@ -93,7 +93,6 @@ class PlaywrightSession(TypedDict, total=False):block_ads: boolretries: intretry_delay: int | float- capture_xhr: str | Nonerecord_requests: boolmax_recorded_requests: intexecutable_path: Optional[str]diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.pyindex a349a77..0202752 100644--- a/scrapling/engines/_browsers/_validators.py+++ b/scrapling/engines/_browsers/_validators.py@@ -1,5 +1,4 @@from pathlib import Path-from re import compile as re_compile, error as RegexErrorfrom typing import Annotatedfrom functools import lru_cachefrom urllib.parse import urlparse@@ -91,7 +90,6 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):block_ads: bool = Falseretries: RetriesCount = 3retry_delay: Seconds = 1- capture_xhr: str | None = Nonerecord_requests: bool = Falsemax_recorded_requests: Annotated[int, Meta(ge=1)] = 1000executable_path: Optional[str] = None@@ -123,13 +121,6 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):self.selector_config = {}if not self.additional_args:self.additional_args = {}- if not self.capture_xhr:- self.capture_xhr = None- else:- try:- re_compile(self.capture_xhr)- except RegexError as error:- raise ValueError(f"Invalid capture_xhr regex: {error}. Use '.*' to match all URLs.") from errorif self.init_script is not None:validation_msg = _is_invalid_file_path(self.init_script)@@ -190,6 +181,8 @@ def validate_fetch(session: Any,model: type[PlaywrightConfig] | type[StealthConfig],) -> _fetch_params: # pragma: no cover+ if "capture_xhr" in method_kwargs:+ raise TypeError("capture_xhr was removed; use record_requests=True and session.network.")result: Dict[str, Any] = {}overrides: Dict[str, Any] = {}kwargs_dict: Dict[str, Any] = dict(method_kwargs)@@ -254,6 +247,8 @@ def validate(params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: .def validate(params: Dict, model: type[PlaywrightConfig] | type[StealthConfig]) -> PlaywrightConfig | StealthConfig:+ if "capture_xhr" in params:+ raise TypeError("capture_xhr was removed; use record_requests=True and session.network.")try:# Filter out params with the default values (no need to validate them) to speed up validationfiltered = _filter_defaults(params, model.__name__)diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.pyindex 14e4151..72cc089 100644--- a/scrapling/engines/toolbelt/convertor.py+++ b/scrapling/engines/toolbelt/convertor.py@@ -11,7 +11,7 @@ from playwright.async_api import Page as AsyncPage, Response as AsyncResponsefrom scrapling.core.utils import logfrom .custom import Response, StatusText-from scrapling.core._types import Dict, List, Optional+from scrapling.core._types import Dict, OptionalSHADOW_SNAPSHOT_JS = r"""() => {@@ -162,7 +162,6 @@ class ResponseFactory:final_response: Optional[SyncResponse],parser_arguments: Dict,meta: Optional[Dict] = None,- xhr_captured: Optional[List[SyncResponse]] = None,collect_history: bool = True,pierce_shadow: bool = False,max_body_bytes: Optional[int] = None,@@ -182,7 +181,6 @@ class ResponseFactory::param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked intothe `Response` object.:param meta: Additional meta data to be saved with the response.- :param xhr_captured: Optional list of captured Playwright XHR/fetch responses to convert and attach to the returned Response.:param collect_history: Optional boolean indicating whether to collect redirections history or not.:param pierce_shadow: Include open shadow roots in the HTML snapshot. Disabled by default.:param max_body_bytes: Optional text-only recorded-body limit. Skips non-text or unknown content types before reading; preserves headers and notes for unavailable bodies.@@ -241,7 +239,7 @@ class ResponseFactory:raiserequest_headers = first_response.request.headersbody_meta["request_headers_partial"] = True- response = cls._build_browser_response(+ return cls._build_browser_response({"url": page.url if page else first_response.url,"content": page_content,@@ -257,11 +255,6 @@ class ResponseFactory:},max_body_bytes,)- if xhr_captured:- response.captured_xhr = [- cls.from_playwright_response(None, p, None, {}, collect_history=False) for p in xhr_captured- ]- return response@classmethodasync def _async_process_response_history(@@ -344,7 +337,6 @@ class ResponseFactory:final_response: Optional[AsyncResponse],parser_arguments: Dict,meta: Optional[Dict] = None,- xhr_captured: Optional[List[AsyncResponse]] = None,collect_history: bool = True,pierce_shadow: bool = False,max_body_bytes: Optional[int] = None,@@ -364,7 +356,6 @@ class ResponseFactory::param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked intothe `Response` object.:param meta: Additional meta data to be saved with the response.- :param xhr_captured: Optional list of captured async Playwright XHR/fetch responses to convert and attach to the returned Response.:param collect_history: Optional boolean indicating whether to collect redirections history or not.:param pierce_shadow: Include open shadow roots in the HTML snapshot. Disabled by default.:param max_body_bytes: Optional text-only recorded-body limit. Skips non-text or unknown content types before reading; preserves headers and notes for unavailable bodies.@@ -424,7 +415,7 @@ class ResponseFactory:raiserequest_headers = first_response.request.headersbody_meta["request_headers_partial"] = True- response = cls._build_browser_response(+ return cls._build_browser_response({"url": page.url if page else first_response.url,"content": page_content,@@ -440,11 +431,6 @@ class ResponseFactory:},max_body_bytes,)- if xhr_captured:- response.captured_xhr = [- await cls.from_async_playwright_response(None, p, None, {}, collect_history=False) for p in xhr_captured- ]- return response@staticmethoddef from_http_request(response: CurlResponse, parser_arguments: Dict, meta: Optional[Dict] = None) -> Response:diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex 729f2b0..0527844 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -36,7 +36,6 @@ class Response(Selector)::param history: List of redirect responses, if any.:param meta: Metadata dictionary (e.g., proxy used).:param request: Associated spider Request object (set by crawler, in the spiders framework).- :param captured_xhr: List of captured XHR/fetch ``Response`` objects. Populated when ``capture_xhr`` is set on a browser session."""def __init__(@@ -61,6 +60,7 @@ class Response(Selector):adaptive_domain: str = cast(str, selector_config.pop("adaptive_domain", ""))self.status = status+ self.method = methodself.reason = reasonself.cookies = cookiesself.headers = headers@@ -80,7 +80,6 @@ class Response(Selector):self.meta: Dict[str, Any] = meta or {}self.request: Optional["Request"] = None # Will be set by crawler- self.captured_xhr: List["Response"] = []@propertydef body(self) -> bytes: