🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(browsers): catch background requests per session + improve encoding detection
Karim shoair197e573f2d6f
diff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.pyindex eced19a..a4e709f 100644--- a/scrapling/core/_shell_signatures.py+++ b/scrapling/core/_shell_signatures.py@@ -66,43 +66,14 @@ _FETCH_PARAMS = {"retries": int,"retry_delay": int | float,"capture_xhr": str | None,+ "record_requests": bool,+ "max_recorded_requests": int,"executable_path": Optional[str],"dns_over_https": bool,}_STEALTHY_FETCH_PARAMS = {- "headless": bool,- "disable_resources": bool,- "network_idle": bool,- "load_dom": bool,- "pierce_shadow": bool,- "wait_selector": Optional[str],- "wait_selector_state": SelectorWaitStates,- "cookies": Sequence[SetCookieParam],- "google_search": bool,- "wait": int | float,- "timezone_id": str | None,- "page_action": Optional[Callable],- "page_setup": Optional[Callable],- "proxy": Optional[str | Dict[str, str] | Tuple],- "extra_headers": Optional[Dict[str, str]],- "timeout": int | float,- "init_script": Optional[str],- "user_data_dir": str,- "selector_config": Optional[Dict],- "additional_args": Optional[Dict],- "locale": Optional[str],- "real_chrome": bool,- "cdp_url": Optional[str],- "useragent": Optional[str],- "extra_flags": Optional[List[str]],- "blocked_domains": Optional[Set[str]],- "block_ads": bool,- "retries": int,- "retry_delay": int | float,- "capture_xhr": str | None,- "executable_path": Optional[str],- "dns_over_https": bool,+ **_FETCH_PARAMS,"allow_webgl": bool,"hide_canvas": bool,"block_webrtc": bool,diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex a7fe0a5..a906a04 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -19,6 +19,7 @@ from playwright._impl._errors import Error as PlaywrightErrorfrom scrapling.parser import Selectorfrom scrapling.engines._browsers._page import PageInfo, PagePool+from scrapling.engines._browsers._network import NetworkRecorderfrom scrapling.engines._browsers._validators import validate, PlaywrightConfig, StealthConfigfrom scrapling.engines._browsers._config_tools import __default_chrome_useragent__, __default_useragent__from scrapling.engines.toolbelt.navigation import (@@ -51,8 +52,9 @@ class SyncSession:if TYPE_CHECKING:_build_context_with_proxy: Callable[..., Dict[str, Any]]- def __init__(self, max_pages: int = 1):+ def __init__(self, max_pages: int = 1, record_requests: bool = False, max_recorded_requests: int = 1000):self.max_pages = max_pages+ self.network = NetworkRecorder(enabled=record_requests, max_requests=max_recorded_requests)self.page_pool = PagePool(max_pages)self._max_wait_for_page = 60self.playwright: Any = None@@ -71,6 +73,7 @@ class SyncSession:def close(self): # pragma: no cover"""Close all resources"""+ self.network._detach()if not self._is_alive:return@@ -104,6 +107,7 @@ class SyncSession:if config.cookies: # pragma: no coverctx.add_cookies(config.cookies)+ self.network._attach(ctx)return ctxdef _get_page(@@ -224,6 +228,7 @@ class SyncSession:finally:if page_info is not None:self.page_pool.remove_page(page_info)+ self.network._detach(context)context.close()else:# Standard mode: use PagePool with persistent context@@ -245,8 +250,9 @@ class AsyncSession:if TYPE_CHECKING:_build_context_with_proxy: Callable[..., Dict[str, Any]]- def __init__(self, max_pages: int = 1):+ def __init__(self, max_pages: int = 1, record_requests: bool = False, max_recorded_requests: int = 1000):self.max_pages = max_pages+ self.network = NetworkRecorder(enabled=record_requests, max_requests=max_recorded_requests, asynchronous=True)self.page_pool = PagePool(max_pages)self._max_wait_for_page = 60self.playwright: Any = None@@ -266,6 +272,7 @@ class AsyncSession:async def close(self):"""Close all resources"""+ self.network._detach()if not self._is_alive: # pragma: no coverreturn@@ -301,6 +308,7 @@ class AsyncSession:if config.cookies: # pragma: no coverawait ctx.add_cookies(config.cookies)+ self.network._attach(ctx)return ctxasync def _get_page(@@ -441,6 +449,7 @@ class AsyncSession:finally:if page_info is not None:self.page_pool.remove_page(page_info)+ self.network._detach(context)await context.close()else:# Standard mode: use PagePool with persistent contextdiff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex 048c90e..31c4179 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -38,6 +38,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin):def __init__(self, **kwargs: Unpack[PlaywrightSession]):"""A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.+ :param record_requests: Save completed browser requests and responses. Failed or unfinished requests are omitted. Defaults to False.+ :param max_recorded_requests: Maximum retained requests per session. Defaults to 1000; older entries are removed.:param headless: Run the browser in headless/hidden (default), or headful/visible mode.:param disable_resources: Drop requests for unnecessary resources for a speed boost.Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.@@ -68,7 +70,10 @@ class DynamicSession(SyncSession, DynamicSessionMixin)::param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings."""self.__validate__(**kwargs)- super().__init__()+ super().__init__(+ record_requests=self._config.record_requests,+ max_recorded_requests=self._config.max_recorded_requests,+ )def start(self):"""Create a browser for this instance and context."""@@ -229,6 +234,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):def __init__(self, **kwargs: Unpack[PlaywrightSession]):"""A Browser session manager with page pooling+ :param record_requests: Save completed browser requests and responses. Failed or unfinished requests are omitted. Defaults to False.+ :param max_recorded_requests: Maximum retained requests per session. Defaults to 1000; older entries are removed.:param headless: Run the browser in headless/hidden (default), or headful/visible mode.:param disable_resources: Drop requests for unnecessary resources for a speed boost.Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.@@ -260,7 +267,11 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin)::param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings."""self.__validate__(**kwargs)- super().__init__(max_pages=self._config.max_pages)+ super().__init__(+ max_pages=self._config.max_pages,+ record_requests=self._config.record_requests,+ max_recorded_requests=self._config.max_recorded_requests,+ )async def start(self) -> None:"""Create a browser for this instance and context."""diff --git a/scrapling/engines/_browsers/_network.py b/scrapling/engines/_browsers/_network.pynew file mode 100644index 0000000..27b4648--- /dev/null+++ b/scrapling/engines/_browsers/_network.py@@ -0,0 +1,154 @@+from re import compile as compile_regex, error as RegexError+from collections import OrderedDict+from contextlib import suppress+from dataclasses import dataclass++from scrapling.core._types import Any, Callable, Dict, List, Optional+from scrapling.engines.toolbelt.custom import Response+from scrapling.engines.toolbelt.convertor import ResponseFactory++MAX_BODY_BYTES = 1024 * 1024+MAX_TOTAL_BODY_BYTES = 20 * 1024 * 1024+++@dataclass(slots=True)+class NetworkRequest:+ """Saved request data and response, with a session-local ID."""++ id: int+ url: str+ method: str+ resource_type: str+ request_headers: Dict[str, str]+ request_body: Optional[bytes]+ response_headers: Dict[str, str]+ status: int+ response: Response+++class NetworkRecorder:+ """Keep bounded request history and saved responses for browser sessions."""++ def __init__(self, enabled: bool = False, max_requests: int = 1000, asynchronous: bool = False):+ if max_requests < 1:+ raise ValueError("max_requests must be greater than zero.")+ self.enabled = enabled+ self.max_requests = max_requests+ self.dropped_count = 0+ self.last_id = 0+ self._records: OrderedDict[int, NetworkRequest] = OrderedDict()+ self._contexts: Dict[Any, Dict[str, Callable[..., Any]]] = {}+ self._body_bytes = 0+ self._asynchronous = asynchronous+ self._generation = 0++ def _attach(self, context: Any) -> None:+ """Save completed responses until the context is detached or closed."""+ if not self.enabled or context in self._contexts:+ return+ capture = self._capture_async if self._asynchronous else self._capture+ listeners: Dict[str, Callable[..., Any]] = {+ "requestfinished": lambda request: capture(request, self._generation, context, listeners),+ "close": lambda *args: self._detach(context),+ }+ self._contexts[context] = listeners+ try:+ for event, listener in listeners.items():+ context.on(event, listener)+ except Exception:+ self._detach(context)++ def _capture(self, request: Any, generation: int, context: Any, listeners: Dict) -> None:+ with suppress(Exception):+ response = ResponseFactory.from_playwright_response(+ None,+ request.existing_response,+ None,+ {"method": request.method, "_log": False},+ collect_history=False,+ max_body_bytes=MAX_BODY_BYTES,+ )+ self._store(request, response, generation, context, listeners)++ async def _capture_async(self, request: Any, generation: int, context: Any, listeners: Dict) -> None:+ with suppress(Exception):+ response = await ResponseFactory.from_async_playwright_response(+ None,+ request.existing_response,+ None,+ {"method": request.method, "_log": False},+ collect_history=False,+ max_body_bytes=MAX_BODY_BYTES,+ )+ self._store(request, response, generation, context, listeners)++ def _store(self, request: Any, response: Response, generation: int, context: Any, listeners: Dict) -> None:+ if generation != self._generation or self._contexts.get(context) is not listeners:+ return+ self.last_id += 1+ self._records[self.last_id] = NetworkRequest(+ self.last_id,+ request.url,+ request.method,+ request.resource_type,+ response.request_headers,+ request.post_data_buffer,+ response.headers,+ response.status,+ response,+ )+ self._body_bytes += len(response.body)+ while len(self._records) > self.max_requests or self._body_bytes > MAX_TOTAL_BODY_BYTES:+ self._body_bytes -= len(self._records.popitem(last=False)[1].response.body)+ self.dropped_count += 1++ def get(self, request_id: int) -> Optional[NetworkRequest]:+ """Return one retained request, or None if its ID is unavailable."""+ return self._records.get(request_id)++ def search(+ self,+ url_pattern: Optional[str] = None,+ method: Optional[str] = None,+ resource_type: Optional[str] = None,+ status: Optional[int] = None,+ after_id: int = 0,+ limit: int = 100,+ include_static: bool = True,+ ) -> List[NetworkRequest]:+ """Return matching requests in ID order; the URL pattern is a regular expression."""+ if limit < 1 or after_id < 0:+ raise ValueError("limit must be positive and after_id must be nonnegative.")+ try:+ pattern = compile_regex(url_pattern) if url_pattern is not None else None+ except RegexError as error:+ raise ValueError(f"Invalid URL pattern: {error}") from error+ found = []+ for record in self._records.values():+ if (+ record.id <= after_id+ or (pattern is not None and not pattern.search(record.url))+ or (method is not None and record.method != method.upper())+ or (resource_type is not None and record.resource_type != resource_type)+ or (status is not None and record.status != status)+ or (not include_static and record.resource_type not in {"xhr", "fetch"} and record.status < 400)+ ):+ continue+ found.append(record)+ if len(found) == limit:+ break+ return found++ def clear(self) -> None:+ """Clear history without reusing request IDs."""+ self._records.clear()+ self._body_bytes = 0+ self.dropped_count = 0+ self._generation += 1++ def _detach(self, context: Any = None) -> None:+ """Stop recording one context, or all contexts, and keep the history."""+ for context in (context,) if context is not None else tuple(self._contexts):+ for event, listener in self._contexts.pop(context, {}).items():+ with suppress(Exception):+ context.remove_listener(event, listener)diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.pyindex 8aabfa2..419f8e0 100644--- a/scrapling/engines/_browsers/_stealth.py+++ b/scrapling/engines/_browsers/_stealth.py@@ -39,6 +39,8 @@ class StealthySession(SyncSession, StealthySessionMixin):def __init__(self, **kwargs: Unpack[StealthSession]):"""A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.+ :param record_requests: Save completed browser requests and responses. Failed or unfinished requests are omitted. Defaults to False.+ :param max_recorded_requests: Maximum retained requests per session. Defaults to 1000; older entries are removed.:param headless: Run the browser in headless/hidden (default), or headful/visible mode.:param disable_resources: Drop requests for unnecessary resources for a speed boost.Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.@@ -73,7 +75,10 @@ class StealthySession(SyncSession, StealthySessionMixin)::param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings."""self.__validate__(**kwargs)- super().__init__()+ super().__init__(+ record_requests=self._config.record_requests,+ max_recorded_requests=self._config.max_recorded_requests,+ )def start(self) -> None:"""Create a browser for this instance and context."""@@ -328,6 +333,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):def __init__(self, **kwargs: Unpack[StealthSession]):"""A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.+ :param record_requests: Save completed browser requests and responses. Failed or unfinished requests are omitted. Defaults to False.+ :param max_recorded_requests: Maximum retained requests per session. Defaults to 1000; older entries are removed.:param headless: Run the browser in headless/hidden (default), or headful/visible mode.:param disable_resources: Drop requests for unnecessary resources for a speed boost.Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.@@ -362,7 +369,11 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin)::param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings."""self.__validate__(**kwargs)- super().__init__(max_pages=self._config.max_pages)+ super().__init__(+ max_pages=self._config.max_pages,+ record_requests=self._config.record_requests,+ max_recorded_requests=self._config.max_recorded_requests,+ )async def start(self) -> None:"""Create a browser for this instance and context."""diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.pyindex 3c37d43..6e8d5ca 100644--- a/scrapling/engines/_browsers/_types.py+++ b/scrapling/engines/_browsers/_types.py@@ -94,6 +94,8 @@ class PlaywrightSession(TypedDict, total=False):retries: intretry_delay: int | floatcapture_xhr: str | None+ record_requests: bool+ max_recorded_requests: intexecutable_path: Optional[str]dns_over_https: booldiff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.pyindex 0d2dd5a..ced5159 100644--- a/scrapling/engines/_browsers/_validators.py+++ b/scrapling/engines/_browsers/_validators.py@@ -91,6 +91,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):retries: RetriesCount = 3retry_delay: Seconds = 1capture_xhr: str | None = None+ record_requests: bool = False+ max_recorded_requests: Annotated[int, Meta(ge=1)] = 1000executable_path: Optional[str] = Nonedns_over_https: bool = Falsediff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.pyindex 60be443..a1ffc80 100644--- a/scrapling/engines/toolbelt/convertor.py+++ b/scrapling/engines/toolbelt/convertor.py@@ -1,6 +1,8 @@+from email.message import Message+from email.utils import collapse_rfc2231_valuefrom functools import lru_cache-from re import compile as re_compile+from lxml.etree import LxmlErrorfrom curl_cffi.requests import Response as CurlResponsefrom playwright._impl._errors import Error as PlaywrightErrorfrom patchright._impl._errors import Error as PatchrightError@@ -11,9 +13,6 @@ from scrapling.core.utils import logfrom .custom import Response, StatusTextfrom scrapling.core._types import Dict, List, Optional-__CHARSET_RE__ = re_compile(r"""charset=["']?([\w-]+)""")--SHADOW_SNAPSHOT_JS = r"""() => {if (!Array.from(document.querySelectorAll('*')).some(node => node.shadowRoot)) return null;@@ -61,17 +60,41 @@ class ResponseFactory:response objects, and managing encoding, headers, cookies, and other attributes."""- @classmethod+ @staticmethod@lru_cache(maxsize=16)- def __extract_browser_encoding(cls, content_type: str | None, default: str = "utf-8") -> str:- """Extract browser encoding from headers.- Ex: from header "content-type: text/html; charset=utf-8" -> "utf-8- """- if content_type:- # Because Playwright can't do that by themselves like all libraries for some reason :3- match = __CHARSET_RE__.search(content_type)- return match.group(1) if match else default- return default+ def _extract_encoding(content_type: Optional[str], default: str = "utf-8") -> str:+ """Read the Content-Type charset, or use the default."""+ message = Message()+ message["content-type"] = content_type or ""+ return collapse_rfc2231_value(message.get_param("charset") or default).strip("'\"") or default++ @staticmethod+ def _check_body_size(size: int | str, max_body_bytes: Optional[int]) -> None:+ if max_body_bytes is not None:+ try:+ size = int(size)+ except ValueError:+ return+ if size > max_body_bytes:+ raise ValueError("too large; not saved.")++ @staticmethod+ def _build_browser_response(arguments: Dict, max_body_bytes: Optional[int]) -> Response:+ if max_body_bytes is not None:+ try:+ arguments["content"].decode("utf-8")+ except UnicodeDecodeError:+ pass+ else:+ arguments["encoding"] = "utf-8"+ try:+ return Response(**arguments)+ except (LookupError, ValueError, LxmlError):+ if max_body_bytes is None:+ raise+ response = Response(**{**arguments, "content": b"", "encoding": "utf-8"})+ response._raw_body = arguments["content"]+ return response@classmethoddef _process_response_history(cls, first_response: SyncResponse, parser_arguments: Dict) -> list[Response]:@@ -94,9 +117,7 @@ class ResponseFactory:"reason": (current_response.status_text or StatusText.get(current_response.status))if current_responseelse StatusText.get(301),- "encoding": cls.__extract_browser_encoding(- current_response.headers.get("content-type", "")- )+ "encoding": cls._extract_encoding(current_response.headers.get("content-type", ""))if current_responseelse "utf-8","cookies": tuple(),@@ -127,6 +148,7 @@ class ResponseFactory:xhr_captured: Optional[List[SyncResponse]] = None,collect_history: bool = True,pierce_shadow: bool = False,+ max_body_bytes: Optional[int] = None,) -> Response:"""Transforms a Playwright response into an internal `Response` object, encapsulating@@ -146,6 +168,7 @@ class ResponseFactory::param xhr_captured: Optional list of captured Playwright XHR/fetch responses to convert and attach to the returned Response.:param collect_history: Optional boolean indicating whether to collect redirections history or not.:param pierce_shadow: Include open shadow roots in the HTML snapshot. Disabled by default.+ :param max_body_bytes: Optional recorded-body limit. Adds a note for skipped or unreadable bodies and preserves partial headers.:return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.:rtype: Response"""@@ -154,41 +177,66 @@ class ResponseFactory:if not final_response:raise ValueError("Failed to get a response from the page")- encoding = cls.__extract_browser_encoding(final_response.headers.get("content-type", ""))+ encoding = cls._extract_encoding(final_response.headers.get("content-type", ""))# PlayWright API sometimes give empty status text for some reason!status_text = final_response.status_text or StatusText.get(final_response.status)history = cls._process_response_history(first_response, parser_arguments) if collect_history else []snapshot = None+ body_meta: Dict = {}try:- if page and "html" in final_response.all_headers().get("content-type", ""):- if pierce_shadow:- try:- snapshot = page.evaluate(SHADOW_SNAPSHOT_JS)- except Exception as e:- log.warning(f"Failed to extract shadow roots, using page HTML: {e}")- page_content = (snapshot or cls._get_page_content(page)).encode("utf-8")- encoding = "utf-8"+ if max_body_bytes is not None and (+ final_response.request.method == "HEAD" or final_response.status in (204, 205, 304)+ ):+ page_content = b""else:- page_content = final_response.body()+ cls._check_body_size(final_response.headers.get("content-length", "0"), max_body_bytes)+ if page and "html" in final_response.all_headers().get("content-type", ""):+ if pierce_shadow:+ try:+ snapshot = page.evaluate(SHADOW_SNAPSHOT_JS)+ except Exception as e:+ log.warning(f"Failed to extract shadow roots, using page HTML: {e}")+ page_content = (snapshot or cls._get_page_content(page)).encode("utf-8")+ encoding = "utf-8"+ else:+ page_content = final_response.body()+ cls._check_body_size(len(page_content), max_body_bytes)except Exception as e: # pragma: no cover- log.error(f"Error getting page content: {e}")+ if max_body_bytes is None:+ log.error(f"Error getting page content: {e}")page_content = b""+ body_meta["body_note"] = str(e) or "could not be saved."- response = Response(- **{+ try:+ headers = first_response.all_headers()+ except Exception:+ if max_body_bytes is None:+ raise+ headers = first_response.headers+ body_meta["headers_partial"] = True+ try:+ request_headers = first_response.request.all_headers()+ except Exception:+ if max_body_bytes is None:+ raise+ request_headers = first_response.request.headers+ body_meta["request_headers_partial"] = True+ response = cls._build_browser_response(+ {"url": page.url if page else first_response.url,"content": page_content,"status": final_response.status,"reason": status_text,"encoding": encoding,"cookies": tuple(dict(cookie) for cookie in page.context.cookies()) if page else {},- "headers": first_response.all_headers(),- "request_headers": first_response.request.all_headers(),+ "headers": headers,+ "request_headers": request_headers,"history": history,- "meta": meta,+ "meta": {**(meta or {}), **body_meta} if max_body_bytes is not None else meta,**parser_arguments,- }+ },+ max_body_bytes,)if xhr_captured:response.captured_xhr = [@@ -219,9 +267,7 @@ class ResponseFactory:"reason": (current_response.status_text or StatusText.get(current_response.status))if current_responseelse StatusText.get(301),- "encoding": cls.__extract_browser_encoding(- current_response.headers.get("content-type", "")- )+ "encoding": cls._extract_encoding(current_response.headers.get("content-type", ""))if current_responseelse "utf-8","cookies": tuple(),@@ -282,6 +328,7 @@ class ResponseFactory:xhr_captured: Optional[List[AsyncResponse]] = None,collect_history: bool = True,pierce_shadow: bool = False,+ max_body_bytes: Optional[int] = None,) -> Response:"""Transforms a Playwright response into an internal `Response` object, encapsulating@@ -301,6 +348,7 @@ class ResponseFactory::param xhr_captured: Optional list of captured async Playwright XHR/fetch responses to convert and attach to the returned Response.:param collect_history: Optional boolean indicating whether to collect redirections history or not.:param pierce_shadow: Include open shadow roots in the HTML snapshot. Disabled by default.+ :param max_body_bytes: Optional recorded-body limit. Adds a note for skipped or unreadable bodies and preserves partial headers.:return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.:rtype: Response@@ -310,41 +358,66 @@ class ResponseFactory:if not final_response:raise ValueError("Failed to get a response from the page")- encoding = cls.__extract_browser_encoding(final_response.headers.get("content-type", ""))+ encoding = cls._extract_encoding(final_response.headers.get("content-type", ""))# PlayWright API sometimes give empty status text for some reason!status_text = final_response.status_text or StatusText.get(final_response.status)history = await cls._async_process_response_history(first_response, parser_arguments) if collect_history else []snapshot = None+ body_meta: Dict = {}try:- if page and "html" in (await final_response.all_headers()).get("content-type", ""):- if pierce_shadow:- try:- snapshot = await page.evaluate(SHADOW_SNAPSHOT_JS)- except Exception as e:- log.warning(f"Failed to extract shadow roots, using page HTML: {e}")- page_content = (snapshot or await cls._get_async_page_content(page)).encode("utf-8")- encoding = "utf-8"+ if max_body_bytes is not None and (+ final_response.request.method == "HEAD" or final_response.status in (204, 205, 304)+ ):+ page_content = b""else:- page_content = await final_response.body()+ cls._check_body_size(final_response.headers.get("content-length", "0"), max_body_bytes)+ if page and "html" in (await final_response.all_headers()).get("content-type", ""):+ if pierce_shadow:+ try:+ snapshot = await page.evaluate(SHADOW_SNAPSHOT_JS)+ except Exception as e:+ log.warning(f"Failed to extract shadow roots, using page HTML: {e}")+ page_content = (snapshot or await cls._get_async_page_content(page)).encode("utf-8")+ encoding = "utf-8"+ else:+ page_content = await final_response.body()+ cls._check_body_size(len(page_content), max_body_bytes)except Exception as e: # pragma: no cover- log.error(f"Error getting page content in async: {e}")+ if max_body_bytes is None:+ log.error(f"Error getting page content in async: {e}")page_content = b""+ body_meta["body_note"] = str(e) or "could not be saved."- response = Response(- **{+ try:+ headers = await first_response.all_headers()+ except Exception:+ if max_body_bytes is None:+ raise+ headers = first_response.headers+ body_meta["headers_partial"] = True+ try:+ request_headers = await first_response.request.all_headers()+ except Exception:+ if max_body_bytes is None:+ raise+ request_headers = first_response.request.headers+ body_meta["request_headers_partial"] = True+ response = cls._build_browser_response(+ {"url": page.url if page else first_response.url,"content": page_content,"status": final_response.status,"reason": status_text,"encoding": encoding,"cookies": tuple(dict(cookie) for cookie in await page.context.cookies()) if page else {},- "headers": await first_response.all_headers(),- "request_headers": await first_response.request.all_headers(),+ "headers": headers,+ "request_headers": request_headers,"history": history,- "meta": meta,+ "meta": {**(meta or {}), **body_meta} if max_body_bytes is not None else meta,**parser_arguments,- }+ },+ max_body_bytes,)if xhr_captured:response.captured_xhr = [diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex ff2f244..729f2b0 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -52,6 +52,8 @@ class Response(Selector):method: str = "GET",history: List | None = None,meta: Dict[str, Any] | None = None,+ *,+ _log: bool = True,**selector_config: Any,):if isinstance(content, str):@@ -70,8 +72,8 @@ class Response(Selector):encoding=encoding,**selector_config,)- # For easier debugging while working from a Python shell- log.info(f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})")+ if _log:+ log.info(f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})")if meta and not isinstance(meta, dict):raise TypeError(f"Response meta should be dictionary but got {type(meta).__name__} instead!")