Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(browsers): catch background requests per session + improve encoding detection

Karim shoair197e573f2d6f
diff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.py
index eced19a..a4e709f 100644
--- a/scrapling/core/_shell_signatures.py
+++ b/scrapling/core/_shell_signatures.py
@@ -66,43 +66,14 @@ _FETCH_PARAMS = {
"retries": int,
"retry_delay": int | float,
"capture_xhr": str | None,
+ "record_requests": bool,
+ "max_recorded_requests": int,
"executable_path": Optional[str],
"dns_over_https": bool,
}
_STEALTHY_FETCH_PARAMS = {
- "headless": bool,
- "disable_resources": bool,
- "network_idle": bool,
- "load_dom": bool,
- "pierce_shadow": bool,
- "wait_selector": Optional[str],
- "wait_selector_state": SelectorWaitStates,
- "cookies": Sequence[SetCookieParam],
- "google_search": bool,
- "wait": int | float,
- "timezone_id": str | None,
- "page_action": Optional[Callable],
- "page_setup": Optional[Callable],
- "proxy": Optional[str | Dict[str, str] | Tuple],
- "extra_headers": Optional[Dict[str, str]],
- "timeout": int | float,
- "init_script": Optional[str],
- "user_data_dir": str,
- "selector_config": Optional[Dict],
- "additional_args": Optional[Dict],
- "locale": Optional[str],
- "real_chrome": bool,
- "cdp_url": Optional[str],
- "useragent": Optional[str],
- "extra_flags": Optional[List[str]],
- "blocked_domains": Optional[Set[str]],
- "block_ads": bool,
- "retries": int,
- "retry_delay": int | float,
- "capture_xhr": str | None,
- "executable_path": Optional[str],
- "dns_over_https": bool,
+ **_FETCH_PARAMS,
"allow_webgl": bool,
"hide_canvas": bool,
"block_webrtc": bool,
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py
index a7fe0a5..a906a04 100644
--- a/scrapling/engines/_browsers/_base.py
+++ b/scrapling/engines/_browsers/_base.py
@@ -19,6 +19,7 @@ from playwright._impl._errors import Error as PlaywrightError
from scrapling.parser import Selector
from scrapling.engines._browsers._page import PageInfo, PagePool
+from scrapling.engines._browsers._network import NetworkRecorder
from scrapling.engines._browsers._validators import validate, PlaywrightConfig, StealthConfig
from scrapling.engines._browsers._config_tools import __default_chrome_useragent__, __default_useragent__
from scrapling.engines.toolbelt.navigation import (
@@ -51,8 +52,9 @@ class SyncSession:
if TYPE_CHECKING:
_build_context_with_proxy: Callable[..., Dict[str, Any]]
- def __init__(self, max_pages: int = 1):
+ def __init__(self, max_pages: int = 1, record_requests: bool = False, max_recorded_requests: int = 1000):
self.max_pages = max_pages
+ self.network = NetworkRecorder(enabled=record_requests, max_requests=max_recorded_requests)
self.page_pool = PagePool(max_pages)
self._max_wait_for_page = 60
self.playwright: Any = None
@@ -71,6 +73,7 @@ class SyncSession:
def close(self): # pragma: no cover
"""Close all resources"""
+ self.network._detach()
if not self._is_alive:
return
@@ -104,6 +107,7 @@ class SyncSession:
if config.cookies: # pragma: no cover
ctx.add_cookies(config.cookies)
+ self.network._attach(ctx)
return ctx
def _get_page(
@@ -224,6 +228,7 @@ class SyncSession:
finally:
if page_info is not None:
self.page_pool.remove_page(page_info)
+ self.network._detach(context)
context.close()
else:
# Standard mode: use PagePool with persistent context
@@ -245,8 +250,9 @@ class AsyncSession:
if TYPE_CHECKING:
_build_context_with_proxy: Callable[..., Dict[str, Any]]
- def __init__(self, max_pages: int = 1):
+ def __init__(self, max_pages: int = 1, record_requests: bool = False, max_recorded_requests: int = 1000):
self.max_pages = max_pages
+ self.network = NetworkRecorder(enabled=record_requests, max_requests=max_recorded_requests, asynchronous=True)
self.page_pool = PagePool(max_pages)
self._max_wait_for_page = 60
self.playwright: Any = None
@@ -266,6 +272,7 @@ class AsyncSession:
async def close(self):
"""Close all resources"""
+ self.network._detach()
if not self._is_alive: # pragma: no cover
return
@@ -301,6 +308,7 @@ class AsyncSession:
if config.cookies: # pragma: no cover
await ctx.add_cookies(config.cookies)
+ self.network._attach(ctx)
return ctx
async def _get_page(
@@ -441,6 +449,7 @@ class AsyncSession:
finally:
if page_info is not None:
self.page_pool.remove_page(page_info)
+ self.network._detach(context)
await context.close()
else:
# Standard mode: use PagePool with persistent context
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index 048c90e..31c4179 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -38,6 +38,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
def __init__(self, **kwargs: Unpack[PlaywrightSession]):
"""A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.
+ :param record_requests: Save completed browser requests and responses. Failed or unfinished requests are omitted. Defaults to False.
+ :param max_recorded_requests: Maximum retained requests per session. Defaults to 1000; older entries are removed.
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
@@ -68,7 +70,10 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
"""
self.__validate__(**kwargs)
- super().__init__()
+ super().__init__(
+ record_requests=self._config.record_requests,
+ max_recorded_requests=self._config.max_recorded_requests,
+ )
def start(self):
"""Create a browser for this instance and context."""
@@ -229,6 +234,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
def __init__(self, **kwargs: Unpack[PlaywrightSession]):
"""A Browser session manager with page pooling
+ :param record_requests: Save completed browser requests and responses. Failed or unfinished requests are omitted. Defaults to False.
+ :param max_recorded_requests: Maximum retained requests per session. Defaults to 1000; older entries are removed.
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
@@ -260,7 +267,11 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
"""
self.__validate__(**kwargs)
- super().__init__(max_pages=self._config.max_pages)
+ super().__init__(
+ max_pages=self._config.max_pages,
+ record_requests=self._config.record_requests,
+ max_recorded_requests=self._config.max_recorded_requests,
+ )
async def start(self) -> None:
"""Create a browser for this instance and context."""
diff --git a/scrapling/engines/_browsers/_network.py b/scrapling/engines/_browsers/_network.py
new file mode 100644
index 0000000..27b4648
--- /dev/null
+++ b/scrapling/engines/_browsers/_network.py
@@ -0,0 +1,154 @@
+from re import compile as compile_regex, error as RegexError
+from collections import OrderedDict
+from contextlib import suppress
+from dataclasses import dataclass
+
+from scrapling.core._types import Any, Callable, Dict, List, Optional
+from scrapling.engines.toolbelt.custom import Response
+from scrapling.engines.toolbelt.convertor import ResponseFactory
+
+MAX_BODY_BYTES = 1024 * 1024
+MAX_TOTAL_BODY_BYTES = 20 * 1024 * 1024
+
+
+@dataclass(slots=True)
+class NetworkRequest:
+ """Saved request data and response, with a session-local ID."""
+
+ id: int
+ url: str
+ method: str
+ resource_type: str
+ request_headers: Dict[str, str]
+ request_body: Optional[bytes]
+ response_headers: Dict[str, str]
+ status: int
+ response: Response
+
+
+class NetworkRecorder:
+ """Keep bounded request history and saved responses for browser sessions."""
+
+ def __init__(self, enabled: bool = False, max_requests: int = 1000, asynchronous: bool = False):
+ if max_requests < 1:
+ raise ValueError("max_requests must be greater than zero.")
+ self.enabled = enabled
+ self.max_requests = max_requests
+ self.dropped_count = 0
+ self.last_id = 0
+ self._records: OrderedDict[int, NetworkRequest] = OrderedDict()
+ self._contexts: Dict[Any, Dict[str, Callable[..., Any]]] = {}
+ self._body_bytes = 0
+ self._asynchronous = asynchronous
+ self._generation = 0
+
+ def _attach(self, context: Any) -> None:
+ """Save completed responses until the context is detached or closed."""
+ if not self.enabled or context in self._contexts:
+ return
+ capture = self._capture_async if self._asynchronous else self._capture
+ listeners: Dict[str, Callable[..., Any]] = {
+ "requestfinished": lambda request: capture(request, self._generation, context, listeners),
+ "close": lambda *args: self._detach(context),
+ }
+ self._contexts[context] = listeners
+ try:
+ for event, listener in listeners.items():
+ context.on(event, listener)
+ except Exception:
+ self._detach(context)
+
+ def _capture(self, request: Any, generation: int, context: Any, listeners: Dict) -> None:
+ with suppress(Exception):
+ response = ResponseFactory.from_playwright_response(
+ None,
+ request.existing_response,
+ None,
+ {"method": request.method, "_log": False},
+ collect_history=False,
+ max_body_bytes=MAX_BODY_BYTES,
+ )
+ self._store(request, response, generation, context, listeners)
+
+ async def _capture_async(self, request: Any, generation: int, context: Any, listeners: Dict) -> None:
+ with suppress(Exception):
+ response = await ResponseFactory.from_async_playwright_response(
+ None,
+ request.existing_response,
+ None,
+ {"method": request.method, "_log": False},
+ collect_history=False,
+ max_body_bytes=MAX_BODY_BYTES,
+ )
+ self._store(request, response, generation, context, listeners)
+
+ def _store(self, request: Any, response: Response, generation: int, context: Any, listeners: Dict) -> None:
+ if generation != self._generation or self._contexts.get(context) is not listeners:
+ return
+ self.last_id += 1
+ self._records[self.last_id] = NetworkRequest(
+ self.last_id,
+ request.url,
+ request.method,
+ request.resource_type,
+ response.request_headers,
+ request.post_data_buffer,
+ response.headers,
+ response.status,
+ response,
+ )
+ self._body_bytes += len(response.body)
+ while len(self._records) > self.max_requests or self._body_bytes > MAX_TOTAL_BODY_BYTES:
+ self._body_bytes -= len(self._records.popitem(last=False)[1].response.body)
+ self.dropped_count += 1
+
+ def get(self, request_id: int) -> Optional[NetworkRequest]:
+ """Return one retained request, or None if its ID is unavailable."""
+ return self._records.get(request_id)
+
+ def search(
+ self,
+ url_pattern: Optional[str] = None,
+ method: Optional[str] = None,
+ resource_type: Optional[str] = None,
+ status: Optional[int] = None,
+ after_id: int = 0,
+ limit: int = 100,
+ include_static: bool = True,
+ ) -> List[NetworkRequest]:
+ """Return matching requests in ID order; the URL pattern is a regular expression."""
+ if limit < 1 or after_id < 0:
+ raise ValueError("limit must be positive and after_id must be nonnegative.")
+ try:
+ pattern = compile_regex(url_pattern) if url_pattern is not None else None
+ except RegexError as error:
+ raise ValueError(f"Invalid URL pattern: {error}") from error
+ found = []
+ for record in self._records.values():
+ if (
+ record.id <= after_id
+ or (pattern is not None and not pattern.search(record.url))
+ or (method is not None and record.method != method.upper())
+ or (resource_type is not None and record.resource_type != resource_type)
+ or (status is not None and record.status != status)
+ or (not include_static and record.resource_type not in {"xhr", "fetch"} and record.status < 400)
+ ):
+ continue
+ found.append(record)
+ if len(found) == limit:
+ break
+ return found
+
+ def clear(self) -> None:
+ """Clear history without reusing request IDs."""
+ self._records.clear()
+ self._body_bytes = 0
+ self.dropped_count = 0
+ self._generation += 1
+
+ def _detach(self, context: Any = None) -> None:
+ """Stop recording one context, or all contexts, and keep the history."""
+ for context in (context,) if context is not None else tuple(self._contexts):
+ for event, listener in self._contexts.pop(context, {}).items():
+ with suppress(Exception):
+ context.remove_listener(event, listener)
diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py
index 8aabfa2..419f8e0 100644
--- a/scrapling/engines/_browsers/_stealth.py
+++ b/scrapling/engines/_browsers/_stealth.py
@@ -39,6 +39,8 @@ class StealthySession(SyncSession, StealthySessionMixin):
def __init__(self, **kwargs: Unpack[StealthSession]):
"""A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.
+ :param record_requests: Save completed browser requests and responses. Failed or unfinished requests are omitted. Defaults to False.
+ :param max_recorded_requests: Maximum retained requests per session. Defaults to 1000; older entries are removed.
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
@@ -73,7 +75,10 @@ class StealthySession(SyncSession, StealthySessionMixin):
:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
"""
self.__validate__(**kwargs)
- super().__init__()
+ super().__init__(
+ record_requests=self._config.record_requests,
+ max_recorded_requests=self._config.max_recorded_requests,
+ )
def start(self) -> None:
"""Create a browser for this instance and context."""
@@ -328,6 +333,8 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
def __init__(self, **kwargs: Unpack[StealthSession]):
"""A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.
+ :param record_requests: Save completed browser requests and responses. Failed or unfinished requests are omitted. Defaults to False.
+ :param max_recorded_requests: Maximum retained requests per session. Defaults to 1000; older entries are removed.
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
@@ -362,7 +369,11 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.
"""
self.__validate__(**kwargs)
- super().__init__(max_pages=self._config.max_pages)
+ super().__init__(
+ max_pages=self._config.max_pages,
+ record_requests=self._config.record_requests,
+ max_recorded_requests=self._config.max_recorded_requests,
+ )
async def start(self) -> None:
"""Create a browser for this instance and context."""
diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py
index 3c37d43..6e8d5ca 100644
--- a/scrapling/engines/_browsers/_types.py
+++ b/scrapling/engines/_browsers/_types.py
@@ -94,6 +94,8 @@ class PlaywrightSession(TypedDict, total=False):
retries: int
retry_delay: int | float
capture_xhr: str | None
+ record_requests: bool
+ max_recorded_requests: int
executable_path: Optional[str]
dns_over_https: bool
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index 0d2dd5a..ced5159 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -91,6 +91,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
retries: RetriesCount = 3
retry_delay: Seconds = 1
capture_xhr: str | None = None
+ record_requests: bool = False
+ max_recorded_requests: Annotated[int, Meta(ge=1)] = 1000
executable_path: Optional[str] = None
dns_over_https: bool = False
diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py
index 60be443..a1ffc80 100644
--- a/scrapling/engines/toolbelt/convertor.py
+++ b/scrapling/engines/toolbelt/convertor.py
@@ -1,6 +1,8 @@
+from email.message import Message
+from email.utils import collapse_rfc2231_value
from functools import lru_cache
-from re import compile as re_compile
+from lxml.etree import LxmlError
from curl_cffi.requests import Response as CurlResponse
from playwright._impl._errors import Error as PlaywrightError
from patchright._impl._errors import Error as PatchrightError
@@ -11,9 +13,6 @@ from scrapling.core.utils import log
from .custom import Response, StatusText
from scrapling.core._types import Dict, List, Optional
-__CHARSET_RE__ = re_compile(r"""charset=["']?([\w-]+)""")
-
-
SHADOW_SNAPSHOT_JS = r"""
() => {
if (!Array.from(document.querySelectorAll('*')).some(node => node.shadowRoot)) return null;
@@ -61,17 +60,41 @@ class ResponseFactory:
response objects, and managing encoding, headers, cookies, and other attributes.
"""
- @classmethod
+ @staticmethod
@lru_cache(maxsize=16)
- def __extract_browser_encoding(cls, content_type: str | None, default: str = "utf-8") -> str:
- """Extract browser encoding from headers.
- Ex: from header "content-type: text/html; charset=utf-8" -> "utf-8
- """
- if content_type:
- # Because Playwright can't do that by themselves like all libraries for some reason :3
- match = __CHARSET_RE__.search(content_type)
- return match.group(1) if match else default
- return default
+ def _extract_encoding(content_type: Optional[str], default: str = "utf-8") -> str:
+ """Read the Content-Type charset, or use the default."""
+ message = Message()
+ message["content-type"] = content_type or ""
+ return collapse_rfc2231_value(message.get_param("charset") or default).strip("'\"") or default
+
+ @staticmethod
+ def _check_body_size(size: int | str, max_body_bytes: Optional[int]) -> None:
+ if max_body_bytes is not None:
+ try:
+ size = int(size)
+ except ValueError:
+ return
+ if size > max_body_bytes:
+ raise ValueError("too large; not saved.")
+
+ @staticmethod
+ def _build_browser_response(arguments: Dict, max_body_bytes: Optional[int]) -> Response:
+ if max_body_bytes is not None:
+ try:
+ arguments["content"].decode("utf-8")
+ except UnicodeDecodeError:
+ pass
+ else:
+ arguments["encoding"] = "utf-8"
+ try:
+ return Response(**arguments)
+ except (LookupError, ValueError, LxmlError):
+ if max_body_bytes is None:
+ raise
+ response = Response(**{**arguments, "content": b"", "encoding": "utf-8"})
+ response._raw_body = arguments["content"]
+ return response
@classmethod
def _process_response_history(cls, first_response: SyncResponse, parser_arguments: Dict) -> list[Response]:
@@ -94,9 +117,7 @@ class ResponseFactory:
"reason": (current_response.status_text or StatusText.get(current_response.status))
if current_response
else StatusText.get(301),
- "encoding": cls.__extract_browser_encoding(
- current_response.headers.get("content-type", "")
- )
+ "encoding": cls._extract_encoding(current_response.headers.get("content-type", ""))
if current_response
else "utf-8",
"cookies": tuple(),
@@ -127,6 +148,7 @@ class ResponseFactory:
xhr_captured: Optional[List[SyncResponse]] = None,
collect_history: bool = True,
pierce_shadow: bool = False,
+ max_body_bytes: Optional[int] = None,
) -> Response:
"""
Transforms a Playwright response into an internal `Response` object, encapsulating
@@ -146,6 +168,7 @@ class ResponseFactory:
:param xhr_captured: Optional list of captured Playwright XHR/fetch responses to convert and attach to the returned Response.
:param collect_history: Optional boolean indicating whether to collect redirections history or not.
:param pierce_shadow: Include open shadow roots in the HTML snapshot. Disabled by default.
+ :param max_body_bytes: Optional recorded-body limit. Adds a note for skipped or unreadable bodies and preserves partial headers.
:return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.
:rtype: Response
"""
@@ -154,41 +177,66 @@ class ResponseFactory:
if not final_response:
raise ValueError("Failed to get a response from the page")
- encoding = cls.__extract_browser_encoding(final_response.headers.get("content-type", ""))
+ encoding = cls._extract_encoding(final_response.headers.get("content-type", ""))
# PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(final_response.status)
history = cls._process_response_history(first_response, parser_arguments) if collect_history else []
snapshot = None
+ body_meta: Dict = {}
try:
- if page and "html" in final_response.all_headers().get("content-type", ""):
- if pierce_shadow:
- try:
- snapshot = page.evaluate(SHADOW_SNAPSHOT_JS)
- except Exception as e:
- log.warning(f"Failed to extract shadow roots, using page HTML: {e}")
- page_content = (snapshot or cls._get_page_content(page)).encode("utf-8")
- encoding = "utf-8"
+ if max_body_bytes is not None and (
+ final_response.request.method == "HEAD" or final_response.status in (204, 205, 304)
+ ):
+ page_content = b""
else:
- page_content = final_response.body()
+ cls._check_body_size(final_response.headers.get("content-length", "0"), max_body_bytes)
+ if page and "html" in final_response.all_headers().get("content-type", ""):
+ if pierce_shadow:
+ try:
+ snapshot = page.evaluate(SHADOW_SNAPSHOT_JS)
+ except Exception as e:
+ log.warning(f"Failed to extract shadow roots, using page HTML: {e}")
+ page_content = (snapshot or cls._get_page_content(page)).encode("utf-8")
+ encoding = "utf-8"
+ else:
+ page_content = final_response.body()
+ cls._check_body_size(len(page_content), max_body_bytes)
except Exception as e: # pragma: no cover
- log.error(f"Error getting page content: {e}")
+ if max_body_bytes is None:
+ log.error(f"Error getting page content: {e}")
page_content = b""
+ body_meta["body_note"] = str(e) or "could not be saved."
- response = Response(
- **{
+ try:
+ headers = first_response.all_headers()
+ except Exception:
+ if max_body_bytes is None:
+ raise
+ headers = first_response.headers
+ body_meta["headers_partial"] = True
+ try:
+ request_headers = first_response.request.all_headers()
+ except Exception:
+ if max_body_bytes is None:
+ raise
+ request_headers = first_response.request.headers
+ body_meta["request_headers_partial"] = True
+ response = cls._build_browser_response(
+ {
"url": page.url if page else first_response.url,
"content": page_content,
"status": final_response.status,
"reason": status_text,
"encoding": encoding,
"cookies": tuple(dict(cookie) for cookie in page.context.cookies()) if page else {},
- "headers": first_response.all_headers(),
- "request_headers": first_response.request.all_headers(),
+ "headers": headers,
+ "request_headers": request_headers,
"history": history,
- "meta": meta,
+ "meta": {**(meta or {}), **body_meta} if max_body_bytes is not None else meta,
**parser_arguments,
- }
+ },
+ max_body_bytes,
)
if xhr_captured:
response.captured_xhr = [
@@ -219,9 +267,7 @@ class ResponseFactory:
"reason": (current_response.status_text or StatusText.get(current_response.status))
if current_response
else StatusText.get(301),
- "encoding": cls.__extract_browser_encoding(
- current_response.headers.get("content-type", "")
- )
+ "encoding": cls._extract_encoding(current_response.headers.get("content-type", ""))
if current_response
else "utf-8",
"cookies": tuple(),
@@ -282,6 +328,7 @@ class ResponseFactory:
xhr_captured: Optional[List[AsyncResponse]] = None,
collect_history: bool = True,
pierce_shadow: bool = False,
+ max_body_bytes: Optional[int] = None,
) -> Response:
"""
Transforms a Playwright response into an internal `Response` object, encapsulating
@@ -301,6 +348,7 @@ class ResponseFactory:
:param xhr_captured: Optional list of captured async Playwright XHR/fetch responses to convert and attach to the returned Response.
:param collect_history: Optional boolean indicating whether to collect redirections history or not.
:param pierce_shadow: Include open shadow roots in the HTML snapshot. Disabled by default.
+ :param max_body_bytes: Optional recorded-body limit. Adds a note for skipped or unreadable bodies and preserves partial headers.
:return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.
:rtype: Response
@@ -310,41 +358,66 @@ class ResponseFactory:
if not final_response:
raise ValueError("Failed to get a response from the page")
- encoding = cls.__extract_browser_encoding(final_response.headers.get("content-type", ""))
+ encoding = cls._extract_encoding(final_response.headers.get("content-type", ""))
# PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(final_response.status)
history = await cls._async_process_response_history(first_response, parser_arguments) if collect_history else []
snapshot = None
+ body_meta: Dict = {}
try:
- if page and "html" in (await final_response.all_headers()).get("content-type", ""):
- if pierce_shadow:
- try:
- snapshot = await page.evaluate(SHADOW_SNAPSHOT_JS)
- except Exception as e:
- log.warning(f"Failed to extract shadow roots, using page HTML: {e}")
- page_content = (snapshot or await cls._get_async_page_content(page)).encode("utf-8")
- encoding = "utf-8"
+ if max_body_bytes is not None and (
+ final_response.request.method == "HEAD" or final_response.status in (204, 205, 304)
+ ):
+ page_content = b""
else:
- page_content = await final_response.body()
+ cls._check_body_size(final_response.headers.get("content-length", "0"), max_body_bytes)
+ if page and "html" in (await final_response.all_headers()).get("content-type", ""):
+ if pierce_shadow:
+ try:
+ snapshot = await page.evaluate(SHADOW_SNAPSHOT_JS)
+ except Exception as e:
+ log.warning(f"Failed to extract shadow roots, using page HTML: {e}")
+ page_content = (snapshot or await cls._get_async_page_content(page)).encode("utf-8")
+ encoding = "utf-8"
+ else:
+ page_content = await final_response.body()
+ cls._check_body_size(len(page_content), max_body_bytes)
except Exception as e: # pragma: no cover
- log.error(f"Error getting page content in async: {e}")
+ if max_body_bytes is None:
+ log.error(f"Error getting page content in async: {e}")
page_content = b""
+ body_meta["body_note"] = str(e) or "could not be saved."
- response = Response(
- **{
+ try:
+ headers = await first_response.all_headers()
+ except Exception:
+ if max_body_bytes is None:
+ raise
+ headers = first_response.headers
+ body_meta["headers_partial"] = True
+ try:
+ request_headers = await first_response.request.all_headers()
+ except Exception:
+ if max_body_bytes is None:
+ raise
+ request_headers = first_response.request.headers
+ body_meta["request_headers_partial"] = True
+ response = cls._build_browser_response(
+ {
"url": page.url if page else first_response.url,
"content": page_content,
"status": final_response.status,
"reason": status_text,
"encoding": encoding,
"cookies": tuple(dict(cookie) for cookie in await page.context.cookies()) if page else {},
- "headers": await first_response.all_headers(),
- "request_headers": await first_response.request.all_headers(),
+ "headers": headers,
+ "request_headers": request_headers,
"history": history,
- "meta": meta,
+ "meta": {**(meta or {}), **body_meta} if max_body_bytes is not None else meta,
**parser_arguments,
- }
+ },
+ max_body_bytes,
)
if xhr_captured:
response.captured_xhr = [
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index ff2f244..729f2b0 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -52,6 +52,8 @@ class Response(Selector):
method: str = "GET",
history: List | None = None,
meta: Dict[str, Any] | None = None,
+ *,
+ _log: bool = True,
**selector_config: Any,
):
if isinstance(content, str):
@@ -70,8 +72,8 @@ class Response(Selector):
encoding=encoding,
**selector_config,
)
- # For easier debugging while working from a Python shell
- log.info(f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})")
+ if _log:
+ log.info(f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})")
if meta and not isinstance(meta, dict):
raise TypeError(f"Response meta should be dictionary but got {type(meta).__name__} instead!")