🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(network): catch text-only background requests
Karim shoaird225c5f4f14c
diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.pyindex a1ffc80..14e4151 100644--- a/scrapling/engines/toolbelt/convertor.py+++ b/scrapling/engines/toolbelt/convertor.py@@ -68,6 +68,23 @@ class ResponseFactory:message["content-type"] = content_type or ""return collapse_rfc2231_value(message.get_param("charset") or default).strip("'\"") or default+ @staticmethod+ def _text_content(headers: Dict[str, str]) -> bool:+ content_type = headers.get("content-type", "").partition(";")[0].strip().lower()+ return (+ content_type.startswith("text/")+ or content_type.endswith(("+json", "+xml"))+ or content_type+ in {+ "application/json",+ "application/xml",+ "application/javascript",+ "application/x-javascript",+ "application/graphql",+ "application/x-www-form-urlencoded",+ }+ )+@staticmethoddef _check_body_size(size: int | str, max_body_bytes: Optional[int]) -> None:if max_body_bytes is not None:@@ -168,7 +185,7 @@ class ResponseFactory::param xhr_captured: Optional list of captured Playwright XHR/fetch responses to convert and attach to the returned Response.:param collect_history: Optional boolean indicating whether to collect redirections history or not.:param pierce_shadow: Include open shadow roots in the HTML snapshot. Disabled by default.- :param max_body_bytes: Optional recorded-body limit. Adds a note for skipped or unreadable bodies and preserves partial headers.+ :param max_body_bytes: Optional text-only recorded-body limit. Skips non-text or unknown content types before reading; preserves headers and notes for unavailable bodies.:return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.:rtype: Response"""@@ -190,6 +207,8 @@ class ResponseFactory:):page_content = b""else:+ if max_body_bytes is not None and not cls._text_content(final_response.headers):+ raise ValueError("Non-text body; not saved.")cls._check_body_size(final_response.headers.get("content-length", "0"), max_body_bytes)if page and "html" in final_response.all_headers().get("content-type", ""):if pierce_shadow:@@ -348,7 +367,7 @@ class ResponseFactory::param xhr_captured: Optional list of captured async Playwright XHR/fetch responses to convert and attach to the returned Response.:param collect_history: Optional boolean indicating whether to collect redirections history or not.:param pierce_shadow: Include open shadow roots in the HTML snapshot. Disabled by default.- :param max_body_bytes: Optional recorded-body limit. Adds a note for skipped or unreadable bodies and preserves partial headers.+ :param max_body_bytes: Optional text-only recorded-body limit. Skips non-text or unknown content types before reading; preserves headers and notes for unavailable bodies.:return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.:rtype: Response@@ -371,6 +390,8 @@ class ResponseFactory:):page_content = b""else:+ if max_body_bytes is not None and not cls._text_content(final_response.headers):+ raise ValueError("Non-text body; not saved.")cls._check_body_size(final_response.headers.get("content-length", "0"), max_body_bytes)if page and "html" in (await final_response.all_headers()).get("content-type", ""):if pierce_shadow: