🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Calculate status text manually if it's not returned by PlayWright API
Karim shoair19ad82caef95
diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.pyindex 2f8a1f9..e55d951 100644--- a/scrapling/engines/camo.py+++ b/scrapling/engines/camo.py@@ -4,6 +4,7 @@ from scrapling.core._types import Union, Callable, Optional, Dict, List, Literalfrom scrapling.engines.toolbelt import (Response,do_nothing,+ StatusText,get_os_name,intercept_route,check_type_validity,@@ -111,12 +112,17 @@ class CamoufoxEngine:if 'charset=' in content_type.lower():encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip()+ status_text = res.status_text+ # PlayWright API sometimes give empty status text for some reason!+ if not status_text:+ status_text = StatusText.get(res.status)+response = Response(url=res.url,text=page.content(),body=page.content().encode('utf-8'),status=res.status,- reason=res.status_text,+ reason=status_text,encoding=encoding,cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},headers=res.all_headers(),diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.pyindex c83bfe3..b049090 100644--- a/scrapling/engines/pw.py+++ b/scrapling/engines/pw.py@@ -6,6 +6,7 @@ from scrapling.engines.constants import DEFAULT_STEALTH_FLAGS, NSTBROWSER_DEFAULfrom scrapling.engines.toolbelt import (Response,do_nothing,+ StatusText,js_bypass_path,intercept_route,generate_headers,@@ -221,12 +222,17 @@ class PlaywrightEngine:if 'charset=' in content_type.lower():encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip()+ status_text = res.status_text+ # PlayWright API sometimes give empty status text for some reason!+ if not status_text:+ status_text = StatusText.get(res.status)+response = Response(url=res.url,text=page.content(),body=page.content().encode('utf-8'),status=res.status,- reason=res.status_text,+ reason=status_text,encoding=encoding,cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},headers=res.all_headers(),diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.pyindex ac3e03d..15fd80c 100644--- a/scrapling/engines/toolbelt/__init__.py+++ b/scrapling/engines/toolbelt/__init__.py@@ -6,6 +6,7 @@ from .fingerprints import (from .custom import (Response,do_nothing,+ StatusText,BaseFetcher,get_variable_name,check_type_validity,diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex de34099..8957d1a 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -4,8 +4,9 @@ Functions related to custom types or type checkingimport inspectimport logging-from scrapling.core.utils import setup_basic_logging+from scrapling.core.custom_types import MappingProxyTypefrom scrapling.parser import Adaptor, SQLiteStorageSystem+from scrapling.core.utils import setup_basic_logging, cachefrom scrapling.core._types import Any, List, Type, Union, Optional, Dict, Callable@@ -67,6 +68,83 @@ class BaseFetcher:self.adaptor_arguments.update({'automatch_domain': automatch_domain})+class StatusText:+ """A class that gets the status text of response status code.++ Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status+ """+ _phrases = MappingProxyType({+ 100: "Continue",+ 101: "Switching Protocols",+ 102: "Processing",+ 103: "Early Hints",+ 200: "OK",+ 201: "Created",+ 202: "Accepted",+ 203: "Non-Authoritative Information",+ 204: "No Content",+ 205: "Reset Content",+ 206: "Partial Content",+ 207: "Multi-Status",+ 208: "Already Reported",+ 226: "IM Used",+ 300: "Multiple Choices",+ 301: "Moved Permanently",+ 302: "Found",+ 303: "See Other",+ 304: "Not Modified",+ 305: "Use Proxy",+ 307: "Temporary Redirect",+ 308: "Permanent Redirect",+ 400: "Bad Request",+ 401: "Unauthorized",+ 402: "Payment Required",+ 403: "Forbidden",+ 404: "Not Found",+ 405: "Method Not Allowed",+ 406: "Not Acceptable",+ 407: "Proxy Authentication Required",+ 408: "Request Timeout",+ 409: "Conflict",+ 410: "Gone",+ 411: "Length Required",+ 412: "Precondition Failed",+ 413: "Payload Too Large",+ 414: "URI Too Long",+ 415: "Unsupported Media Type",+ 416: "Range Not Satisfiable",+ 417: "Expectation Failed",+ 418: "I'm a teapot",+ 421: "Misdirected Request",+ 422: "Unprocessable Entity",+ 423: "Locked",+ 424: "Failed Dependency",+ 425: "Too Early",+ 426: "Upgrade Required",+ 428: "Precondition Required",+ 429: "Too Many Requests",+ 431: "Request Header Fields Too Large",+ 451: "Unavailable For Legal Reasons",+ 500: "Internal Server Error",+ 501: "Not Implemented",+ 502: "Bad Gateway",+ 503: "Service Unavailable",+ 504: "Gateway Timeout",+ 505: "HTTP Version Not Supported",+ 506: "Variant Also Negotiates",+ 507: "Insufficient Storage",+ 508: "Loop Detected",+ 510: "Not Extended",+ 511: "Network Authentication Required"+ })++ @classmethod+ @cache(maxsize=128)+ def get(cls, status_code: int) -> str:+ """Get the phrase for a given HTTP status code."""+ return cls._phrases.get(status_code, "Unknown Status Code")++def check_if_engine_usable(engine: Callable) -> Union[Callable, None]:"""This function check if the passed engine can be used by a Fetcher-type class or not.