Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

Calculate status text manually if it's not returned by PlayWright API

Karim shoair19ad82caef95
diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index 2f8a1f9..e55d951 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -4,6 +4,7 @@ from scrapling.core._types import Union, Callable, Optional, Dict, List, Literal
from scrapling.engines.toolbelt import (
Response,
do_nothing,
+ StatusText,
get_os_name,
intercept_route,
check_type_validity,
@@ -111,12 +112,17 @@ class CamoufoxEngine:
if 'charset=' in content_type.lower():
encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip()
+ status_text = res.status_text
+ # PlayWright API sometimes give empty status text for some reason!
+ if not status_text:
+ status_text = StatusText.get(res.status)
+
response = Response(
url=res.url,
text=page.content(),
body=page.content().encode('utf-8'),
status=res.status,
- reason=res.status_text,
+ reason=status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
headers=res.all_headers(),
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index c83bfe3..b049090 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -6,6 +6,7 @@ from scrapling.engines.constants import DEFAULT_STEALTH_FLAGS, NSTBROWSER_DEFAUL
from scrapling.engines.toolbelt import (
Response,
do_nothing,
+ StatusText,
js_bypass_path,
intercept_route,
generate_headers,
@@ -221,12 +222,17 @@ class PlaywrightEngine:
if 'charset=' in content_type.lower():
encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip()
+ status_text = res.status_text
+ # PlayWright API sometimes give empty status text for some reason!
+ if not status_text:
+ status_text = StatusText.get(res.status)
+
response = Response(
url=res.url,
text=page.content(),
body=page.content().encode('utf-8'),
status=res.status,
- reason=res.status_text,
+ reason=status_text,
encoding=encoding,
cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
headers=res.all_headers(),
diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.py
index ac3e03d..15fd80c 100644
--- a/scrapling/engines/toolbelt/__init__.py
+++ b/scrapling/engines/toolbelt/__init__.py
@@ -6,6 +6,7 @@ from .fingerprints import (
from .custom import (
Response,
do_nothing,
+ StatusText,
BaseFetcher,
get_variable_name,
check_type_validity,
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index de34099..8957d1a 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -4,8 +4,9 @@ Functions related to custom types or type checking
import inspect
import logging
-from scrapling.core.utils import setup_basic_logging
+from scrapling.core.custom_types import MappingProxyType
from scrapling.parser import Adaptor, SQLiteStorageSystem
+from scrapling.core.utils import setup_basic_logging, cache
from scrapling.core._types import Any, List, Type, Union, Optional, Dict, Callable
@@ -67,6 +68,83 @@ class BaseFetcher:
self.adaptor_arguments.update({'automatch_domain': automatch_domain})
+class StatusText:
+ """A class that gets the status text of response status code.
+
+ Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status
+ """
+ _phrases = MappingProxyType({
+ 100: "Continue",
+ 101: "Switching Protocols",
+ 102: "Processing",
+ 103: "Early Hints",
+ 200: "OK",
+ 201: "Created",
+ 202: "Accepted",
+ 203: "Non-Authoritative Information",
+ 204: "No Content",
+ 205: "Reset Content",
+ 206: "Partial Content",
+ 207: "Multi-Status",
+ 208: "Already Reported",
+ 226: "IM Used",
+ 300: "Multiple Choices",
+ 301: "Moved Permanently",
+ 302: "Found",
+ 303: "See Other",
+ 304: "Not Modified",
+ 305: "Use Proxy",
+ 307: "Temporary Redirect",
+ 308: "Permanent Redirect",
+ 400: "Bad Request",
+ 401: "Unauthorized",
+ 402: "Payment Required",
+ 403: "Forbidden",
+ 404: "Not Found",
+ 405: "Method Not Allowed",
+ 406: "Not Acceptable",
+ 407: "Proxy Authentication Required",
+ 408: "Request Timeout",
+ 409: "Conflict",
+ 410: "Gone",
+ 411: "Length Required",
+ 412: "Precondition Failed",
+ 413: "Payload Too Large",
+ 414: "URI Too Long",
+ 415: "Unsupported Media Type",
+ 416: "Range Not Satisfiable",
+ 417: "Expectation Failed",
+ 418: "I'm a teapot",
+ 421: "Misdirected Request",
+ 422: "Unprocessable Entity",
+ 423: "Locked",
+ 424: "Failed Dependency",
+ 425: "Too Early",
+ 426: "Upgrade Required",
+ 428: "Precondition Required",
+ 429: "Too Many Requests",
+ 431: "Request Header Fields Too Large",
+ 451: "Unavailable For Legal Reasons",
+ 500: "Internal Server Error",
+ 501: "Not Implemented",
+ 502: "Bad Gateway",
+ 503: "Service Unavailable",
+ 504: "Gateway Timeout",
+ 505: "HTTP Version Not Supported",
+ 506: "Variant Also Negotiates",
+ 507: "Insufficient Storage",
+ 508: "Loop Detected",
+ 510: "Not Extended",
+ 511: "Network Authentication Required"
+ })
+
+ @classmethod
+ @cache(maxsize=128)
+ def get(cls, status_code: int) -> str:
+ """Get the phrase for a given HTTP status code."""
+ return cls._phrases.get(status_code, "Unknown Status Code")
+
+
def check_if_engine_usable(engine: Callable) -> Union[Callable, None]:
"""This function check if the passed engine can be used by a Fetcher-type class or not.