Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(response): Force the body of the response to always be bytes

Karim shoair05f62fba8186
diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py
index 9dff73d..4b4caa2 100644
--- a/scrapling/engines/toolbelt/convertor.py
+++ b/scrapling/engines/toolbelt/convertor.py
@@ -117,12 +117,12 @@ class ResponseFactory:
history = cls._process_response_history(first_response, parser_arguments)
try:
if "html" in final_response.all_headers().get("content-type", ""):
- page_content = cls._get_page_content(page)
+ page_content = cls._get_page_content(page).encode("utf-8")
else:
page_content = final_response.body()
except Exception as e: # pragma: no cover
log.error(f"Error getting page content: {e}")
- page_content = ""
+ page_content = b""
return Response(
**{
@@ -254,12 +254,12 @@ class ResponseFactory:
history = await cls._async_process_response_history(first_response, parser_arguments)
try:
if "html" in (await final_response.all_headers()).get("content-type", ""):
- page_content = await cls._get_async_page_content(page)
+ page_content = (await cls._get_async_page_content(page)).encode("utf-8")
else:
page_content = await final_response.body()
except Exception as e: # pragma: no cover
log.error(f"Error getting page content in async: {e}")
- page_content = ""
+ page_content = b""
return Response(
**{
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index 29b9e17..58a7485 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -14,6 +14,7 @@ from scrapling.core._types import (
Union,
Optional,
Callable,
+ Sequence,
TYPE_CHECKING,
AsyncGenerator,
)
@@ -42,6 +43,9 @@ class Response(Selector):
meta: Dict[str, Any] | None = None,
**selector_config: Any,
):
+ if isinstance(content, str):
+ content = content.encode("utf-8")
+
adaptive_domain: str = cast(str, selector_config.pop("adaptive_domain", ""))
self.status = status
self.reason = reason
@@ -64,6 +68,11 @@ class Response(Selector):
self.meta: Dict[str, Any] = meta or {}
self.request: Optional["Request"] = None # Will be set by crawler
+ @property
+ def body(self) -> bytes:
+ """Return the raw body of the response as bytes."""
+ return cast(bytes, cast(Sequence, self._raw_body))
+
def follow(
self,
url: str,