🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(response): Force the body of the response to always be bytes
Karim shoair05f62fba8186
diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.pyindex 9dff73d..4b4caa2 100644--- a/scrapling/engines/toolbelt/convertor.py+++ b/scrapling/engines/toolbelt/convertor.py@@ -117,12 +117,12 @@ class ResponseFactory:history = cls._process_response_history(first_response, parser_arguments)try:if "html" in final_response.all_headers().get("content-type", ""):- page_content = cls._get_page_content(page)+ page_content = cls._get_page_content(page).encode("utf-8")else:page_content = final_response.body()except Exception as e: # pragma: no coverlog.error(f"Error getting page content: {e}")- page_content = ""+ page_content = b""return Response(**{@@ -254,12 +254,12 @@ class ResponseFactory:history = await cls._async_process_response_history(first_response, parser_arguments)try:if "html" in (await final_response.all_headers()).get("content-type", ""):- page_content = await cls._get_async_page_content(page)+ page_content = (await cls._get_async_page_content(page)).encode("utf-8")else:page_content = await final_response.body()except Exception as e: # pragma: no coverlog.error(f"Error getting page content in async: {e}")- page_content = ""+ page_content = b""return Response(**{diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex 29b9e17..58a7485 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -14,6 +14,7 @@ from scrapling.core._types import (Union,Optional,Callable,+ Sequence,TYPE_CHECKING,AsyncGenerator,)@@ -42,6 +43,9 @@ class Response(Selector):meta: Dict[str, Any] | None = None,**selector_config: Any,):+ if isinstance(content, str):+ content = content.encode("utf-8")+adaptive_domain: str = cast(str, selector_config.pop("adaptive_domain", ""))self.status = statusself.reason = reason@@ -64,6 +68,11 @@ class Response(Selector):self.meta: Dict[str, Any] = meta or {}self.request: Optional["Request"] = None # Will be set by crawler+ @property+ def body(self) -> bytes:+ """Return the raw body of the response as bytes."""+ return cast(bytes, cast(Sequence, self._raw_body))+def follow(self,url: str,