🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Fixing the way `Response` object handles sub items in some edge cases
Karim shoair572df6b3b489
diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.pyindex 4131151..62fdd39 100644--- a/scrapling/engines/camo.py+++ b/scrapling/engines/camo.py@@ -114,14 +114,14 @@ class CamoufoxEngine:response = Response(url=res.url,text=page.content(),- content=res.body(),+ body=res.body(),status=res.status,reason=res.status_text,encoding=encoding,cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},headers=res.all_headers(),request_headers=res.request.all_headers(),- adaptor_arguments=self.adaptor_arguments+ **self.adaptor_arguments)page.close()diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.pyindex 6c80900..3395ece 100644--- a/scrapling/engines/pw.py+++ b/scrapling/engines/pw.py@@ -224,14 +224,14 @@ class PlaywrightEngine:response = Response(url=res.url,text=page.content(),- content=res.body(),+ body=res.body(),status=res.status,reason=res.status_text,encoding=encoding,cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},headers=res.all_headers(),request_headers=res.request.all_headers(),- adaptor_arguments=self.adaptor_arguments+ **self.adaptor_arguments)page.close()return responsediff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex d424752..c106332 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -53,14 +53,14 @@ class StaticEngine:return Response(url=str(response.url),text=response.text,- content=response.content,+ body=response.content,status=response.status_code,reason=response.reason_phrase,encoding=response.encoding or 'utf-8',cookies=dict(response.cookies),headers=dict(response.headers),request_headers=dict(response.request.headers),- adaptor_arguments=self.adaptor_arguments+ **self.adaptor_arguments)def get(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex 274654c..025efda 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -12,15 +12,14 @@ from scrapling.core._types import Any, List, Type, Union, Optional, Dict, Callabclass Response(Adaptor):"""This class is returned by all engines as a way to unify response type between different libraries."""- def __init__(self, url: str, text: str, content: bytes, status: int, reason: str, cookies: Dict, headers: Dict, request_headers: Dict, adaptor_arguments: Dict, encoding: str = 'utf-8'):+ def __init__(self, url: str, text: str, body: bytes, status: int, reason: str, cookies: Dict, headers: Dict, request_headers: Dict, encoding: str = 'utf-8', **adaptor_arguments: Dict):automatch_domain = adaptor_arguments.pop('automatch_domain', None)- super().__init__(text=text, body=content, url=automatch_domain or url, encoding=encoding, **adaptor_arguments)-self.status = statusself.reason = reasonself.cookies = cookiesself.headers = headersself.request_headers = request_headers+ super().__init__(text=text, body=body, url=automatch_domain or url, encoding=encoding, **adaptor_arguments)# For back-ward compatibilityself.adaptor = selfdiff --git a/scrapling/parser.py b/scrapling/parser.pyindex 8d03dda..79cfa14 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -32,6 +32,7 @@ class Adaptor(SelectorsGeneration):storage: Any = SQLiteStorageSystem,storage_args: Optional[Dict] = None,debug: Optional[bool] = True,+ **kwargs):"""The main class that works as a wrapper for the HTML input data. Using this class, you can search for elementswith expressions in CSS, XPath, or with simply text. Check the docs for more info.@@ -117,6 +118,10 @@ class Adaptor(SelectorsGeneration):self.__attributes = Noneself.__tag = Noneself.__debug = debug+ # No need to check if all response attributes exist or not because if `status` exist, then the rest exist (Save some CPU cycles for speed)+ self.__response_data = {+ key: getattr(self, key) for key in ('status', 'reason', 'cookies', 'headers', 'request_headers',)+ } if hasattr(self, 'status') else {}# Node functionalities, I wanted to move to separate Mixin class but it had slight impact on performance@staticmethod@@ -138,10 +143,14 @@ class Adaptor(SelectorsGeneration):return TextHandler(str(element))else:if issubclass(type(element), html.HtmlMixin):+return self.__class__(- root=element, url=self.url, encoding=self.encoding, auto_match=self.__auto_match_enabled,+ root=element,+ text='', body=b'', # Since root argument is provided, both `text` and `body` will be ignored so this is just a filler+ url=self.url, encoding=self.encoding, auto_match=self.__auto_match_enabled,keep_comments=True, # if the comments are already removed in initialization, no need to try to delete them in sub-elements- huge_tree=self.__huge_tree_enabled, debug=self.__debug+ huge_tree=self.__huge_tree_enabled, debug=self.__debug,+ **self.__response_data)return element