🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor: replace's Selector inpt (text/body) with 1 argument called `content`
Karim shoairb9c7a5af2e81
diff --git a/benchmarks.py b/benchmarks.pyindex 0dc451f..1ff696c 100644--- a/benchmarks.py+++ b/benchmarks.py@@ -80,7 +80,7 @@ def test_scrapling():@benchmarkdef test_parsel():- return Selector(text=large_html).css(".item::text").extract()+ return Selector(content=large_html).css(".item::text").extract()@benchmarkdiff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.pyindex 6cee891..dd0e8b1 100644--- a/scrapling/engines/toolbelt/convertor.py+++ b/scrapling/engines/toolbelt/convertor.py@@ -34,8 +34,7 @@ class ResponseFactory:Response(url=current_request.url,# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"- text="",- body=b"",+ content="",status=current_response.status if current_response else 301,reason=(current_response.status_text@@ -112,8 +111,7 @@ class ResponseFactory:return Response(url=page.url,- text=page_content,- body=page_content.encode("utf-8"),+ content=page_content,status=final_response.status,reason=status_text,encoding=encoding,@@ -141,8 +139,7 @@ class ResponseFactory:Response(url=current_request.url,# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"- text="",- body=b"",+ content="",status=current_response.status if current_response else 301,reason=(current_response.status_text@@ -221,8 +218,7 @@ class ResponseFactory:return Response(url=page.url,- text=page_content,- body=page_content.encode("utf-8"),+ content=page_content,status=final_response.status,reason=status_text,encoding=encoding,@@ -243,8 +239,7 @@ class ResponseFactory:"""return Response(url=response.url,- text=response.text,- body=response.content+ content=response.contentif type(response.content) is byteselse response.content.encode(),status=response.status_code,diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex 0416308..bc7ce15 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -103,8 +103,7 @@ class Response(Selector):def __init__(self,url: str,- text: str,- body: bytes,+ content: str | bytes,status: int,reason: str,cookies: Union[Tuple[Dict[str, str], ...], Dict[str, str]],@@ -122,10 +121,11 @@ class Response(Selector):self.headers = headersself.request_headers = request_headersself.history = history or []- encoding = ResponseEncoding.get_value(encoding, text)+ encoding = ResponseEncoding.get_value(+ encoding, content.decode("utf-8") if isinstance(content, bytes) else content+ )super().__init__(- text=text,- body=body,+ content=content,url=adaptive_domain or url,encoding=encoding,**selector_config,diff --git a/scrapling/parser.py b/scrapling/parser.pyindex 41c1031..128d8d1 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -50,9 +50,8 @@ class Selector(SelectorsGeneration):def __init__(self,- text: Optional[str] = None,+ content: Optional[Union[str, bytes]] = None,url: Optional[str] = None,- body: bytes = b"",encoding: str = "utf8",huge_tree: bool = True,root: Optional[html.HtmlElement] = None,@@ -72,9 +71,8 @@ class Selector(SelectorsGeneration):not possible. You can test it here and see code explodes with `AssertionError: invalid Element proxy at...`.It's an old issue with lxml, see `this entry <https://bugs.launchpad.net/lxml/+bug/736708>`- :param text: HTML body passed as text.+ :param content: HTML content as either string or bytes.:param url: It allows storing a URL with the HTML data for retrieving later.- :param body: HTML body as an ``bytes`` object. It can be used instead of the ``text`` argument.:param encoding: The encoding type that will be used in HTML parsing, default is `UTF-8`:param huge_tree: Enabled by default, should always be enabled when parsing large HTML documents. This controlsthe libxml2 feature that forbids parsing certain large documents to protect from possible memory exhaustion.@@ -88,27 +86,23 @@ class Selector(SelectorsGeneration)::param storage_args: A dictionary of ``argument->value`` pairs to be passed for the storage class.If empty, default values will be used."""- if root is None and not body and text is None:+ if root is None and content is None:raise ValueError(- "Selector class needs text, body, or root arguments to work"+ "Selector class needs HTML content, or root arguments to work")self.__text = ""if root is None:- if text is None:- if not body or not isinstance(body, bytes):- raise TypeError(- f"body argument must be valid and of type bytes, got {body.__class__}"- )-- body = body.replace(b"\x00", b"").strip()+ if isinstance(content, bytes):+ body = content.replace(b"\x00", b"").strip()+ elif isinstance(content, str):+ body = (+ content.strip().replace("\x00", "").encode(encoding) or b"<html/>"+ )else:- if not isinstance(text, str):- raise TypeError(- f"text argument must be of type str, got {text.__class__}"- )-- body = text.strip().replace("\x00", "").encode(encoding) or b"<html/>"+ raise TypeError(+ f"content argument must be str or bytes, got {type(content)}"+ )# https://lxml.de/api/lxml.etree.HTMLParser-class.htmlparser = html.HTMLParser(@@ -122,8 +116,10 @@ class Selector(SelectorsGeneration):strip_cdata=(not keep_cdata),)self._root = etree.fromstring(body, parser=parser, base_url=url)- if is_jsonable(text or body.decode()):- self.__text = TextHandler(text or body.decode())++ jsonable_text = content if isinstance(content, str) else body.decode()+ if is_jsonable(jsonable_text):+ self.__text = TextHandler(jsonable_text)else:# All HTML types inherit from HtmlMixin so this to check for all at once@@ -930,7 +926,7 @@ class Selector(SelectorsGeneration):) -> None:"""Saves the element's unique properties to the storage for retrieval and relocation later- :param element: The element itself that we want to save to storage, it can be an ` Selector ` or pure ` HtmlElement `+ :param element: The element itself that we want to save to storage, it can be a ` Selector ` or pure ` HtmlElement `:param identifier: This is the identifier that will be used to retrieve the element later from the storage. Seethe docs for more info."""diff --git a/tests/parser/test_general.py b/tests/parser/test_general.pyindex 0fbac33..266e72a 100644--- a/tests/parser/test_general.py+++ b/tests/parser/test_general.py@@ -173,10 +173,7 @@ class TestErrorHandling:_ = Selector(root="ayo", adaptive=False)with pytest.raises(TypeError):- _ = Selector(text=1, adaptive=False)-- with pytest.raises(TypeError):- _ = Selector(body=1, adaptive=False)+ _ = Selector(content=1, adaptive=False)def test_invalid_storage(self, page, html_content):"""Test invalid storage parameter"""