🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Correcting the return string for the `Response` object in all functions/methods
Karim shoair90e38af55258
diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.pyindex 19d8726..4131151 100644--- a/scrapling/engines/camo.py+++ b/scrapling/engines/camo.py@@ -72,7 +72,7 @@ class CamoufoxEngine:"""Opens up the browser and do your request based on your chosen options.:param url: Target url.- :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.+ :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""with Camoufox(proxy=self.proxy,diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.pyindex b97bcac..6c80900 100644--- a/scrapling/engines/pw.py+++ b/scrapling/engines/pw.py@@ -116,7 +116,7 @@ class PlaywrightEngine:"""Opens up the browser and do your request based on your chosen options.:param url: Target url.- :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.+ :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""if not self.stealth:from playwright.sync_api import sync_playwrightdiff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex ffc8822..d424752 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -48,7 +48,7 @@ class StaticEngine:"""Takes httpx response and generates `Response` object from it.:param response: httpx response object- :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.+ :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""return Response(url=str(response.url),@@ -69,7 +69,7 @@ class StaticEngine::param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers andcreate a referer header as if this request had came from Google's search of this URL's domain.:param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.- :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.+ :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""headers = self._headers_job(kwargs.pop('headers', {}), url, stealthy_headers)request = httpx.get(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)@@ -81,7 +81,7 @@ class StaticEngine::param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers andcreate a referer header as if this request had came from Google's search of this URL's domain.:param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.- :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.+ :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""headers = self._headers_job(kwargs.pop('headers', {}), url, stealthy_headers)request = httpx.post(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)@@ -93,7 +93,7 @@ class StaticEngine::param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers andcreate a referer header as if this request had came from Google's search of this URL's domain.:param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.- :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.+ :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""headers = self._headers_job(kwargs.pop('headers', {}), url, stealthy_headers)request = httpx.delete(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)@@ -105,7 +105,7 @@ class StaticEngine::param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers andcreate a referer header as if this request had came from Google's search of this URL's domain.:param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.- :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.+ :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""headers = self._headers_job(kwargs.pop('headers', {}), url, stealthy_headers)request = httpx.put(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)diff --git a/scrapling/fetchers.py b/scrapling/fetchers.pyindex a63edcb..294baca 100644--- a/scrapling/fetchers.py+++ b/scrapling/fetchers.py@@ -17,7 +17,7 @@ class Fetcher(BaseFetcher)::param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers andcreate a referer header as if this request had came from Google's search of this URL's domain.:param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.- :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.+ :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).get(url, stealthy_headers, **kwargs)return response_object@@ -30,7 +30,7 @@ class Fetcher(BaseFetcher)::param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers andcreate a referer header as if this request came from Google's search of this URL's domain.:param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.- :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.+ :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).post(url, stealthy_headers, **kwargs)return response_object@@ -43,7 +43,7 @@ class Fetcher(BaseFetcher)::param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers andcreate a referer header as if this request came from Google's search of this URL's domain.:param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.- :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.+ :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).put(url, stealthy_headers, **kwargs)return response_object@@ -56,7 +56,7 @@ class Fetcher(BaseFetcher)::param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers andcreate a referer header as if this request came from Google's search of this URL's domain.:param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.- :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.+ :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).delete(url, stealthy_headers, **kwargs)return response_object@@ -97,7 +97,7 @@ class StealthyFetcher(BaseFetcher)::param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.- :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.+ :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""engine = CamoufoxEngine(proxy=proxy,@@ -167,7 +167,7 @@ class PlayWrightFetcher(BaseFetcher)::param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.:param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.:param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.- :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.+ :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""engine = PlaywrightEngine(proxy=proxy,