🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat: adding the `retries` argument for all methods of `Fetcher` class
Karim shoair299793af3f36
diff --git a/README.md b/README.mdindex 8191284..7477a14 100644--- a/README.md+++ b/README.md@@ -236,7 +236,7 @@ Also, the `Response` object returned from all fetchers is the same as the `Adapt### FetcherThis class is built on top of [httpx](https://www.python-httpx.org/) with additional configuration options, here you can do `GET`, `POST`, `PUT`, and `DELETE` requests.-For all methods, you have `stealth_headers` which makes `Fetcher` create and use real browser's headers then create a referer header as if this request came from Google's search of this URL's domain. It's enabled by default.+For all methods, you have `stealth_headers` which makes `Fetcher` create and use real browser's headers then create a referer header as if this request came from Google's search of this URL's domain. It's enabled by default. You can also set the number of retries with the argument `retries` for all methods and this will make httpx retry requests if it failed for any reason. The default number of retries for all `Fetcher` methods is 3.You can route all traffic (HTTP and HTTPS) to a proxy for any of these methods in this format `http://username:password@localhost:8030````pythondiff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex 557c6b9..4329ec4 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -8,7 +8,7 @@ from .toolbelt import Response, generate_convincing_referer, generate_headersclass StaticEngine:- def __init__(self, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, adaptor_arguments: Dict = None):+ def __init__(self, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, retries: Optional[int] = 3, adaptor_arguments: Dict = None):"""An engine that utilizes httpx library, check the `Fetcher` class for more documentation.:param follow_redirects: As the name says -- if enabled (default), redirects will be followed.@@ -17,6 +17,7 @@ class StaticEngine:"""self.timeout = timeoutself.follow_redirects = bool(follow_redirects)+ self.retries = retriesself._extra_headers = generate_headers(browser_mode=False)self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}@@ -75,7 +76,7 @@ class StaticEngine::return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""headers = self._headers_job(kwargs.pop('headers', {}), url, stealthy_headers)- with httpx.Client(proxy=proxy) as client:+ with httpx.Client(proxy=proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:request = client.get(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)return self._prepare_response(request)@@ -91,7 +92,7 @@ class StaticEngine::return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""headers = self._headers_job(kwargs.pop('headers', {}), url, stealthy_headers)- with httpx.Client(proxy=proxy) as client:+ with httpx.Client(proxy=proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:request = client.post(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)return self._prepare_response(request)@@ -107,7 +108,7 @@ class StaticEngine::return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""headers = self._headers_job(kwargs.pop('headers', {}), url, stealthy_headers)- with httpx.Client(proxy=proxy) as client:+ with httpx.Client(proxy=proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:request = client.delete(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)return self._prepare_response(request)@@ -123,7 +124,7 @@ class StaticEngine::return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""headers = self._headers_job(kwargs.pop('headers', {}), url, stealthy_headers)- with httpx.Client(proxy=proxy) as client:+ with httpx.Client(proxy=proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:request = client.put(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)return self._prepare_response(request)diff --git a/scrapling/fetchers.py b/scrapling/fetchers.pyindex f0f3c02..1983fb8 100644--- a/scrapling/fetchers.py+++ b/scrapling/fetchers.py@@ -10,7 +10,9 @@ class Fetcher(BaseFetcher):Any additional keyword arguments passed to the methods below are passed to the respective httpx's method directly."""- def get(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, proxy: Optional[str] = None, **kwargs: Dict) -> Response:+ def get(+ self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,+ proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:"""Make basic HTTP GET request for you but with some added flavors.:param url: Target url.@@ -19,13 +21,16 @@ class Fetcher(BaseFetcher)::param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers andcreate a referer header as if this request had came from Google's search of this URL's domain.:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`+ :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.:param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""- response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).get(url, proxy, stealthy_headers, **kwargs)+ response_object = StaticEngine(follow_redirects, timeout, retries, adaptor_arguments=self.adaptor_arguments).get(url, proxy, stealthy_headers, **kwargs)return response_object- def post(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, proxy: Optional[str] = None, **kwargs: Dict) -> Response:+ def post(+ self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,+ proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:"""Make basic HTTP POST request for you but with some added flavors.:param url: Target url.@@ -34,13 +39,16 @@ class Fetcher(BaseFetcher)::param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers andcreate a referer header as if this request came from Google's search of this URL's domain.:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`+ :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.:param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""- response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).post(url, proxy, stealthy_headers, **kwargs)+ response_object = StaticEngine(follow_redirects, timeout, retries, adaptor_arguments=self.adaptor_arguments).post(url, proxy, stealthy_headers, **kwargs)return response_object- def put(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, proxy: Optional[str] = None, **kwargs: Dict) -> Response:+ def put(+ self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,+ proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:"""Make basic HTTP PUT request for you but with some added flavors.:param url: Target url@@ -49,14 +57,17 @@ class Fetcher(BaseFetcher)::param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers andcreate a referer header as if this request came from Google's search of this URL's domain.:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`+ :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.:param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""- response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).put(url, proxy, stealthy_headers, **kwargs)+ response_object = StaticEngine(follow_redirects, timeout, retries, adaptor_arguments=self.adaptor_arguments).put(url, proxy, stealthy_headers, **kwargs)return response_object- def delete(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, proxy: Optional[str] = None, **kwargs: Dict) -> Response:+ def delete(+ self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,+ proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:"""Make basic HTTP DELETE request for you but with some added flavors.:param url: Target url@@ -65,10 +76,13 @@ class Fetcher(BaseFetcher)::param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers andcreate a referer header as if this request came from Google's search of this URL's domain.:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`+ :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.:param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""- response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).delete(url, proxy, stealthy_headers, **kwargs)+ response_object = StaticEngine(follow_redirects, timeout, retries, adaptor_arguments=self.adaptor_arguments).delete(url, proxy, stealthy_headers, **kwargs)+ return response_object+return response_object