🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor/fix(Fetcher engine): Improvements and fixes
Karim shoairb5c7a8c36ddc
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex a58aa39..86d6c68 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -10,7 +10,7 @@ from .toolbelt import Response, generate_convincing_referer, generate_headers@lru_cache(typed=True)class StaticEngine:def __init__(- self, url: str, proxy: Optional[str] = None, stealthy_headers: Optional[bool] = True, follow_redirects: bool = True,+ self, url: str, proxy: Optional[str] = None, stealthy_headers: bool = True, follow_redirects: bool = True,timeout: Optional[Union[int, float]] = None, retries: Optional[int] = 3, adaptor_arguments: Tuple = None):"""An engine that utilizes httpx library, check the `Fetcher` class for more documentation.@@ -79,17 +79,25 @@ class StaticEngine:**self.adaptor_arguments)+ def _make_request(self, method: str, **kwargs) -> Response:+ headers = self._headers_job(kwargs.pop('headers', {}))+ with httpx.Client(proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:+ request = getattr(client, method)(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)+ return self._prepare_response(request)++ async def _async_make_request(self, method: str, **kwargs) -> Response:+ headers = self._headers_job(kwargs.pop('headers', {}))+ async with httpx.AsyncClient(proxy=self.proxy, transport=httpx.AsyncHTTPTransport(retries=self.retries)) as client:+ request = await getattr(client, method)(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)+ return self._prepare_response(request)+def get(self, **kwargs: Dict) -> Response:"""Make basic HTTP GET request for you but with some added flavors.:param kwargs: Any keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""- headers = self._headers_job(kwargs.pop('headers', {}))- with httpx.Client(proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:- request = client.get(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)-- return self._prepare_response(request)+ return self._make_request('get', **kwargs)async def async_get(self, **kwargs: Dict) -> Response:"""Make basic async HTTP GET request for you but with some added flavors.@@ -97,11 +105,7 @@ class StaticEngine::param kwargs: Any keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""- headers = self._headers_job(kwargs.pop('headers', {}))- async with httpx.AsyncClient(proxy=self.proxy) as client:- request = await client.get(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)-- return self._prepare_response(request)+ return await self._async_make_request('get', **kwargs)def post(self, **kwargs: Dict) -> Response:"""Make basic HTTP POST request for you but with some added flavors.@@ -109,11 +113,7 @@ class StaticEngine::param kwargs: Any keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""- headers = self._headers_job(kwargs.pop('headers', {}))- with httpx.Client(proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:- request = client.post(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)-- return self._prepare_response(request)+ return self._make_request('post', **kwargs)async def async_post(self, **kwargs: Dict) -> Response:"""Make basic async HTTP POST request for you but with some added flavors.@@ -121,11 +121,7 @@ class StaticEngine::param kwargs: Any keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""- headers = self._headers_job(kwargs.pop('headers', {}))- async with httpx.AsyncClient(proxy=self.proxy) as client:- request = await client.post(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)-- return self._prepare_response(request)+ return await self._async_make_request('post', **kwargs)def delete(self, **kwargs: Dict) -> Response:"""Make basic HTTP DELETE request for you but with some added flavors.@@ -133,11 +129,7 @@ class StaticEngine::param kwargs: Any keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""- headers = self._headers_job(kwargs.pop('headers', {}))- with httpx.Client(proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:- request = client.delete(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)-- return self._prepare_response(request)+ return self._make_request('delete', **kwargs)async def async_delete(self, **kwargs: Dict) -> Response:"""Make basic async HTTP DELETE request for you but with some added flavors.@@ -145,11 +137,7 @@ class StaticEngine::param kwargs: Any keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""- headers = self._headers_job(kwargs.pop('headers', {}))- async with httpx.AsyncClient(proxy=self.proxy) as client:- request = await client.delete(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)-- return self._prepare_response(request)+ return await self._async_make_request('delete', **kwargs)def put(self, **kwargs: Dict) -> Response:"""Make basic HTTP PUT request for you but with some added flavors.@@ -157,11 +145,7 @@ class StaticEngine::param kwargs: Any keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""- headers = self._headers_job(kwargs.pop('headers', {}))- with httpx.Client(proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:- request = client.put(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)-- return self._prepare_response(request)+ return self._make_request('put', **kwargs)async def async_put(self, **kwargs: Dict) -> Response:"""Make basic async HTTP PUT request for you but with some added flavors.@@ -169,8 +153,4 @@ class StaticEngine::param kwargs: Any keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""- headers = self._headers_job(kwargs.pop('headers', {}))- async with httpx.AsyncClient(proxy=self.proxy) as client:- request = await client.put(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)-- return self._prepare_response(request)+ return await self._async_make_request('put', **kwargs)