🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
perf: Give repeated usage of `Fetcher` a slight performance increase
Karim shoair445af3c702d1
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex 4329ec4..69f54e6 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -1,14 +1,15 @@import httpxfrom httpx._models import Response as httpxResponse-from scrapling.core._types import Dict, Optional, Union-from scrapling.core.utils import log+from scrapling.core._types import Dict, Optional, Tuple, Union+from scrapling.core.utils import log, lru_cachefrom .toolbelt import Response, generate_convincing_referer, generate_headers+@lru_cache(typed=True)class StaticEngine:- def __init__(self, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, retries: Optional[int] = 3, adaptor_arguments: Dict = None):+ def __init__(self, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, retries: Optional[int] = 3, adaptor_arguments: Tuple = None):"""An engine that utilizes httpx library, check the `Fetcher` class for more documentation.:param follow_redirects: As the name says -- if enabled (default), redirects will be followed.@@ -19,7 +20,9 @@ class StaticEngine:self.follow_redirects = bool(follow_redirects)self.retries = retriesself._extra_headers = generate_headers(browser_mode=False)- self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}+ # Because we are using `lru_cache` for a slight optimization but both dict/dict_items are not hashable so they can't be cached+ # So my solution here was to convert it to tuple then convert it back to dictionary again here as tuples are hashable, ofc `tuple().__hash__()`+ self.adaptor_arguments = dict(adaptor_arguments) if adaptor_arguments else {}@staticmethoddef _headers_job(headers: Optional[Dict], url: str, stealth: bool) -> Dict:diff --git a/scrapling/fetchers.py b/scrapling/fetchers.pyindex 1983fb8..61332c6 100644--- a/scrapling/fetchers.py+++ b/scrapling/fetchers.py@@ -25,7 +25,8 @@ class Fetcher(BaseFetcher)::param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""- response_object = StaticEngine(follow_redirects, timeout, retries, adaptor_arguments=self.adaptor_arguments).get(url, proxy, stealthy_headers, **kwargs)+ adaptor_arguments = tuple(self.adaptor_arguments.items())+ response_object = StaticEngine(follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).get(url, proxy, stealthy_headers, **kwargs)return response_objectdef post(@@ -43,7 +44,8 @@ class Fetcher(BaseFetcher)::param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""- response_object = StaticEngine(follow_redirects, timeout, retries, adaptor_arguments=self.adaptor_arguments).post(url, proxy, stealthy_headers, **kwargs)+ adaptor_arguments = tuple(self.adaptor_arguments.items())+ response_object = StaticEngine(follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).post(url, proxy, stealthy_headers, **kwargs)return response_objectdef put(@@ -62,7 +64,8 @@ class Fetcher(BaseFetcher)::return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""- response_object = StaticEngine(follow_redirects, timeout, retries, adaptor_arguments=self.adaptor_arguments).put(url, proxy, stealthy_headers, **kwargs)+ adaptor_arguments = tuple(self.adaptor_arguments.items())+ response_object = StaticEngine(follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).put(url, proxy, stealthy_headers, **kwargs)return response_objectdef delete(@@ -80,7 +83,8 @@ class Fetcher(BaseFetcher)::param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`"""- response_object = StaticEngine(follow_redirects, timeout, retries, adaptor_arguments=self.adaptor_arguments).delete(url, proxy, stealthy_headers, **kwargs)+ adaptor_arguments = tuple(self.adaptor_arguments.items())+ response_object = StaticEngine(follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).delete(url, proxy, stealthy_headers, **kwargs)return response_objectreturn response_object